diff --git a/.github/workflows/cultmesh-portability.yml b/.github/workflows/cultmesh-portability.yml new file mode 100644 index 00000000..ce361fc8 --- /dev/null +++ b/.github/workflows/cultmesh-portability.yml @@ -0,0 +1,97 @@ +name: CultMesh portability and performance + +on: + push: + branches: + - main + - "codex/**" + pull_request: + workflow_dispatch: + +jobs: + getting-started: + name: ${{ matrix.os }} one-command multiplayer onboarding + runs-on: ${{ matrix.os }} + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + os: [windows-latest, ubuntu-latest] + + steps: + - name: Check out CultLib + uses: actions/checkout@v4 + with: + path: CultLib + + - name: Check out the matching Eve contract + uses: actions/checkout@v4 + with: + repository: GameCult/Eve + # Eve main is the contract this sample is written against; the codex + # branch this used to pin was never merged and stopped on 2026-08-25. + ref: ${{ vars.EVE_INTEGRATION_REF || 'main' }} + path: Eve + + - name: Set up .NET + uses: actions/setup-dotnet@v4 + with: + dotnet-version: 10.0.x + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: 24 + + - name: Run the documented one-command checkpoint + shell: pwsh + run: ./CultLib/scripts/verify-eve-getting-started.ps1 -EveRoot ./Eve + + reactive-and-resource-lifetime: + name: ${{ matrix.os }} reactive semantics and leases + runs-on: ${{ matrix.os }} + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + os: [windows-latest, ubuntu-latest] + + steps: + - name: Check out CultLib + uses: actions/checkout@v4 + + - name: Set up .NET + uses: actions/setup-dotnet@v4 + with: + dotnet-version: 10.0.x + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: 24 + cache: npm + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install TypeScript workspace dependencies + run: npm install --no-package-lock + + - name: Install Python CultLib packages + run: python -m pip install -e packages/cultcache-py -e packages/cultnet-py -e packages/cultmesh-py + + - name: Verify explicit reactive scheduling across runtimes + shell: pwsh + run: ./scripts/verify-reactive-document-scaling.ps1 + + - name: Measure representative reactive workload + shell: pwsh + run: ./scripts/measure-reactive-document-performance.ps1 -Quick -SkipRuntimeBuild + + - name: Churn 100,000 distinct document leases + run: >- + dotnet test tests/GameCult.Mesh.Tests/GameCult.Mesh.Tests.csproj + --filter FullyQualifiedName~Document_OneHundredThousandDistinctLeaseCyclesLeaveNoResourcesOrCallbacks + --logger "console;verbosity=minimal" diff --git a/.github/workflows/cultnet-interop.yml b/.github/workflows/cultnet-interop.yml index 1bde67ef..6876a76a 100644 --- a/.github/workflows/cultnet-interop.yml +++ b/.github/workflows/cultnet-interop.yml @@ -33,10 +33,6 @@ jobs: uses: actions/setup-node@v4 with: node-version: 24 - cache: npm - cache-dependency-path: | - packages/cultcache-ts/package-lock.json - packages/cultnet-ts/package-lock.json - name: Set up Python uses: actions/setup-python@v5 @@ -73,7 +69,7 @@ jobs: run: npm install --no-package-lock - name: Install Python package dependencies - run: python -m pip install -e packages/cultcache-py + run: python -m pip install -e packages/cultcache-py -e packages/cultnet-py -e packages/cultmesh-py - name: Build TypeScript workspace dependencies run: npm run build --workspace packages/cultcache-ts @@ -87,6 +83,24 @@ jobs: npm run build --workspace packages/cultnet-ts npx tsc -p packages/cultnet-ts/tsconfig.test.json --pretty false + - name: Verify TypeScript RUDP reliability + run: npm test --workspace packages/cultnet-ts + + - name: Verify Python RUDP reliability + run: | + python -m unittest discover -s packages/cultcache-py/tests + python -m unittest discover -s packages/cultnet-py/tests + python -m unittest discover -s packages/cultmesh-py/tests + + - name: Verify Rust RUDP reliability + run: cargo test --manifest-path packages/cultnet-rs/Cargo.toml rudp + + - name: Verify Rust CultMesh runtime-local surface + run: cargo test --manifest-path packages/cultmesh-rs/Cargo.toml + + - name: Verify C# RUDP reliability + run: dotnet test tests/GameCult.Networking.Tests/GameCult.Networking.Tests.csproj --filter "FullyQualifiedName~Rudp" + - name: Interop RUDP Python lane timeout-minutes: 8 run: node --test --test-name-pattern "TypeScript and Python|Python and TypeScript" packages/cultnet-ts/dist-test/test/interop/cultnet-interop.test.js diff --git a/.github/workflows/publish-packages.yml b/.github/workflows/publish-packages.yml new file mode 100644 index 00000000..6061ca7a --- /dev/null +++ b/.github/workflows/publish-packages.yml @@ -0,0 +1,146 @@ +name: Publish packages + +# Non-Rust consumers cannot take a package from a subdirectory of a git repo the +# way cargo can, which is why they ended up vendoring forks instead. This gives +# them a versioned artifact to depend on so CultLib stays the only source. +# +# Rust needs nothing here: cargo git dependencies pin a CultLib revision +# directly, which is already how Ghostlight, Epiphany, CodexConnector and Odin +# consume the core stack. +# +# Tag a release to publish, for example `cultcache-ts-v0.13.6` or +# `cultnet-py-v0.1.0`. The tag's version must match the version in the package +# manifest; the workflow refuses the publish otherwise rather than shipping a +# mislabelled artifact. + +on: + push: + tags: + - "cultcache-ts-v*" + - "cultcache-py-v*" + - "cultnet-py-v*" + - "cultmesh-py-v*" + workflow_dispatch: + inputs: + package: + description: Package to build and test (publishes nothing on manual runs) + required: true + type: choice + options: + - cultcache-ts + - cultcache-py + - cultnet-py + - cultmesh-py + +permissions: + contents: read + +jobs: + cultcache-ts: + name: "@gamecult/cultcache-ts to npm" + if: startsWith(github.ref, 'refs/tags/cultcache-ts-v') || inputs.package == 'cultcache-ts' + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - name: Check out CultLib + uses: actions/checkout@v4 + + - name: Set up Node + uses: actions/setup-node@v4 + with: + node-version: 24.x + registry-url: https://registry.npmjs.org + + - name: Install workspace dependencies + run: npm ci + + - name: Verify the tag matches the manifest version + if: startsWith(github.ref, 'refs/tags/cultcache-ts-v') + run: | + manifest="$(node -p "require('./packages/cultcache-ts/package.json').version")" + tagged="${GITHUB_REF_NAME#cultcache-ts-v}" + if [ "$manifest" != "$tagged" ]; then + echo "tag $GITHUB_REF_NAME does not match manifest version $manifest" >&2 + exit 1 + fi + + - name: Build and test + working-directory: packages/cultcache-ts + run: npm test + + # Scoped packages default to restricted even on public npm, and this is + # MIT like the rest of the repo, so say public explicitly. + - name: Publish + if: startsWith(github.ref, 'refs/tags/cultcache-ts-v') + working-directory: packages/cultcache-ts + run: npm publish --access public + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + + # The three Python packages layer cultcache-py < cultnet-py < cultmesh-py, the + # same way the Rust crates and npm packages do. One job shape serves all + # three; the matrix entry names the package and the tag prefix that releases it. + python: + name: ${{ matrix.package }} to PyPI + if: startsWith(github.ref, 'refs/tags/cultcache-py-v') || startsWith(github.ref, 'refs/tags/cultnet-py-v') || startsWith(github.ref, 'refs/tags/cultmesh-py-v') || endsWith(inputs.package, '-py') + strategy: + fail-fast: false + matrix: + package: [cultcache-py, cultnet-py, cultmesh-py] + runs-on: ubuntu-latest + timeout-minutes: 20 + environment: pypi + permissions: + contents: read + # PyPI trusted publishing exchanges this for a short-lived token, so no + # long-lived API token is stored in the repository. + id-token: write + steps: + - name: Skip packages this run does not name + id: selected + run: | + if [ "${GITHUB_REF_NAME#${{ matrix.package }}-v}" != "$GITHUB_REF_NAME" ] || [ "${{ inputs.package }}" = "${{ matrix.package }}" ]; then + echo "run=true" >> "$GITHUB_OUTPUT" + else + echo "run=false" >> "$GITHUB_OUTPUT" + fi + + - name: Check out CultLib + if: steps.selected.outputs.run == 'true' + uses: actions/checkout@v4 + + - name: Set up Python + if: steps.selected.outputs.run == 'true' + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Verify the tag matches the manifest version + if: steps.selected.outputs.run == 'true' && startsWith(github.ref, 'refs/tags/') + run: | + manifest="$(python -c "import tomllib,pathlib;print(tomllib.loads(pathlib.Path('packages/${{ matrix.package }}/pyproject.toml').read_text())['project']['version'])")" + tagged="${GITHUB_REF_NAME#${{ matrix.package }}-v}" + if [ "$manifest" != "$tagged" ]; then + echo "tag $GITHUB_REF_NAME does not match manifest version $manifest" >&2 + exit 1 + fi + + # Lower layers install from source so a release of one package is tested + # against the sibling revision it was cut with, not whatever PyPI has. + - name: Install and test + if: steps.selected.outputs.run == 'true' + run: | + python -m pip install -e packages/cultcache-py -e packages/cultnet-py -e packages/cultmesh-py + python -m unittest discover -s packages/${{ matrix.package }}/tests + + - name: Build sdist and wheel + if: steps.selected.outputs.run == 'true' + run: | + python -m pip install --upgrade build + python -m build packages/${{ matrix.package }} + + - name: Publish + if: steps.selected.outputs.run == 'true' && startsWith(github.ref, 'refs/tags/') + uses: pypa/gh-action-pypi-publish@release/v1 + with: + packages-dir: packages/${{ matrix.package }}/dist diff --git a/.gitignore b/.gitignore index 2fdb5151..e4661501 100644 --- a/.gitignore +++ b/.gitignore @@ -206,17 +206,6 @@ PublishScripts/ *.snupkg # The packages folder can be ignored because of Package Restore **/[Pp]ackages/* -!/packages/gamecult-geometry-rs/ -!/packages/gamecult-geometry-rs/** -packages/gamecult-geometry-rs/target/ -.tools/ -!/packages/gamecult-geometry-ts/ -!/packages/gamecult-geometry-ts/** -!/unity/org.gamecult.geometry/Samples~/ -!/unity/org.gamecult.geometry/Samples~/** -!/unity/org.gamecult.geometry/Runtime/PlanetaryPatchMeshAdapter.cs.meta -!/unity/org.gamecult.geometry/Runtime/PlanetaryPageUpload.cs.meta -!/src/GameCult.Geometry/Shaders/*.hlsl.meta # except build/, which is used as an MSBuild target. !**/[Pp]ackages/build/ # Uncomment if necessary however generally it will be regenerated when needed @@ -252,8 +241,6 @@ _pkginfo.txt ClientBin/ ~$* *~ -!/unity/org.gamecult.geometry/Samples~/ -!/unity/org.gamecult.geometry/Samples~/** *.dbmdl *.dbproj.schemaview *.jfm @@ -316,10 +303,22 @@ node_modules/ !packages/cultmesh-ts/** !packages/cultcache-py/ !packages/cultcache-py/** +!packages/cultnet-py/ +!packages/cultnet-py/** +!packages/cultmesh-py/ +!packages/cultmesh-py/** !packages/cultcache-rs/ !packages/cultcache-rs/** !packages/cultnet-rs/ !packages/cultnet-rs/** +!packages/cultmesh-rs/ +!packages/cultmesh-rs/** +!packages/cultmath/ +!packages/cultmath/** +!packages/cultmesh-browser/ +!packages/cultmesh-browser/** +!packages/cultmesh-kotlin/ +!packages/cultmesh-kotlin/** packages/**/node_modules/ packages/**/dist/ packages/**/dist-test/ @@ -517,3 +516,7 @@ $RECYCLE.BIN/ # Vim temporary swap files *.swp + +# Python package build output +packages/*-py/build/ +*.egg-info/ diff --git a/.voidbot/status/void-memory-maintenance.json b/.voidbot/status/void-memory-maintenance.json deleted file mode 100644 index 0d0b876a..00000000 --- a/.voidbot/status/void-memory-maintenance.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "status": "starting", - "startedAt": "2026-06-14T19:46:12.715Z", - "stateFile": "E:\\Projects\\CultLib\\.voidbot\\state\\libby.cc", - "launcher": "repo_face_heartbeat", - "reason": "repo_face_stale_short_term_memory" -} diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 00000000..640c1a14 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,3 @@ +Repository doctrine for this project lives in AGENTS.md, shared with Codex. + +@AGENTS.md diff --git a/CultLib.sln b/CultLib.sln index 65f940c8..fbc20922 100644 --- a/CultLib.sln +++ b/CultLib.sln @@ -11,10 +11,6 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "GameCult.Caching", "src\Gam EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "GameCult.Caching.MessagePack", "src\GameCult.Caching.MessagePack\GameCult.Caching.MessagePack.csproj", "{26EE4C38-F06E-4E7A-A89C-908318BCB0DB}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "GameCult.Caching.MessagePack.Generator", "src\GameCult.Caching.MessagePack.Generator\GameCult.Caching.MessagePack.Generator.csproj", "{0E990C96-3808-450E-9349-A0C3262195ED}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "GameCult.Caching.MessagePack.Analyzers", "src\GameCult.Caching.MessagePack.Analyzers\GameCult.Caching.MessagePack.Analyzers.csproj", "{DBE06954-EC85-4034-9049-72C7D27556DF}" -EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "GameCult.Networking", "src\GameCult.Networking\GameCult.Networking.csproj", "{67383F6C-C9C3-4C5B-95D4-5C50C20EDD4F}" EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "GameCult.Mesh", "src\GameCult.Mesh\GameCult.Mesh.csproj", "{8D1B01A7-BF9E-44E6-88E1-B17255728B42}" @@ -29,14 +25,30 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "GameCult.Geometry", "src\Ga EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "GameCult.Geometry.Tests", "tests\GameCult.Geometry.Tests\GameCult.Geometry.Tests.csproj", "{C95A9A21-34FA-4D82-BE5B-88FBDA836D31}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "GameCult.Geometry.Planetary.Tests", "tests\GameCult.Geometry.Planetary.Tests\GameCult.Geometry.Planetary.Tests.csproj", "{CE447820-B407-48DE-8957-F373E032C4AE}" -EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "GameCult.Mesh.Tests", "tests\GameCult.Mesh.Tests\GameCult.Mesh.Tests.csproj", "{F8A5060A-FB0B-4B8B-AE06-5147BB4CA127}" EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "GameCult.Mesh.Quic", "src\GameCult.Mesh.Quic\GameCult.Mesh.Quic.csproj", "{B67042B8-1135-4E54-B1EB-D45342C720D7}" EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "GameCult.Mesh.Quic.Tests", "tests\GameCult.Mesh.Quic.Tests\GameCult.Mesh.Quic.Tests.csproj", "{21F26329-2119-4037-93CC-DD1801A1AB78}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "GameCult.Networking.WebSockets", "src\GameCult.Networking.WebSockets\GameCult.Networking.WebSockets.csproj", "{7F32D96B-AB5D-4A72-9217-B5DCCFEFC7E5}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "GameCult.Networking.WebSockets.Tests", "tests\GameCult.Networking.WebSockets.Tests\GameCult.Networking.WebSockets.Tests.csproj", "{B9A441EC-DADE-4500-B092-C635F3EB296A}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "samples", "samples", "{5D20AA90-6969-D8BD-9DCD-8634F4692FDA}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "EveBrowserNetworkSample", "samples\eve-browser-network\EveBrowserNetworkSample.csproj", "{5FA81CC7-87B2-4585-9D8E-9BBEC0AD85FB}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "packages", "packages", "{809F86A1-1C4C-B159-0CD4-DF9D33D876CE}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "cultmath", "cultmath", "{B48EAE30-152F-01E3-6E31-82B350494D3C}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CultMath", "packages\cultmath\src\CultMath\CultMath.csproj", "{54D543F7-8EB7-4A11-977E-22D3EE2D8C3F}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CultMath.Tests", "packages\cultmath\tests\CultMath.Tests\CultMath.Tests.csproj", "{FEF9DA88-2177-4FD0-885D-7096ACFA8A9D}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "GameCult.Caching.ConsumerBuild", "tests\GameCult.Caching.ConsumerBuild\GameCult.Caching.ConsumerBuild.csproj", "{D3138F3C-9531-4906-A5D3-74BD857D0685}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -83,30 +95,6 @@ Global {26EE4C38-F06E-4E7A-A89C-908318BCB0DB}.Release|x64.Build.0 = Release|Any CPU {26EE4C38-F06E-4E7A-A89C-908318BCB0DB}.Release|x86.ActiveCfg = Release|Any CPU {26EE4C38-F06E-4E7A-A89C-908318BCB0DB}.Release|x86.Build.0 = Release|Any CPU - {0E990C96-3808-450E-9349-A0C3262195ED}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {0E990C96-3808-450E-9349-A0C3262195ED}.Debug|Any CPU.Build.0 = Debug|Any CPU - {0E990C96-3808-450E-9349-A0C3262195ED}.Debug|x64.ActiveCfg = Debug|Any CPU - {0E990C96-3808-450E-9349-A0C3262195ED}.Debug|x64.Build.0 = Debug|Any CPU - {0E990C96-3808-450E-9349-A0C3262195ED}.Debug|x86.ActiveCfg = Debug|Any CPU - {0E990C96-3808-450E-9349-A0C3262195ED}.Debug|x86.Build.0 = Debug|Any CPU - {0E990C96-3808-450E-9349-A0C3262195ED}.Release|Any CPU.ActiveCfg = Release|Any CPU - {0E990C96-3808-450E-9349-A0C3262195ED}.Release|Any CPU.Build.0 = Release|Any CPU - {0E990C96-3808-450E-9349-A0C3262195ED}.Release|x64.ActiveCfg = Release|Any CPU - {0E990C96-3808-450E-9349-A0C3262195ED}.Release|x64.Build.0 = Release|Any CPU - {0E990C96-3808-450E-9349-A0C3262195ED}.Release|x86.ActiveCfg = Release|Any CPU - {0E990C96-3808-450E-9349-A0C3262195ED}.Release|x86.Build.0 = Release|Any CPU - {DBE06954-EC85-4034-9049-72C7D27556DF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {DBE06954-EC85-4034-9049-72C7D27556DF}.Debug|Any CPU.Build.0 = Debug|Any CPU - {DBE06954-EC85-4034-9049-72C7D27556DF}.Debug|x64.ActiveCfg = Debug|Any CPU - {DBE06954-EC85-4034-9049-72C7D27556DF}.Debug|x64.Build.0 = Debug|Any CPU - {DBE06954-EC85-4034-9049-72C7D27556DF}.Debug|x86.ActiveCfg = Debug|Any CPU - {DBE06954-EC85-4034-9049-72C7D27556DF}.Debug|x86.Build.0 = Debug|Any CPU - {DBE06954-EC85-4034-9049-72C7D27556DF}.Release|Any CPU.ActiveCfg = Release|Any CPU - {DBE06954-EC85-4034-9049-72C7D27556DF}.Release|Any CPU.Build.0 = Release|Any CPU - {DBE06954-EC85-4034-9049-72C7D27556DF}.Release|x64.ActiveCfg = Release|Any CPU - {DBE06954-EC85-4034-9049-72C7D27556DF}.Release|x64.Build.0 = Release|Any CPU - {DBE06954-EC85-4034-9049-72C7D27556DF}.Release|x86.ActiveCfg = Release|Any CPU - {DBE06954-EC85-4034-9049-72C7D27556DF}.Release|x86.Build.0 = Release|Any CPU {67383F6C-C9C3-4C5B-95D4-5C50C20EDD4F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {67383F6C-C9C3-4C5B-95D4-5C50C20EDD4F}.Debug|Any CPU.Build.0 = Debug|Any CPU {67383F6C-C9C3-4C5B-95D4-5C50C20EDD4F}.Debug|x64.ActiveCfg = Debug|Any CPU @@ -179,18 +167,6 @@ Global {C95A9A21-34FA-4D82-BE5B-88FBDA836D31}.Release|x64.Build.0 = Release|Any CPU {C95A9A21-34FA-4D82-BE5B-88FBDA836D31}.Release|x86.ActiveCfg = Release|Any CPU {C95A9A21-34FA-4D82-BE5B-88FBDA836D31}.Release|x86.Build.0 = Release|Any CPU - {CE447820-B407-48DE-8957-F373E032C4AE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {CE447820-B407-48DE-8957-F373E032C4AE}.Debug|Any CPU.Build.0 = Debug|Any CPU - {CE447820-B407-48DE-8957-F373E032C4AE}.Debug|x64.ActiveCfg = Debug|Any CPU - {CE447820-B407-48DE-8957-F373E032C4AE}.Debug|x64.Build.0 = Debug|Any CPU - {CE447820-B407-48DE-8957-F373E032C4AE}.Debug|x86.ActiveCfg = Debug|Any CPU - {CE447820-B407-48DE-8957-F373E032C4AE}.Debug|x86.Build.0 = Debug|Any CPU - {CE447820-B407-48DE-8957-F373E032C4AE}.Release|Any CPU.ActiveCfg = Release|Any CPU - {CE447820-B407-48DE-8957-F373E032C4AE}.Release|Any CPU.Build.0 = Release|Any CPU - {CE447820-B407-48DE-8957-F373E032C4AE}.Release|x64.ActiveCfg = Release|Any CPU - {CE447820-B407-48DE-8957-F373E032C4AE}.Release|x64.Build.0 = Release|Any CPU - {CE447820-B407-48DE-8957-F373E032C4AE}.Release|x86.ActiveCfg = Release|Any CPU - {CE447820-B407-48DE-8957-F373E032C4AE}.Release|x86.Build.0 = Release|Any CPU {F8A5060A-FB0B-4B8B-AE06-5147BB4CA127}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {F8A5060A-FB0B-4B8B-AE06-5147BB4CA127}.Debug|Any CPU.Build.0 = Debug|Any CPU {F8A5060A-FB0B-4B8B-AE06-5147BB4CA127}.Debug|x64.ActiveCfg = Debug|Any CPU @@ -227,6 +203,78 @@ Global {21F26329-2119-4037-93CC-DD1801A1AB78}.Release|x64.Build.0 = Release|Any CPU {21F26329-2119-4037-93CC-DD1801A1AB78}.Release|x86.ActiveCfg = Release|Any CPU {21F26329-2119-4037-93CC-DD1801A1AB78}.Release|x86.Build.0 = Release|Any CPU + {7F32D96B-AB5D-4A72-9217-B5DCCFEFC7E5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {7F32D96B-AB5D-4A72-9217-B5DCCFEFC7E5}.Debug|Any CPU.Build.0 = Debug|Any CPU + {7F32D96B-AB5D-4A72-9217-B5DCCFEFC7E5}.Debug|x64.ActiveCfg = Debug|Any CPU + {7F32D96B-AB5D-4A72-9217-B5DCCFEFC7E5}.Debug|x64.Build.0 = Debug|Any CPU + {7F32D96B-AB5D-4A72-9217-B5DCCFEFC7E5}.Debug|x86.ActiveCfg = Debug|Any CPU + {7F32D96B-AB5D-4A72-9217-B5DCCFEFC7E5}.Debug|x86.Build.0 = Debug|Any CPU + {7F32D96B-AB5D-4A72-9217-B5DCCFEFC7E5}.Release|Any CPU.ActiveCfg = Release|Any CPU + {7F32D96B-AB5D-4A72-9217-B5DCCFEFC7E5}.Release|Any CPU.Build.0 = Release|Any CPU + {7F32D96B-AB5D-4A72-9217-B5DCCFEFC7E5}.Release|x64.ActiveCfg = Release|Any CPU + {7F32D96B-AB5D-4A72-9217-B5DCCFEFC7E5}.Release|x64.Build.0 = Release|Any CPU + {7F32D96B-AB5D-4A72-9217-B5DCCFEFC7E5}.Release|x86.ActiveCfg = Release|Any CPU + {7F32D96B-AB5D-4A72-9217-B5DCCFEFC7E5}.Release|x86.Build.0 = Release|Any CPU + {B9A441EC-DADE-4500-B092-C635F3EB296A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {B9A441EC-DADE-4500-B092-C635F3EB296A}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B9A441EC-DADE-4500-B092-C635F3EB296A}.Debug|x64.ActiveCfg = Debug|Any CPU + {B9A441EC-DADE-4500-B092-C635F3EB296A}.Debug|x64.Build.0 = Debug|Any CPU + {B9A441EC-DADE-4500-B092-C635F3EB296A}.Debug|x86.ActiveCfg = Debug|Any CPU + {B9A441EC-DADE-4500-B092-C635F3EB296A}.Debug|x86.Build.0 = Debug|Any CPU + {B9A441EC-DADE-4500-B092-C635F3EB296A}.Release|Any CPU.ActiveCfg = Release|Any CPU + {B9A441EC-DADE-4500-B092-C635F3EB296A}.Release|Any CPU.Build.0 = Release|Any CPU + {B9A441EC-DADE-4500-B092-C635F3EB296A}.Release|x64.ActiveCfg = Release|Any CPU + {B9A441EC-DADE-4500-B092-C635F3EB296A}.Release|x64.Build.0 = Release|Any CPU + {B9A441EC-DADE-4500-B092-C635F3EB296A}.Release|x86.ActiveCfg = Release|Any CPU + {B9A441EC-DADE-4500-B092-C635F3EB296A}.Release|x86.Build.0 = Release|Any CPU + {5FA81CC7-87B2-4585-9D8E-9BBEC0AD85FB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {5FA81CC7-87B2-4585-9D8E-9BBEC0AD85FB}.Debug|Any CPU.Build.0 = Debug|Any CPU + {5FA81CC7-87B2-4585-9D8E-9BBEC0AD85FB}.Debug|x64.ActiveCfg = Debug|Any CPU + {5FA81CC7-87B2-4585-9D8E-9BBEC0AD85FB}.Debug|x64.Build.0 = Debug|Any CPU + {5FA81CC7-87B2-4585-9D8E-9BBEC0AD85FB}.Debug|x86.ActiveCfg = Debug|Any CPU + {5FA81CC7-87B2-4585-9D8E-9BBEC0AD85FB}.Debug|x86.Build.0 = Debug|Any CPU + {5FA81CC7-87B2-4585-9D8E-9BBEC0AD85FB}.Release|Any CPU.ActiveCfg = Release|Any CPU + {5FA81CC7-87B2-4585-9D8E-9BBEC0AD85FB}.Release|Any CPU.Build.0 = Release|Any CPU + {5FA81CC7-87B2-4585-9D8E-9BBEC0AD85FB}.Release|x64.ActiveCfg = Release|Any CPU + {5FA81CC7-87B2-4585-9D8E-9BBEC0AD85FB}.Release|x64.Build.0 = Release|Any CPU + {5FA81CC7-87B2-4585-9D8E-9BBEC0AD85FB}.Release|x86.ActiveCfg = Release|Any CPU + {5FA81CC7-87B2-4585-9D8E-9BBEC0AD85FB}.Release|x86.Build.0 = Release|Any CPU + {54D543F7-8EB7-4A11-977E-22D3EE2D8C3F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {54D543F7-8EB7-4A11-977E-22D3EE2D8C3F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {54D543F7-8EB7-4A11-977E-22D3EE2D8C3F}.Debug|x64.ActiveCfg = Debug|Any CPU + {54D543F7-8EB7-4A11-977E-22D3EE2D8C3F}.Debug|x64.Build.0 = Debug|Any CPU + {54D543F7-8EB7-4A11-977E-22D3EE2D8C3F}.Debug|x86.ActiveCfg = Debug|Any CPU + {54D543F7-8EB7-4A11-977E-22D3EE2D8C3F}.Debug|x86.Build.0 = Debug|Any CPU + {54D543F7-8EB7-4A11-977E-22D3EE2D8C3F}.Release|Any CPU.ActiveCfg = Release|Any CPU + {54D543F7-8EB7-4A11-977E-22D3EE2D8C3F}.Release|Any CPU.Build.0 = Release|Any CPU + {54D543F7-8EB7-4A11-977E-22D3EE2D8C3F}.Release|x64.ActiveCfg = Release|Any CPU + {54D543F7-8EB7-4A11-977E-22D3EE2D8C3F}.Release|x64.Build.0 = Release|Any CPU + {54D543F7-8EB7-4A11-977E-22D3EE2D8C3F}.Release|x86.ActiveCfg = Release|Any CPU + {54D543F7-8EB7-4A11-977E-22D3EE2D8C3F}.Release|x86.Build.0 = Release|Any CPU + {FEF9DA88-2177-4FD0-885D-7096ACFA8A9D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {FEF9DA88-2177-4FD0-885D-7096ACFA8A9D}.Debug|Any CPU.Build.0 = Debug|Any CPU + {FEF9DA88-2177-4FD0-885D-7096ACFA8A9D}.Debug|x64.ActiveCfg = Debug|Any CPU + {FEF9DA88-2177-4FD0-885D-7096ACFA8A9D}.Debug|x64.Build.0 = Debug|Any CPU + {FEF9DA88-2177-4FD0-885D-7096ACFA8A9D}.Debug|x86.ActiveCfg = Debug|Any CPU + {FEF9DA88-2177-4FD0-885D-7096ACFA8A9D}.Debug|x86.Build.0 = Debug|Any CPU + {FEF9DA88-2177-4FD0-885D-7096ACFA8A9D}.Release|Any CPU.ActiveCfg = Release|Any CPU + {FEF9DA88-2177-4FD0-885D-7096ACFA8A9D}.Release|Any CPU.Build.0 = Release|Any CPU + {FEF9DA88-2177-4FD0-885D-7096ACFA8A9D}.Release|x64.ActiveCfg = Release|Any CPU + {FEF9DA88-2177-4FD0-885D-7096ACFA8A9D}.Release|x64.Build.0 = Release|Any CPU + {FEF9DA88-2177-4FD0-885D-7096ACFA8A9D}.Release|x86.ActiveCfg = Release|Any CPU + {FEF9DA88-2177-4FD0-885D-7096ACFA8A9D}.Release|x86.Build.0 = Release|Any CPU + {D3138F3C-9531-4906-A5D3-74BD857D0685}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {D3138F3C-9531-4906-A5D3-74BD857D0685}.Debug|Any CPU.Build.0 = Debug|Any CPU + {D3138F3C-9531-4906-A5D3-74BD857D0685}.Debug|x64.ActiveCfg = Debug|Any CPU + {D3138F3C-9531-4906-A5D3-74BD857D0685}.Debug|x64.Build.0 = Debug|Any CPU + {D3138F3C-9531-4906-A5D3-74BD857D0685}.Debug|x86.ActiveCfg = Debug|Any CPU + {D3138F3C-9531-4906-A5D3-74BD857D0685}.Debug|x86.Build.0 = Debug|Any CPU + {D3138F3C-9531-4906-A5D3-74BD857D0685}.Release|Any CPU.ActiveCfg = Release|Any CPU + {D3138F3C-9531-4906-A5D3-74BD857D0685}.Release|Any CPU.Build.0 = Release|Any CPU + {D3138F3C-9531-4906-A5D3-74BD857D0685}.Release|x64.ActiveCfg = Release|Any CPU + {D3138F3C-9531-4906-A5D3-74BD857D0685}.Release|x64.Build.0 = Release|Any CPU + {D3138F3C-9531-4906-A5D3-74BD857D0685}.Release|x86.ActiveCfg = Release|Any CPU + {D3138F3C-9531-4906-A5D3-74BD857D0685}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -235,17 +283,21 @@ Global {72AEDD42-E881-4EB7-99A8-FD3B9667189C} = {E733CBFF-42AE-4B6A-B5B9-0071ECFB45F1} {9DDA917D-B4DB-4AC1-9B9A-4ECF06A5ABA1} = {E733CBFF-42AE-4B6A-B5B9-0071ECFB45F1} {26EE4C38-F06E-4E7A-A89C-908318BCB0DB} = {E733CBFF-42AE-4B6A-B5B9-0071ECFB45F1} - {0E990C96-3808-450E-9349-A0C3262195ED} = {E733CBFF-42AE-4B6A-B5B9-0071ECFB45F1} - {DBE06954-EC85-4034-9049-72C7D27556DF} = {E733CBFF-42AE-4B6A-B5B9-0071ECFB45F1} {67383F6C-C9C3-4C5B-95D4-5C50C20EDD4F} = {E733CBFF-42AE-4B6A-B5B9-0071ECFB45F1} {8D1B01A7-BF9E-44E6-88E1-B17255728B42} = {E733CBFF-42AE-4B6A-B5B9-0071ECFB45F1} {E2DC8FF6-D39E-4FCA-9549-1F7681F0AA81} = {39D316AB-ABB5-49A2-8E83-CAF4C75B48F4} {EB15435B-7147-4BD9-BB9C-B0A14318D3D5} = {39D316AB-ABB5-49A2-8E83-CAF4C75B48F4} {94C086DA-3359-424B-9688-29A198806481} = {E733CBFF-42AE-4B6A-B5B9-0071ECFB45F1} {C95A9A21-34FA-4D82-BE5B-88FBDA836D31} = {39D316AB-ABB5-49A2-8E83-CAF4C75B48F4} - {CE447820-B407-48DE-8957-F373E032C4AE} = {39D316AB-ABB5-49A2-8E83-CAF4C75B48F4} {F8A5060A-FB0B-4B8B-AE06-5147BB4CA127} = {39D316AB-ABB5-49A2-8E83-CAF4C75B48F4} {B67042B8-1135-4E54-B1EB-D45342C720D7} = {E733CBFF-42AE-4B6A-B5B9-0071ECFB45F1} {21F26329-2119-4037-93CC-DD1801A1AB78} = {39D316AB-ABB5-49A2-8E83-CAF4C75B48F4} + {7F32D96B-AB5D-4A72-9217-B5DCCFEFC7E5} = {E733CBFF-42AE-4B6A-B5B9-0071ECFB45F1} + {B9A441EC-DADE-4500-B092-C635F3EB296A} = {39D316AB-ABB5-49A2-8E83-CAF4C75B48F4} + {5FA81CC7-87B2-4585-9D8E-9BBEC0AD85FB} = {5D20AA90-6969-D8BD-9DCD-8634F4692FDA} + {B48EAE30-152F-01E3-6E31-82B350494D3C} = {809F86A1-1C4C-B159-0CD4-DF9D33D876CE} + {54D543F7-8EB7-4A11-977E-22D3EE2D8C3F} = {B48EAE30-152F-01E3-6E31-82B350494D3C} + {FEF9DA88-2177-4FD0-885D-7096ACFA8A9D} = {B48EAE30-152F-01E3-6E31-82B350494D3C} + {D3138F3C-9531-4906-A5D3-74BD857D0685} = {39D316AB-ABB5-49A2-8E83-CAF4C75B48F4} EndGlobalSection EndGlobal diff --git a/CultLib.txt b/CultLib.txt deleted file mode 100644 index 4755ac07..00000000 --- a/CultLib.txt +++ /dev/null @@ -1,132 +0,0 @@ - - -// E:\Projects\CultLib\src\GameCult.Caching\GameCult.Caching.csproj - - - - CultCache: Concurrent polymorphic cache with modular backing stores for .NET - - - - - - - - - - -// E:\Projects\CultLib\src\GameCult.Caching.MessagePack\GameCult.Caching.MessagePack.csproj - - - - File-based MessagePack backing store for CultCache - - - - - - - - - -// E:\Projects\CultLib\src\GameCult.Caching.MessagePack.Generator\GameCult.Caching.MessagePack.Generator.csproj - - - - - - true - enable - true - false - - - Source generator that emits MessagePack formatters for DatabaseEntry subclasses. - MessagePack;SourceGenerator;DatabaseEntry - - - - - - - - - - - - - - - - -// E:\Projects\CultLib\src\GameCult.Logging\GameCult.Logging.csproj - - - - Simple pluggable logging interface for CultLib - - - -// E:\Projects\CultLib\src\GameCult.Logging.Unity\GameCult.Logging.Unity.csproj - - - - Unity integration for GameCult.Logging - - - - - - - - -// E:\Projects\CultLib\src\GameCult.Networking\GameCult.Networking.csproj - - - - CultNet: Secure, high-performance messaging for games/servers - - - - - - - - - - - - - -// E:\Projects\CultLib\tests\GameCult.Networking.Tests\GameCult.Networking.Tests.csproj - - - - - net8.0 - false - GameCult.Networking.Tests - - - - - - - - - - runtime; build; native; contentfiles; analyzers; buildtransitive - all - - - - - - - all - - - - diff --git a/Directory.Build.props b/Directory.Build.props index a9390c76..10c8d749 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -14,9 +14,8 @@ MIT © Meta Vens and contributors true - 1.0.33 - $(MSBuildThisFileDirectory)..\CultMath\.tools\local-feed - $(RestoreAdditionalProjectSources);$(CultMathCandidateFeed) + 1.0.35 + $(CultLibPackageVersion) true diff --git a/README.md b/README.md index 80dc90a5..4977a02a 100644 --- a/README.md +++ b/README.md @@ -11,8 +11,9 @@ The stack is less a single product than a set of capabilities: - logging primitives and implementations - CultCache typed documents, `.cc` persistence, and SoA memory access -- CultNet schema-v0 messages over TCP, WebSocket, LiteNetLib adapters, and - CultLib's native cross-runtime RUDP pipe +- CultNet schema-v0 messages over TCP and LiteNetLib adapters, a bounded + ASP.NET Core WebSocket host adapter, and CultLib's native cross-runtime RUDP + pipe - CultMesh distributed realtime database, Verse discovery, authority leases, shard replication, and witness-authoritative simulation consensus - typed geometry domain/chunk documents for distributed CSG and LOD streaming @@ -37,8 +38,9 @@ and pretending the label makes it furniture. | Job | Start With | Owns | Use When | Do Not Use It For | | --- | --- | --- | --- | --- | | Local typed state | `GameCult.Caching` / CultCache | Document identity, schema compatibility, record keys, indexes, globals, and local persistence | You need a typed cache, file-compatible save data, local reactive reads, or a stable domain document model | Peer discovery, transport security, shard routing, or mesh consensus | -| Procedural geometry state | `GameCult.Geometry` | CultCache-native domain trees, LOD build requests, selected-cut diagnostics, and chunk artifact payloads | Rust, Unity, or remote workers need to share CSG/LOD geometry and graph metadata as typed state | Transport policy, peer discovery, or gameplay authority | -| Network transport and database plumbing | `GameCult.Networking` / CultNet | Native RUDP sessions, LiteNetLib/TCP/WebSocket adapters, authentication, schema-v0 wire contracts, shard authority, raw document replication, snapshots, and subscriptions | You need a client/server pipe, login/session flow, schema discovery, reliable UDP, or a low-level distributed CultCache lane | Gameplay-facing mesh ergonomics, Verse policy, mod branches, or simulation consensus composition | +| Portable math | `GameCult.Math` / CultMath | Shader-shaped numeric primitives, deterministic random, noise, and CPU/GPU parity helpers | Shared simulation and geometry code must not depend on an engine's math or RNG types | Domain-specific geometry, rendering, or world authority | +| Procedural geometry | `GameCult.Geometry` / CultGeometry | Isosurface extraction plus CultCache-native domains, LOD requests, selected cuts, and mesh artifacts | C#, Rust, Godot, or remote workers need shared geometry algorithms and typed results | Transport policy, peer discovery, or gameplay authority | +| Network transport and database plumbing | `GameCult.Networking` / CultNet | Native RUDP sessions, LiteNetLib/TCP adapters, an authenticated WebSocket host adapter, schema-v0 wire contracts, shard authority, raw document replication, snapshots, and subscriptions | You need a client/server pipe, login/session flow, schema discovery, reliable UDP, or a low-level distributed CultCache lane | Gameplay-facing mesh ergonomics, Verse policy, mod branches, or simulation consensus composition | | Distributed realtime gameplay state | `GameCult.Mesh` / CultMesh | Public mesh entrypoints, Verse discovery, peer exchange, shard replication defaults, authority leases, client prediction, and witness consensus | You want the game to treat clients and servers as one reactive database for persistent state, input state, and simulation facts | A tiny local-only tool, a bare transport client, or a storage format contract | | Realtime media/frame streams | `GameCult.Mesh` / CultMesh streaming mode | Stream identity, authority, clock metadata, body transport negotiation, frame cursors, and backpressure state | Audio/video/tensor frames need to move between runtimes through shared memory, GPU handles, platform buffers, or CultCache page refs | Durable document mutation, mesh consensus facts, or pretending inline bytes are zero-copy | | Daemon-published UI | Eve / CultUI | Typed operator and user-facing surfaces over CultMesh state | A daemon should expose controls, inspection, or workflow UI that can lower into GUI, TUI, web, native, overlay, or agent clients | One-off vendor dashboards, untyped debug panels, or runtime-local UI that cannot travel with the daemon | @@ -46,8 +48,10 @@ and pretending the label makes it furniture. Quick rule: - Choose CultCache when the problem is "how do I model and persist typed state?" -- Choose GameCult.Geometry when the problem is "how do geometry workers share - domain trees, LOD build requests, and mesh chunks as typed state?" +- Choose GameCult.Math when the problem is "which portable numeric and random + semantics do these runtimes share?" +- Choose GameCult.Geometry when the problem is "how do geometry workers build + and share engine-neutral geometry?" - Choose CultNet when the problem is "how do peers exchange authenticated, schema-aware database messages?" - Choose CultMesh when the problem is "how does a game join a Verse and share @@ -62,8 +66,25 @@ the stack. If a design needs another peer-to-peer category, first check whether it is really Verse policy, peer exchange, or shard authority wearing a fake mustache. -If you want the smallest durable "open node -> put typed doc -> get typed doc" -path, start with +If you want to build an application rather than tour the component drawer, +start with the maintained +[`CultMesh And Eve Getting Started`](src/GameCult.Mesh/docs/getting-started/README.md) +series. Its first executable package-consumer checkpoint is +[`samples/eve-two-runtime`](samples/eve-two-runtime/README.md). +The real-process browser continuation is +[`samples/eve-browser-network`](samples/eve-browser-network/README.md). +One documented command runs both layers: + +```powershell +pwsh -File ./scripts/verify-eve-getting-started.ps1 +``` + +The implemented browser client boundary and remaining deployed-infrastructure +gates are mapped explicitly in +[`browser-verse-transport.md`](src/GameCult.Mesh/docs/browser-verse-transport.md). + +If you want only the smallest durable "open node -> put typed doc -> get typed doc" +path, use [`src/GameCult.Mesh/docs/durable-node-quickstart.md`](src/GameCult.Mesh/docs/durable-node-quickstart.md). If you want the lower-level "typed local document -> raw wire document -> mesh/runtime watch surface" handoff, follow it with @@ -77,10 +98,10 @@ The solution includes: - `GameCult.Caching`: `DatabaseEntry`-based cache, indexes, global entries, and backing-store abstractions - `GameCult.Caching.MessagePack`: MessagePack-backed persistence for the cache - `GameCult.Caching.NewtonsoftJson`: Newtonsoft.Json-backed persistence for the cache -- `GameCult.Caching.MessagePack.Generator`: source generator for MessagePack formatters for cache models -- `GameCult.Caching.MessagePack.Analyzers`: packaging project that delivers the generator to consuming projects -- `GameCult.Geometry`: geometry, planetary mapping, CSG, LOD selection, mesh assembly, and CultCache-native geometry documents built on CultMath primitives +- `GameCult.Math` / CultMath: portable numeric primitives, deterministic random, noise, and shader-parity helpers +- `GameCult.Geometry` / CultGeometry: engine-neutral geometry algorithms plus CultCache-native domain, selected-cut, and mesh artifact documents - `GameCult.Networking`: encrypted login/register/verify flows, schema-v0 contracts, transport adapters, and native RUDP sessions +- `GameCult.Networking.WebSockets`: bounded ASP.NET Core binary WebSocket host adapter for browser CultMesh clients - `GameCult.Mesh`: CultMesh package home for distributed realtime database, shard replication, client prediction, Verse discovery, and mesh witness consensus - `GameCult.Caching.Tests`: NUnit tests for cache and backing-store behavior - `GameCult.Networking.Tests`: NUnit tests for networking behavior @@ -88,7 +109,10 @@ The solution includes: - `packages/cultcache-ts`: TypeScript CultCache with MessagePack persistence and inspector tooling - `packages/cultnet-ts`: TypeScript CultNet schema-v0 contracts, framing, discovery, raw document replication, and interop tests - `packages/cultmesh-ts`: TypeScript CultMesh local node and mesh catalog surface for local runtimes such as VoidBot -- `packages/cultcache-py`: Python CultCache/CultNet/CultMesh package with CultCache v1 wire parity +- `packages/cultmesh-browser`: browser-safe, read-only CultMesh document leases and typed operations over an Odin-resolved WebSocket route +- `packages/cultcache-py`: Python CultCache with CultCache v1 wire parity +- `packages/cultnet-py`: Python CultNet contracts, framing, RUDP transport, replication, and interop peer +- `packages/cultmesh-py`: Python CultMesh local node, discovery, sessions, local server, and daemon - `packages/cultcache-rs`: Rust CultCache and derive macro - `packages/cultnet-rs`: Rust CultNet contracts, framing, discovery, and interop peer - `packages/cultmesh-kotlin`: Kotlin/JVM CultMesh and CultNet surface for Android-adjacent runtimes @@ -102,10 +126,9 @@ src/ GameCult.Caching/ GameCult.Caching.MessagePack/ GameCult.Caching.NewtonsoftJson/ - GameCult.Caching.MessagePack.Generator/ - GameCult.Caching.MessagePack.Analyzers/ GameCult.Geometry/ GameCult.Networking/ + GameCult.Networking.WebSockets/ GameCult.Mesh/ GameCult.Unity/ tests/ @@ -113,11 +136,15 @@ tests/ GameCult.Geometry.Tests/ GameCult.Networking.Tests/ packages/ + cultmath/ cultcache-ts/ cultnet-ts/ cultmesh-ts/ + cultmesh-browser/ cultmesh-kotlin/ cultcache-py/ + cultnet-py/ + cultmesh-py/ cultcache-rs/ cultnet-rs/ ``` @@ -141,8 +168,22 @@ npm test --workspace packages/cultcache-ts npm test --workspace packages/cultnet-ts npm run test:interop --workspace packages/cultnet-ts npm test --workspace packages/cultmesh-ts +npm test --workspace packages/cultmesh-browser ``` +Measured reactive, retained-session, and shared-memory hot-body gates: + +```powershell +pwsh -File ./scripts/measure-reactive-document-performance.ps1 -Quick +pwsh -File ./scripts/verify-eve-getting-started.ps1 +``` + +The first command measures the same 16 KiB document workload in C#, +TypeScript, and Python and includes a mapped hot-body probe with explicit copy +telemetry on Windows named maps and Unix file-backed maps. The second includes +10,000 real WebSocket operations on one retained +identity-owned C# session after provider route replacement. + Rust package tests: ```powershell @@ -192,41 +233,23 @@ public class ItemData : DatabaseEntry, INamedEntry - backing stores are persistence adapters - indexes and name maps are maintained inside the cache, not inside the store -### Important: Multiple Backing Stores +### Multiple Backing Stores -When multiple backing stores are added, behavior depends on how they are registered. - -If a store is added with domain types: +Every document type has exactly one home store. There are no mirrors. ```csharp -cache.AddBackingStore(playerStore, typeof(PlayerData)); -cache.AddBackingStore(settingsStore, typeof(AppSettings)); +cache.AddBackingStore(catalogStore, typeof(ItemData), typeof(Faction)); +cache.AddBackingStore(runStore, typeof(SavedGame)); ``` -then that store becomes the direct persistence target for those types. +- the home is chosen by the document's runtime type: the most derived routed type wins, otherwise the untyped store +- at most one untyped store; a type claimed by two stores is an error +- a write with no home is an error once any store is attached +- attaching a store reads it; attach routed stores first and the untyped store (if any) last, because a record's home cannot move after it is loaded +- loading never writes: opening and flushing without a mutation leaves every file byte-identical +- a read-only store refuses writes and is skipped by flush -If a store is added without domain types: - -```csharp -cache.AddBackingStore(primaryStore); -cache.AddBackingStore(mirrorStore); -``` - -then the first generic store acts as the primary writable store for non-domain-specific entries. Additional generic stores subscribe to the existing stores and mirror their change events. - -Implications: - -- order matters for generic stores -- `AddAsync` writes to the type-specific store when one exists -- otherwise `AddAsync` writes to the first generic store -- later generic stores do not become co-primaries; they mirror earlier stores -- `PullAllBackingStoresAsync` pulls from every registered store - -Recommended patterns: - -- use one generic primary store if you want simple persistence -- use domain-specific stores when different entry types belong in different persistence layers -- treat additional generic stores as mirrors or downstream replicas, not independent write targets +The contract is `src/GameCult.Caching/Contracts/cultcache-store-composition.md`. Until the C# cache implements it, `AddBackingStore` takes no home types and every attached store receives every record; attach one store per cache. ## Example: Cache + Networking @@ -322,8 +345,6 @@ Each subproject has a local README with package-specific detail: - [GameCult.Caching](src/GameCult.Caching/README.md) - [GameCult.Caching.MessagePack](src/GameCult.Caching.MessagePack/README.md) - [GameCult.Caching.NewtonsoftJson](src/GameCult.Caching.NewtonsoftJson/README.md) -- [GameCult.Caching.MessagePack.Generator](src/GameCult.Caching.MessagePack.Generator/README.md) -- [GameCult.Caching.MessagePack.Analyzers](src/GameCult.Caching.MessagePack.Analyzers/README.md) - [GameCult.Geometry](src/GameCult.Geometry/README.md) - [GameCult.Networking](src/GameCult.Networking/README.md) - [GameCult.Mesh](src/GameCult.Mesh/README.md) diff --git a/docs/cultmath-gamecult-geometry-migration-plan.md b/docs/cultmath-gamecult-geometry-migration-plan.md deleted file mode 100644 index e8e961c4..00000000 --- a/docs/cultmath-gamecult-geometry-migration-plan.md +++ /dev/null @@ -1,552 +0,0 @@ -# CultMath and GameCult.Geometry Staged Migration Plan - -Status: proposed for execution - -Date: 2026-07-22 - -## Decision - -`GameCult.Geometry` is the sole owner of geometric meaning and geometry -machinery. It builds on CultMath numeric primitives and participates in -CultCache, CultNet, and CultMesh through the normal CultLib integration paths. - -CultMath owns the HLSL-shaped numeric substrate: scalar operations, vectors, -quaternions, and general numeric kernels that have no spatial domain authority. -It does not own regions, topology, projections, surfaces, planetary fields, -patches, geometry residency, meshes, or geometry workflow state. - -VibeGeometry and `vg-csg` are sources to mine. They are not surviving owners or -dependencies. - -Source transfer does not erase provenance. CultMath is MPL-2.0 and CultLib is -MIT-licensed; moved files require an explicit licensing decision and preserved -notices before they enter a differently licensed package. - -## Objective - -Produce one coherent geometry stack in which: - -- consumers use `CultMath.float2`, `float3`, and `float4` as the canonical - floating-point vector primitives; -- `GameCult.Geometry` owns reusable geometry, planetary geometry, CSG, mesh - production, spatial queries, and the typed documents that coordinate those - systems; -- CultMath no longer publishes planetary or other geometry-owned APIs; -- `CultVec2` and `CultVec3` no longer exist; -- VibeGeometry can be archived after its useful implementation, evidence, and - doctrine have been absorbed; -- persisted geometry remains readable through an explicit migration path when - a live persisted corpus requires one. - -## Authority Map - -| Organ | Owns | Does not own | -| --- | --- | --- | -| CultMath | `float2`, `float3`, `float4`, integer/double/bool vector primitives, quaternion algebra, scalar and component-wise intrinsics, coordinate-agnostic numeric kernels, CPU/HLSL parity for those primitives | Rectangles, spheres, topology, projections, planetary fields, geometry pages, meshes, CSG, persistence schemas, worker commands | -| GameCult.Geometry | Spatial value types built from CultMath primitives, spatial queries, topology and projection, planetary field and page machinery, mesh and CSG implementation, geometry schema documents, stable geometry identities, geometry-specific HLSL and cross-runtime fixtures | Competing vector types, generic storage/network ownership, application gameplay consequences | -| CultCache/CultNet/CultMesh | Typed persistence, schema resolution, transport, discovery, publication, commands, and interface projection | Geometry semantics or geometry algorithm ownership | -| VibeGeometry / `vg-csg` | Nothing after migration | Any live runtime, schema, package, or compatibility authority | - -Authority sentence: - -> CultMath owns how numeric primitives behave; GameCult.Geometry owns what those -> primitives mean in space. - -## Current Mechanism - -The live body is split across three places: - -1. CultMath publishes the canonical HLSL-shaped vector types, but also contains - `rect`, planetary topology/projection/field/page/residency systems, spherical - erosion, planetary HLSL, Unity adapters, web fixtures, and planetary tools. -2. `GameCult.Geometry` contains duplicate `CultVec2` and `CultVec3` values, - geometric primitives, and four CultCache document roots. Its document - members still encode many vectors, bounds, transforms, vertices, normals, - and UVs as anonymous `float[]` values. -3. VibeGeometry wraps the `vg-csg` Rust implementation for domain-tree intent, - LOD-aware selected cuts, CSG lowering, and triangle/collider output. - -CultCache's current managed SoA projection only admits primitive scalar members. -It neither preserves arbitrary unmanaged structs as columns nor recursively -splits them. A `float2` or `float3` member is therefore omitted from the SoA -until that storage seam is repaired. - -## Invariants - -The migration must preserve these statements at every published boundary: - -1. There is one canonical vector vocabulary: CultMath's HLSL-shaped primitive - types. -2. Geometry algorithms and geometry state have one owner: `GameCult.Geometry`. -3. CultMath never depends on CultLib, CultCache, MessagePack, or - `GameCult.Geometry`. -4. `GameCult.Geometry` may depend on CultMath and CultLib. -5. Persistence decoration and formatters do not contaminate CultMath primitive - definitions. -6. A geometry schema has one active writer. Legacy readers or offline - migrators may exist temporarily, but they cannot decide live behavior. -7. Record keys and stable fingerprints are identical across supported runtimes - for the same semantic geometry value. -8. SoA storage preserves each authored unmanaged member as one column of its - exact type. It never decomposes `float2`, `float3`, quaternion, or another - value type by reflection. -9. CPU, HLSL, Unity, web, and Rust fixtures compare the same claimed layer of - geometry behavior. -10. VibeGeometry is not retained as a wrapper, submodule, compatibility crate, - or second source of truth. - -## Target Package Shape - -The exact directory names may follow existing CultLib conventions, but the -ownership should lower into these runtime surfaces: - -```text -CultMath - C# and Unity numeric primitives - CultMath.hlsl numeric mirror - coordinate-agnostic native numeric kernels - -CultLib - src/GameCult.Geometry/ - C# geometry API - planetary geometry - CultCache document contracts and formatters - geometry-specific HLSL packaged as content - packages/gamecult-geometry-rs/ - Rust geometry, CSG, and cross-runtime fixtures - packages/gamecult-geometry-ts/ # only if the web runtime remains earned - web projection/tile helpers and parity fixtures - src/GameCult.Unity/Assets/Geometry/ # Unity-owned lowering/adapters - mesh/page upload and presentation adapters - Packages/org.gamecult.geometry/ # final location follows packaging audit - Unity package depending on CultMath and CultLib -``` - -The runtime directories are projections of one geometry authority. They are not -independent products allowed to invent their own vector, topology, schema, or -fingerprint semantics. - -Each stage is a commit/review boundary. Do not begin the next stage until the -current gate passes, and never publish a state in which both repositories claim -the same geometry authority. - -## Source Classification - -### Keep in CultMath - -- HLSL-shaped primitive types such as `float2`, `float3`, and `float4`. -- Integer, double, and boolean vector primitives. -- Quaternion algebra. -- `math` scalar/component-wise operations and the core `CultMath.hlsl` mirror. -- Numeric batch kernels whose contracts are independent of regions, topology, - surfaces, planets, meshes, CSG, rendering policy, and persistence. - -### Move from CultMath to GameCult.Geometry - -The initial move set is: - -- `rect` and other spatial-region values; -- `PlanetaryTopology`; -- `PlanetaryProjection`; -- `PlanetaryQueries`; -- `PlanetaryPatch`; -- `PlanetaryPages`; -- `PlanetaryGpuPages`; -- `PlanetaryResidency`; -- `PlanetaryRadialRefinement`; -- `PlanetaryMapTileEncoding`; -- `PlanetaryField`; -- `AdvancedErosionFilter`; -- `ErosionFrequencyBands`; -- `SphericalErosion`; -- `AdvancedErosionFilter.hlsl`; -- `Planetary.hlsl`; -- spherical-erosion and planetary-radial-refinement shader bodies currently - embedded in `CultMath.hlsl`; -- the versioned `CMPT` planetary tile format and its TypeScript decoder; -- planetary tests, docs, tools, Unity adapters, web fixtures, and samples. - -`AdvancedErosionFilter` and `ErosionFrequencyBands` move with planetary -geometry. Their public contracts encode terrain-field policy, they are direct -dependencies of the planetary field definition, and their HLSL is coupled to -that field. Voronoi tone rendering and `native/cultmath-core` are outside this -migration: the native crate currently owns only that tone kernel and has no -planetary implementation. Audit those separately rather than dragging them -across this boundary because their names look spatial. - -### Rebuild inside GameCult.Geometry - -- Make `CultRect`, `CultCircle`, and `CultSphere` the GameCult.Geometry-owned - spatial values, implemented over CultMath primitives, and delete - `CultMath.rect` at cutover. -- Replace `CultVec2` and `CultVec3` with `float2` and `float3` at every typed - boundary. -- Replace vector-shaped `float[]` document members with typed CultMath values or - intentional packed buffers. -- Keep packed mesh buffers packed when they are genuinely bulk artifact storage; - do not turn every vertex buffer into a forest of object wrappers. -- Put MessagePack/CultCache formatters for CultMath values in the integration - package or generated formatter output, not on the CultMath types. - -### Mine from VibeGeometry and `vg-csg` - -Evaluate and extract, in dependency order: - -- convex plane splitting and brush operations; -- CSG tree and ordered operation semantics; -- domain-tree DSL and lowering; -- dirty-frontier and prefix-checkpoint work; -- selected-cut and LOD planning; -- triangle/collider assembly; -- performance fixtures and parity evidence; -- useful RealtimeCSG-derived behavioral fixtures; -- runtime doctrine that still describes the adopted machine. - -Rewrite extracted code under GameCult.Geometry ownership. Preserve third-party -license notices and source provenance. Do not retain a `vg-csg` forwarding -crate or a VibeGeometry submodule after consumers have moved. - -## Staged Migration - -### Stage 0: Freeze and Baseline - -Purpose: establish the evidence needed to move authority without guessing. - -Work: - -- Freeze new planetary and geometry API additions in CultMath, VibeGeometry, - and the current duplicate primitive surface. -- Do not publish an intermediate release containing duplicate live planetary - implementations. A coordinated working branch may hold source temporarily - for comparison, but only one owner may enter a package feed. -- Inventory public symbols, package versions, HLSL entrypoints, Unity samples, - web fixtures, native exports, persisted schema IDs, record-key algorithms, - and known consumers. -- Inventory exact contract constants: `FieldVersion`, enum ordinals, tile stable - keys, `CMPT` versions and bytes, MessagePack slots, GPU buffer layouts, - integer hashes, topology face/seam/corner rules, and shader include paths. -- Search deployed and checked-in `.cc` stores for the four - `gamecult.geometry.*.v1` schemas. -- Record whether any external consumer relies on CultMath planetary namespaces, - `CultVec*`, `CultMath.rect`, `vg-csg`, or the current HLSL function names. -- Capture passing baselines for CultMath tests, GameCult.Geometry tests, - `vg-csg` tests/clippy, cross-runtime planetary fixtures, and package builds. -- Add representative fixture hashes for projection, topology, field sampling, - patch selection, page encoding, CSG output, and geometry record keys. -- Resolve licensing and provenance for MPL-2.0 CultMath sources and any - third-party-derived `vg-csg` behavior before copying source into CultLib. - -Gate: - -- Every published surface has a named consumer or is marked for deletion. -- The persisted v1 corpus is known rather than assumed. -- Baseline fixtures can be run without booting an entire daemon. - -### Stage 1: Establish the Destination and Dependency Direction - -Purpose: make `GameCult.Geometry` capable of receiving geometry without growing -a second primitive vocabulary. - -Work: - -- Add the explicit `GameCult.Geometry -> CultMath` dependency. -- Define the target namespaces and package content layout for C#, Rust, HLSL, - Unity, and any retained web surface. -- Establish distribution before source transfer: an explicit NuGet dependency - and a dedicated `org.gamecult.geometry` Unity package depending on - `org.gamecult.cultmath` and `org.gamecult.cultlib`, unless package inspection - proves an existing surface already owns that role. -- Retain `net8.0` and `netstandard2.1` targets and Unity 2021.3 compatibility. - The geometry asmdef references CultMath; the Unity lowering assembly - references both. CultMath's asmdef remains reference-free with - `noEngineReferences: true`. -- Add GameCult.Geometry-owned MessagePack formatters for `float2`, `float3`, - `float4`, and quaternion with fixed positional layouts. -- Add cross-runtime byte fixtures for those value encodings. -- Repair CultCache's SoA member discovery so any direct unmanaged member is one - exact typed column. Add `Column` and `Column` tests and a - negative test proving no scalar decomposition occurs. -- Ensure generated schema metadata describes the authored vector member rather - than synthetic x/y/z members. - -Gate: - -- CultMath has no new dependency. -- `GameCult.Geometry` can serialize, deserialize, index, and expose intact - CultMath vectors. -- CultCache hot-path access returns intact vector spans. -- Built NuGet and UPM artifacts contain the expected dependency metadata, - formatters, shaders, and owned runtime assets. - -### Stage 2: Replace Duplicate Primitives and Repair Geometry Schemas - -Purpose: remove the duplicate vocabulary before moving larger geometry systems. - -Work: - -- Rewrite `CultRect`, `CultCircle`, and `CultSphere` with CultMath primitives. -- Delete `CultVec2` and `CultVec3`; do not add aliases or conversion wrappers. -- Replace typed references and opaque native-slice metadata using the obsolete - names. -- Classify every vector-shaped `float[]` in `CultGeometryDocuments.cs`: - - `Translation`, `SupportCenter`, `SupportSize`, `CameraPosition`, - `FrustumMin`, `FrustumMax`, `BoundsMin`, and `BoundsMax` become `float3`; - - `RotationXyzw` becomes quaternion; - - mesh `Positions` and `Normals` become `float3[]`; - - mesh `Uvs` becomes `float2[]`; - - scalar and index buffers remain packed arrays; - - use a named geometry value when bounds/topology carries an invariant. -- Define v2 schemas for every document whose field types change. CultCache - includes CLR type identity in the canonical schema and must reject an in-place - slot type mutation even when no durable v1 corpus is found. -- If live v1 documents exist, add bounded v1 reader DTOs and an offline - v1-to-v2 migrator. Stop all v1 writers at cutover. If no live persisted corpus - exists, omit the migrator and delete v1 runtime support, but still publish the - corrected contract under v2 identity. -- Reject malformed legacy vectors: fixed vectors must have the exact expected - component count, while packed mesh streams must be divisible by their element - width before conversion. -- Recompute and cross-check stable fingerprints and record keys against Rust - fixtures before accepting the new schemas. Preserve an old record key only - when the typed representation reproduces the prior IEEE-bit canonicalization - exactly; otherwise version the identity deliberately. -- Use CultMath's lowercase component names in v2 JSON inspection projections. - JSON is not the runtime transport, and uppercase `X/Y/Z` aliases must not keep - the deleted vector vocabulary alive. Any xenos-facing alternate casing is an - explicit boundary projection. - -Gate: - -- Repository-wide searches find no `CultVec2` or `CultVec3` outside historical - evidence. -- Repository and package searches find no `CultMath.rect`. -- No live writer emits vector-shaped anonymous arrays for semantic single - values. -- The old schema cannot overwrite, repair, or become a second owner of v2 - state. - -### Stage 3: Transfer Planetary Geometry out of CultMath - -Purpose: move planetary authority as one coherent vertical slice. - -Work: - -- Move the classified planetary C# implementation and tests into - `GameCult.Geometry`. -- Move `Planetary.hlsl` and rename its include ownership and public entrypoints - to GameCult.Geometry. It may include CultMath numeric HLSL; CultMath must no - longer include planetary geometry. Remove the current implicit planetary - include from `CultMath.hlsl` only after consumers include the new owner. -- Move `AdvancedErosionFilter.hlsl` plus the spherical-erosion and - planetary-radial-refinement bodies embedded in `CultMath.hlsl`. Publish one - geometry-owned umbrella include that imports CultMath first and geometry - organs afterward. -- Move planetary documentation, fixture tools, web tile code, Unity page/mesh - adapters, and the planetary viewer sample to their consumer-owned runtime - locations. -- Move ownership of the `CMPT` little-endian tile format and TypeScript decoder. - Preserve v1 bytes exactly or introduce a new format version; never reuse a - wire version after changing its bytes. -- Rename public namespaces and symbols to their GameCult.Geometry authority. - Prefer a coordinated breaking release over permanent forwarding types. -- Move spherical/terrain erosion components classified as geometry. Keep only - genuinely coordinate-agnostic numeric kernels in CultMath. -- Run CPU/HLSL and cross-runtime fixtures from the new owner before deleting - the old files. -- Preserve exact fixtures for field versions, enum ordinals, integer hashes, - tile keys/bytes, GPU buffer packing, MessagePack slots, topology seams and - corners, and record hashes. Use explicit tolerances only for floating - CPU/HLSL comparisons. -- Delete the corresponding CultMath sources, package entries, and implicit - includes in the same authority-transfer stage after the destination passes. - Do not defer the actual ownership cut to consumer cleanup. - -Gate: - -- GameCult.Geometry passes the captured planetary baseline and owns the fixture - commands. -- CultMath packages no `Planetary.hlsl`, planetary sample, planetary tool, or - planetary public namespace. -- No source or shader include can select an old CultMath planetary path. -- Package inspection finds no planetary API, Unity adapter, sample, tile - decoder, documentation claim, or implicit shader include in CultMath. - -### Stage 4: Mine and Absorb `vg-csg` - -Purpose: preserve useful CSG machinery without preserving the defunct owner. - -Work: - -- Import the earned Rust implementation into the GameCult.Geometry runtime - surface with source provenance and applicable license notices. -- Replace `bevy_math`-shaped public contracts where they conflict with the - canonical cross-runtime primitive/layout contract. Internal use may remain - where it protects a measured invariant and does not leak a second public - vocabulary. -- Connect domain trees, selected cuts, build requests, and emitted artifacts to - the GameCult.Geometry schema definitions. -- Port useful fixtures before refactoring algorithms so observable behavior has - a witness. -- Preserve the existing Rust v1 tuple MessagePack and stable-hash fixtures as - migration witnesses, then make the GameCult.Geometry-owned Rust package emit - the v2 contract. -- Re-run performance fixtures and retain thresholds only where the fixture - represents a real workload. -- Move surviving doctrine into GameCult.Geometry docs and discard historical - instructions that would recreate VibeGeometry ownership. - -Gate: - -- No production build or test depends on the VibeGeometry repository or - `vg-csg` submodule. -- CSG output satisfies the adopted correctness and performance fixtures from - its new package. -- The Rust runtime speaks the same schema IDs, vector layouts, fingerprints, - and record keys as C#. - -### Stage 5: Integrate Geometry Through CultCache, CultNet, and CultMesh - -Purpose: prove the new owner through the real typed-state path. - -Work: - -- Register the adopted geometry schemas and formatters through the generated - registry path. -- Prove raw envelope interchange for geometry documents across C# and Rust, - adding TypeScript only if an actual web consumer remains. -- Publish geometry worker state and commands through the provider-owned - CultMesh surface. -- Ensure geometry build requests, selected cuts, chunk artifacts, and planetary - page state share one commit/derivation path for direct and programmatic use. -- Add a development probe exposing schema version, owner, source record key, - selected cut/page, content hash, and served package version. - -Gate: - -- A pipeline smoke proves typed handoff from domain input through selected cut - or planetary page selection to a persisted and republished artifact. -- The visible/debug projection and persisted state identify the same owner and - version. -- No ad hoc JSON or parallel status model becomes load-bearing geometry truth. - -### Stage 6: Consumer Cutover and Coordinated Release - -Purpose: switch consumers without maintaining two live authorities. - -Release order: - -1. Build an unpublished CultMath candidate that contains the stable primitive - contract and removes geometry-owned APIs. -2. Build an unpublished GameCult.Geometry candidate against that exact CultMath - version, containing adopted schemas, planetary geometry, and runtime packages. -3. Update and test GameCult services, Unity projects, web consumers, and tools - against a local candidate feed. -4. Publish the CultMath and GameCult.Geometry candidates in one coordinated - release window, with the primitives-only CultMath dependency entering the - feed first. -5. Publish consumer updates after both owning packages are available. -6. Convert persisted v1 geometry stores, if any, and remove migration tooling - after the retained-corpus policy is satisfied. - -Work: - -- Pin compatible package versions across the coordinated release. -- Update examples and developer-facing entrypoints to use - `GameCult.Geometry` plus CultMath primitives directly. -- Cut Fensalir's Engine/Contracts public planetary types and shader includes to - the new assembly before removing CultMath geometry. Its CPU/GPU parity tests - are release gates. -- Add the geometry package to Aetheria/Unity, move its directly compiled - planetary sources to the new assembly, preserve moved Unity `.meta` GUIDs, - and regenerate Unity project files. -- Verify served package, shader, schema, and migration versions in at least one - real consumer for each retained runtime. -- Run the preserved parity corpus through known consumers including Fensalir, - Zyphos, Aetheria/Unity, the web tile decoder when retained, C#, Rust, and HLSL. -- Refuse mixed old/new geometry writers rather than reconciling them later. - -Gate: - -- Every known consumer runs on the new owner. -- No published package graph has `CultMath -> CultLib` or - `CultMath -> GameCult.Geometry`. -- Old and new writers cannot target the same state surface. - -### Stage 7: Delete Obsolete Authorities - -Purpose: make the old ownership structurally unable to return. - -Work: - -- Confirm deletion of all CultMath planetary/geometry sources, tests, shaders, - samples, docs, tools, web surfaces, package content entries, and Unity - adapters; remove any residue found after consumer cutover. -- Delete obsolete GameCult.Geometry duplicate primitive code and stale schema - paths. -- Delete VibeGeometry workspace wiring and the `vg-csg` submodule dependency; - archive the repositories after provenance links and release tags are - recorded. -- Remove compatibility aliases, forwarding namespaces, dual registries, and - temporary migration flags. -- Update source indexes and repository memory so future agents retrieve the new - authority first. - -Gate: - -- Negative searches and package inspection prove the old paths are absent. -- Building CultMath alone cannot access planetary or geometry APIs. -- Building GameCult.Geometry cannot access `CultVec*` or VibeGeometry. -- The old persisted schema path cannot write or override current geometry. - -## Schema Migration Policy - -Do not create a permanent compatibility model pre-emptively. - -- If no durable v1 corpus exists, change the schema once, update fixtures, and - publish the corrected contract as v2, then delete the old form without - building a migrator. -- If a durable v1 corpus exists, create an offline migrator with explicit input - and output schema IDs, content hashes, counts, and a dry-run report. -- A v1 reader may exist only inside migration/inspection tooling. -- The runtime writes only the current schema after cutover. -- Migration completes only when retained stores have receipts and no active - service advertises or writes v1. - -## Verification Matrix - -| Layer | Required evidence | -| --- | --- | -| CultMath primitives | C#/HLSL parity, unmanaged layout/size checks, package inspection | -| Geometry values | Exact `float2`/`float3` field ownership, intersection/query unit tests, MessagePack byte fixtures | -| CultCache SoA | Intact vector columns, no recursive flattening, add/update/remove/snapshot behavior | -| Geometry schemas | Catalog IDs and slots, v1 inventory or migration receipts, stable record-key parity | -| Planetary geometry | Projection round trips, topology adjacency, field/normal samples, radial refinement, page encoding/residency, CPU/HLSL parity | -| CSG | Boolean/convex fixtures, dirty rebuild behavior, selected-cut correctness, triangle/collider receipts, performance fixtures | -| Cross-runtime | C#/Rust payload and fingerprint parity; web parity only for retained web consumers | -| Packaging | CultMath contains no geometry assets; GameCult.Geometry includes its owned HLSL/docs/runtime assets | -| End to end | Domain or planetary input to persisted artifact through CultMesh with visible owner/version evidence | - -## Negative Completion Checks - -The migration is not complete until all are true: - -- `CultVec2` and `CultVec3` cannot appear in newly generated schemas or native - slice metadata. -- CultMath cannot publish or execute planetary geometry through an old entrypoint. -- VibeGeometry and `vg-csg` cannot influence a build, runtime, fixture, or - generated artifact. -- VibeGeometry is absent from project references, package manifests, - submodules, CI, runtime identifiers, and authoritative documentation. -- Legacy v1 state cannot override or repair current geometry after the fact. -- CultCache cannot decompose a vector into independently owned scalar columns. -- Manual, programmatic, imported, persisted, and network-received geometry use - the same schema and commit paths. -- Passing unit tests are accompanied by package inspection and cross-runtime - evidence at the layer users actually consume. - -## Definition of Done - -The work is complete when `GameCult.Geometry` is the only geometry authority, -CultMath is a clean numeric substrate, all retained runtime projections agree on -typed layouts and stable identities, VibeGeometry has no live dependency path, -and the old owners are structurally unable to emit current geometry state. diff --git a/docs/cultmath-gamecult-geometry-migration-progress.md b/docs/cultmath-gamecult-geometry-migration-progress.md deleted file mode 100644 index 74a473c5..00000000 --- a/docs/cultmath-gamecult-geometry-migration-progress.md +++ /dev/null @@ -1,320 +0,0 @@ -# CultMath and GameCult.Geometry Migration Progress - -Plan: `docs/cultmath-gamecult-geometry-migration-plan.md` (locked) - -Started: 2026-07-22 - -Status: Stages 0-5 complete; Stage 6 consumer cutover implemented with two -external verification/publication gates; Stage 7 source-authority cut complete - -## Objective - -Implement the locked migration so `GameCult.Geometry` becomes the sole owner of -geometry, planetary geometry, CSG, geometry schemas, and runtime projections, -while CultMath remains the HLSL-shaped numeric substrate and VibeGeometry loses -all live authority. - -## Execution Authority Map - -- Owner: `GameCult.Geometry` owns geometry decisions and geometry state. -- Inputs: CultMath numeric primitives, mined `vg-csg` implementation/evidence, - current geometry v1 documents, and named downstream consumer contracts. -- Outputs: GameCult.Geometry C#/Rust/HLSL/Unity/web surfaces, v2 typed schemas, - persisted artifacts, and CultMesh projections. -- Derived state: JSON inspection, UI/debug projections, caches, migration - receipts, and compatibility reads are non-authoritative. -- Forbidden writers: CultMath planetary APIs, v1 geometry runtime writers, - `CultVec*`, VibeGeometry/`vg-csg`, legacy shader entrypoints, and anonymous - fixed-vector arrays. -- Shared paths: direct API calls, persisted reloads, imports, network envelopes, - worker commands, Unity consumption, web tile reads, and Rust emitters must use - the same current schema/layout/fingerprint rules. -- Deletion line: `CultVec2`, `CultVec3`, `CultMath.rect`, CultMath planetary and - erosion ownership, v1 writers, VibeGeometry wiring/submodule authority, and - stale package/docs/schema paths. - -Demotions: - -- Geometry v1 DTOs are no longer owners; when a durable corpus requires them, - they are bounded read-only migration inputs to v2. -- JSON is no longer an owner; it is an inspection or xenos-boundary projection. -- VibeGeometry is no longer an owner; it is a provenance-bearing source quarry. -- CultMath planetary state is no longer an owner; it is source material pending - transfer and deletion in the same unpublished migration cut. - -## Worktree Safety - -- CultLib began with unrelated modified `.voidbot`, CultMesh, CultCache TS, - CultNet Rust, README, and test files plus scratch PID files. These changes are - outside the migration unless an exact overlap is required and reviewed. -- CultMath began clean on `codex/rust-cultmath-core` at `cb481b4`. -- VibeGeometry began clean on `main` at `85ac60a`. -- The locked plan is not edited during implementation. - -## Stage Ledger - -| Stage | State | Evidence / next gate | -| --- | --- | --- | -| 0. Freeze and baseline | Complete | Source bodies, contracts, consumers, licenses, fixtures, and focused baselines mapped | -| 1. Destination/dependency/formatters/SoA | Complete | NuGet and dedicated UPM artifacts prove dependency direction and owned vector integration | -| 2. Primitive deletion and v2 schemas | Complete | C# and Geometry-owned Rust emit identical v2 bytes and record keys for all four roots | -| 3. Planetary transfer | Complete | Managed, HLSL, CMPT, Unity, web, tool, docs, tests, and package authority moved; CultMath owners deleted | -| 4. Mine `vg-csg` | Complete | Geometry-owned Rust kernel passes correctness, parity, performance, formatting, and warning-denied lint gates | -| 5. CultCache/CultNet/CultMesh integration | Complete | One provider commit path persists, watches, replicates, and probes typed geometry state | -| 6. Consumer cutover/release | Gate pending | Fensalir green; Aetheria source cut pushed but full build/Unity regeneration await external dependency/editor state and immutable UPM tag | -| 7. Delete obsolete authorities/audit | Complete | VibeGeometry workspace authority archived; tracked negative audit is clean apart from retained provenance/witnesses | - -## Verification Receipts - -Receipts will be appended here with command, repository, commit, result, and -relevant artifact/version. A passing narrow test is not accepted as evidence for -a broader stage gate. - -### Stage 0 receipts - -- CultLib: `dotnet test tests/GameCult.Geometry.Tests/GameCult.Geometry.Tests.csproj --no-build --no-restore --disable-build-servers` passed 13/13. A full rebuild remains a Stage 1 gate; shared .NET build contention made attached build attempts exceed 60 seconds. -- CultMath: `dotnet test CultMath.slnx --no-restore` passed 72/72. -- CultMath native: `cargo test --locked` passed 1/1. -- CultMath web tile unit: `node --test web/planetary-tile.test.mjs` passed 1/1. -- CultMath pack and the combined web/C# cross-runtime script exceeded the - baseline time window and remain required package-transfer gates. -- VibeGeometry: `cargo test` passed 56 unit and 9 integration tests; doc tests - passed; `cargo clippy --all-targets --all-features` passed. -- Authoritative Rust quarry: VibeGeometry-pinned `vg-csg` submodule `a3197f4`. - Standalone `E:\Projects\vg-csg` is stale and excluded. -- v1 witnesses: current C#/Rust slots and hash algorithms match; Rust expected - record keys include domain `02c9...ed95` and chunk `b94d...21af5`. Exact - checked-in MessagePack byte witnesses still need to be generated before v2. -- Persisted corpus: targeted file inventory covered CultLib, CultMath, - VibeGeometry, Fensalir, and Aetheria. CultLib contains only the two checked-in - `vg-csg-ragnarok` payload fixtures. Binary scanning of Aetheria `.cc`, - `.msgpack`, and `.mpack` artifacts found no `gamecult.geometry.*.v1` catalog - identity. Indexed-source retrieval found definitions/tests/docs but no durable - geometry store. Execution therefore uses v2 plus retained test witnesses, - with no live v1 runtime migrator. - -### Stage 1 receipts - -- CultCache MessagePack now resolves external member values through a - document-owner assembly declaration. Resolver precedence is core CultCache, - owner-declared resolvers, then MessagePack standard fallback; the options are - cached per owner assembly and generated document codecs use that same path. - CultCache does not reference Geometry or CultMath. -- CultCache SoA discovery now admits direct reference-free value types as one - exact typed column. It rejects managed-reference and byref-like values and - does not synthesize component columns from nested struct fields. -- `dotnet test tests/GameCult.Caching.Tests/GameCult.Caching.Tests.csproj - --no-restore --filter - "FullyQualifiedName~DocumentAssemblyResolverTests|FullyQualifiedName~SoaValueColumnTests" - --verbosity minimal -m:1 -nodeReuse:false` passed 4/4 after a full rebuild. -- The initial SoA reflection probe used `Type.IsFunctionPointer`, which is not - available on the library's `netstandard2.1` target. It was removed rather - than papered over; structural managed-reference inspection is the owning - eligibility rule. -- CultMath candidate `0.1.0-geometry-migration.1` was packed from commit - `cb481b4c75444426a693781a0e5b35ade08a8938` with build servers disabled. The - 123,307-byte package SHA-256 is - `07C0CE79453AF9BA29910809F8779916E7B8F92DCA7ACEB5E2B8F9D3EBED3E93`; - a local-feed consumer restored, compiled, and executed `CultMath.float3`. -- GameCult.Geometry consumes that candidate as an explicit NuGet dependency; - its sibling `.tools/local-feed` is only an optional restore source, not a - project-reference authority. Geometry owns strict positional formatters for - `float2`, `float3`, `float4`, and `quaternion`. -- `dotnet test tests/GameCult.Geometry.Tests/GameCult.Geometry.Tests.csproj - --no-restore --filter FullyQualifiedName~CultMathValueIntegrationTests - --verbosity minimal --disable-build-servers -p:UseSharedCompilation=false - -m:1` - passed 6/6. Evidence covers exact bytes, round trips, malformed-width - rejection, actual `Column`/`Column` access, and absence of - synthetic component columns. -- GameCult.Geometry candidate `0.1.0-geometry-migration.1` packed successfully; - its nuspec declares exact candidate dependencies on CultMath, CultCache, and - CultCache.MessagePack. The dedicated UPM artifact remains the open Stage 1 - gate. -- Authoritative `vg-csg` commit `8f070f4` now pins exact v1 MessagePack bytes - for representative domain/chunk tuples plus complete domain, request, - selected-cut, and chunk record keys. Focused Rust tests passed 4/4; formatting - and clippy passed. VibeGeometry commit `82b56db` advances its submodule to - that witness commit. Both migration branches are pushed. -- `scripts/build-geometry-unity-package.ps1` staged - `org.gamecult.geometry@0.1.0` successfully. Its manifest depends explicitly - on `org.gamecult.cultmath@0.1.0` and `org.gamecult.cultlib@1.0.7`; the package - contains only its owned `GameCult.Geometry.dll`/symbols plus its asmdef and - documentation. It neither duplicates CultMath nor buries Geometry inside the - CultLib Unity package. - -## Decisions and Deviations - -- Licensing: no relicensing is assumed. Moved CultMath files retain MPL-2.0 - notices and provenance; the GameCult.Geometry package will carry explicit - mixed-license package documentation rather than falsely stamping those files - MIT. `AdvancedErosionFilter` source and shader notices remain verbatim. -- Serialization: Geometry-owned external value formatters require a declarative - per-document-assembly resolver seam in `GameCult.Caching.MessagePack`. - CultMath remains annotation-free and core caching does not reference Geometry. -- v1 persistence: no durable corpus was found in the named repositories and - artifacts. v1 remains fixture evidence only; it will not survive as a runtime - writer or general compatibility model. -- No deviations from the locked target architecture. -- Stage 2 versions all four root geometry documents as one v2 cohort. Domain's - nested CLR layout changes are invisible to the current root-only schema - fingerprint, and leaving SelectedCut as the lone live v1 writer would retain - a split cohort even though its direct fields are unchanged. -- Typed vectors preserve existing record keys by flattening components in the - same authored order into the existing IEEE-754 bit canonicalization. Wire - payloads still change—especially typed mesh arrays—so byte fixtures and v2 - identities remain mandatory. - -### Stage 2 receipts - -- `CultVec2` and `CultVec3` definitions were deleted; `CultRect`, `CultCircle`, - and `CultSphere` now expose CultMath `float2`/`float3` directly. No alias or - conversion owner remains. -- `dotnet test tests/GameCult.Geometry.Tests/GameCult.Geometry.Tests.csproj - --no-restore --verbosity minimal --disable-build-servers - -p:UseSharedCompilation=false -m:1` passed 19/19 for the primitive cut. -- CultMesh native-slice metadata fixtures now name `CultMath.float2` and - `CultMath.float3` while retaining 8/12-byte element widths. Only those three - overlapping lines were staged from the already-dirty TypeScript test file. -- All four root geometry schemas now form a v2 cohort with their original slot - numbers. Semantic single vectors use `float3`/`quaternion`; mesh vertices, - normals, and UVs use `float3[]`/`float2[]`. The runtime contains no v1 DTO, - reader, writer, or conversion authority. -- Live v2 semantic SchemaIds are pinned as domain `sha256:e335...421d`, build - request `sha256:2de3...21d3`, selected cut `sha256:8644...9ff8`, and chunk - artifact `sha256:2d28...6156`. Full values live in the executable tests. -- Typed stable fingerprints flatten components into the prior ordered IEEE-bit - stream. Domain/chunk round trips preserve record identity, while the checked- - in flat-mesh v1 payload is retained as negative evidence and rejected by the - v2 decoder. The Geometry suite passes 19/19 after the cohort cut. -- `packages/gamecult-geometry-rs` is the Geometry-owned Rust v2 destination. - It has no VibeGeometry, `vg-csg`, `bevy_math`, CultCache, or CultNet - dependency and preserves the MIT provenance of the mined `vg-csg` contract. -- C# and Rust pin identical exact MessagePack bytes and record keys for domain, - build request, selected cut, and chunk roots. C# decodes the Rust witnesses - and re-emits the same bytes. The Geometry suite passes 24/24; Rust passes 4/4, - doc tests, rustfmt, and warning-denied clippy. -- The parity pass exposed an old split identity: C# request scalar floats used - round-trip decimal text while `vg-csg` used IEEE-bit hex. v2 deliberately - standardizes every float on IEEE bits. The request key therefore changes at - the v2 boundary; signed zero, a chosen NaN payload, and infinity are pinned - across MessagePack so this cannot quietly diverge again. - -### Stage 3 receipts - -- Thirteen planetary/erosion C# sources and thirteen xUnit suites now live - under GameCult.Geometry; `CultMath.rect`, its factories, and all original - planetary sources/tests are deleted. Destination planetary tests pass 41/41; - remaining CultMath tests pass 28/28 for both target frameworks. -- Geometry owns five HLSL files behind `GameCult.Geometry.hlsl`; public geometry - symbols use `gamecult_geometry_*`. CultMath's umbrella and NuGet candidate - contain only numeric `CultMath.hlsl`. A DXC SPIR-V smoke compiled the joined - umbrella successfully to a 1,428-byte module. -- The CMPT v1 tool and TypeScript decoder moved intact. The end-to-end fixture - is exactly 1,221 bytes with SHA-256 - `8b2bc46e4123b8ac8936b43f8d08a34e3f2e74f38d0bb3f101cf3728cd201962`; - the C# emitter and TypeScript decoder pass together. -- Unity adapters retain their original `.meta` GUIDs; the Planetary Viewer uses - the Geometry umbrella and owned shader symbols. The staged UPM artifact has - one owned DLL, both adapters, the sample, all five shaders, and explicit - MIT/MPL license and provenance documents. -- Stale candidate cache was made impossible by rotating CultMath and Geometry - to `0.1.0-geometry-migration.2`. CultMath's candidate contains only its DLLs, - README, and numeric shader. Geometry's candidate declares the exact CultMath - `.2` dependency and contains all Geometry shaders/docs/notices under a - mixed-license file declaration rather than the inherited blanket MIT claim. -- Full Geometry tests pass 27/27 after the joined shader/Unity integration; - planetary tests pass 41/41. Negative searches find obsolete CultMath geometry - names only inside tests that assert their absence or ignored generated output. - -### Stage 4 receipts - -- `packages/gamecult-geometry-rs` now owns the mined convex splitting, ordered - brush CSG, tree lowering, dirty frontiers, prefix checkpoints, domain/LOD - selection, feature lowering, and triangle/collider assembly kernels. No build - or runtime dependency points at VibeGeometry or `vg-csg`. -- The public `GeometryBrushAssembler` facade accepts Geometry-owned - `Float3` values and returns `GeometryTriangleMesh`; `bevy_math` remains a - private implementation dependency and is not re-exported. -- The duplicate quarry persistence authority (`DomainSpecDocument`, - `DomainNodeDocument`, `FeatureClaimDocument`, and its local schema version) - was deleted. Private specs are in-memory builders; the v2 `Geometry*` - documents remain the sole wire authority. -- Nine RealtimeCSG behavioral fixtures and a bounded 512-distant-cutter - performance fixture moved under Geometry ownership. `cargo test --locked` - passes 67/67 plus doc tests; rustfmt and warning-denied Clippy pass. -- The authority transfer commits are CultLib `756f5a33`, CultMath `6a16260`, - and the post-transfer label cleanup `99c893bd`. The Rust absorption commit is - CultLib `6b44f969`. - -### Stage 5 receipts - -- `CultGeometryWorkerProvider` owns the build invocation and output commit - sequence. Direct calls and the `gamecult.geometry.worker.build` CultMesh - operation share `BuildAsync`; compute is injected through the persistence-free - `ICultGeometryBuildPipeline` port. -- Domain/request reads and selected-cut, artifact, and observational worker-state - writes use `CultNetDatabase` typed APIs. Watches and raw MessagePack envelopes - are consequences of that commit path rather than parallel publishers. -- The development probe derives owner, v2 schema, request/cut keys, selected - nodes, artifact identities, content hashes, and served assembly version from - persisted documents. Mutating worker display state cannot overwrite geometry - outputs or forge the served version. -- Worker state registers through the generated CultDocument registry. Focused - provider tests pass 4/4 and the full GameCult.Geometry suite passes 31/31. -- Rust raw-envelope duplication was deliberately omitted: CultNet owns that - protocol. Existing C#/Rust exact v2 document bytes and record keys remain the - cross-runtime geometry witness. - -### Stage 6 receipts - -- Coordinated local candidates use - `scripts/build-geometry-candidate.ps1`. CultMath `.2` is primitives-only; - Geometry `.2` contains its managed assembly, notices/docs, and all five owned - shaders with exact support-package and CultMath candidate dependencies. A - clean net10 consumer restored and built; the Unity package staged one owned - assembly plus adapters, sample, and shaders. The Rust crate passed an - independent `cargo package --allow-dirty --no-verify` dry run. -- Fensalir cutover commit `42778c1` adds direct Geometry dependencies, migrates - planetary C# consumers, compiler include roots, shaders, and parity harnesses - while retaining CultMath for numeric primitives. The solution builds with no - warnings/errors; fractal tests pass 170/170, erosion parity 1/1, topology - parity 19/19, and planetary page tests 16/16. -- Aetheria cutover commits `26948f81` and `557bc71f` replace `CultVec*` fixtures - with CultMath vectors under an exact legacy MessagePack byte witness, move - spatial rectangles to `GameCult.Geometry.CultRect`, add the real .NET/Unity - Geometry dependencies, and remove stale ownership guidance. -- Aetheria's geometry compiler errors are gone. Its full build is presently - blocked by concurrent CultMesh API drift (`CultMeshBody*` and persisted-record - types), outside this migration. Unity regeneration is also deferred while an - active Unity editor owns the project. The coordinated local file dependency - must become an immutable Geometry Unity tag before publication. -- Legacy D3DCompiler exposed that macro-expanded HLSL includes were not - portable. CultLib commit `cdeea474` gives the Geometry umbrella direct default - and Unity include branches; Geometry shader ownership tests pass 3/3 and the - downstream D3D12 corpus passes. - -### Stage 7 receipts - -- Tracked negative audits across CultLib, CultMath, Fensalir, Aetheria, and - VibeGeometry find no live `CultVec*`, `CultMath.rect`, CultMath planetary - implementation, old planetary shader vocabulary, geometry v1 writer, or - VibeGeometry/`vg-csg` build dependency. Frozen fixtures, source attribution, - and negative ownership tests remain deliberately. -- VibeGeometry commit `bc9604a` removes its tracked Cargo workspace, lockfile, - submodule declaration/gitlink, scratch/build wiring, and live mission claims. - Its remaining documents point to source witness `8f070f4f` and Geometry - absorption commit `6b44f969`; repository history preserves provenance. -- CultLib commits `8a7eee71` and `b9f20a64` remove manual-registry guidance and - update the root package map. Generated CultDocument metadata remains the only - registration owner. - -## Next Actions - -1. Resolve Aetheria's concurrent CultMesh API/worktree drift, then rerun its - State/Freeze verifier and daemon smoke against this migration branch. -2. Close the active Unity editor/import workers, regenerate Aetheria Unity - project/lock files through Unity, and run the canonical EditMode suite. -3. Publish an immutable `org.gamecult.geometry` Unity package tag in the - coordinated release window, replace Aetheria's local candidate reference, - and repeat package/version inspection before declaring Stage 6 complete. diff --git a/docs/cultnet-transport-parity.md b/docs/cultnet-transport-parity.md index 5f0fec91..d0779196 100644 --- a/docs/cultnet-transport-parity.md +++ b/docs/cultnet-transport-parity.md @@ -31,7 +31,7 @@ CultNet owns cross-runtime transport semantics: | C# interop peer | Schema-v0 MessagePack over TCP-framed or shared RUDP transport | UDP multicast probe/announce with TCP and RUDP transport profiles | Test harness transport parity | | TypeScript `cultnet-ts` | `CultNetPeer` over any Node `Duplex`, TCP-framed transport, or single-peer RUDP socket transport; interop uses TCP | UDP multicast probe/announce in the interop peer | First UDP socket binding for the shared RUDP reliability owner | | Rust `cultnet-rs` | Interop example serves and dials schema-v0 MessagePack over TCP-framed or shared RUDP transport; single-peer RUDP socket transport exists in the library | UDP multicast probe/announce with TCP and RUDP transport profiles | UDP socket binding for the shared RUDP reliability owner | -| Python `cultcache-py` | TCP sockets with 4-byte length-prefixed MessagePack frames for local CultMesh/CultNet server and client; single-peer RUDP socket transport exists in the library | Endpoint lists and CultMesh peer/Verse catalogs | UDP socket binding for the shared RUDP reliability owner | +| Python `cultnet-py` | TCP sockets with 4-byte length-prefixed MessagePack frames for local CultMesh/CultNet server and client; single-peer RUDP socket transport exists in the library | Endpoint lists and CultMesh peer/Verse catalogs | UDP socket binding for the shared RUDP reliability owner | | Kotlin `cultmesh-kotlin` | Channel-aware WebSocket transport connection for reliable ordered `schema` frames; single-peer RUDP socket transport exists in the library; interop CLI serves and dials schema-v0 MessagePack over TCP-framed or shared RUDP transport | CultMesh Verse and peer catalogs, plus endpoint lists carried in schema-v0 catalog messages | WebSocket/TCP adapters plus UDP socket binding for the shared RUDP reliability owner, with build-script and TypeScript harness proof | The live split is therefore: payload language and the native RUDP packet/session @@ -258,11 +258,19 @@ Current progress: echo against C#/Kotlin/Python/Rust UDP peers; local runtime tests prove timeout state transitions. - TypeScript, C#, Rust, Python, and Kotlin now share bounded reliable-send - backpressure for RUDP sessions. `maxPendingReliablePackets` is advertised on - RUDP transport-profile channels and enforced by the session before reliable - connect, accept, single-packet sends, or fragmented sends enqueue anything; - TCP-framed profiles do not publish the field because they do not own the RUDP - pending-reliable queue. + backpressure for RUDP sessions. At most 32 reliable packets are admitted to + the wire at once; later fragments remain in a session-owned FIFO and + acknowledgements promote the next packets. Receivers emit a cumulative ACK + for packets inside the 32-sequence ACK horizon and an exact ACK for an older + retransmit, so unrelated unreliable traffic cannot strand reliable work. + This keeps the send frontier well + inside the 4,096-sequence receive history instead of blasting an entire large + document and hoping the UDP buffers impersonate flow control. + `maxPendingReliablePackets` bounds both queued and in-flight packets before a + reliable frame is accepted. Socket transports expose an explicit reliable + flush where the runtime supports one, so callers that require confirmed + transport delivery do not substitute a fixed sleep. TCP-framed profiles do + not publish the field because they do not own the RUDP pending-reliable queue. - TypeScript, C#, Rust, Python, and Kotlin now share a portable `cultnet.reconnect_policy.v0` document and deterministic exponential delay helper, and RUDP transport profiles advertise that policy under diff --git a/docs/nuget-packaging.md b/docs/nuget-packaging.md index a269090a..afc94570 100644 --- a/docs/nuget-packaging.md +++ b/docs/nuget-packaging.md @@ -19,5 +19,5 @@ closure through UPM rather than NuGet restore. Install the Unity package from the immutable release tag: ```text -https://github.com/GameCult/CultLib.git?path=/unity/org.gamecult.cultlib#cultlib-unity-v1.0.8 +https://github.com/GameCult/CultLib.git?path=/unity/org.gamecult.cultlib#cultlib-unity-v1.0.46 ``` diff --git a/docs/parked-features.md b/docs/parked-features.md new file mode 100644 index 00000000..aa65294b --- /dev/null +++ b/docs/parked-features.md @@ -0,0 +1,32 @@ +# Parked Features + +A parked feature was built for a stated reason, has no consumer, and was +removed from the tree. Its last working form lives at an annotated tag so it +can be restored instead of reinvented. + +## CultCache SoA table + +- Tag: `parked/cultcache-soa` +- Intent: ECS-style structure-of-arrays columns over cached documents, for + hot-loop performance without giving up POCO document ergonomics. +- Span at the tag: `src/GameCult.Caching/CultManagedDocument.cs:133-414` + (`CultSoaTable`, `CultSoaColumn`, `CultCacheSoaStore`, + `CultCacheSoaTypeTable`, `CultCacheSoaMember`); `CultCache.Soa()` and the + `_soa` field with its upsert/remove hooks in `CultCache.cs`; the SoA tests in + `tests/GameCult.Caching.Tests/BackingStoreTests.cs` and + `tests/GameCult.Mesh.Tests/CultMeshStreamingTests.cs`. +- Managed document: `CultManagedDocument` (`CultManagedDocument.cs:72-131` at + the tag) with `CultCache.Document` and `CultNetDatabase.Document`, the + reactive POCO handle added with the SoA storage in 5428870 "Make SoA + cache-managed document storage". Same tag, same intent: the document stays a + POCO while the cache owns its columnar storage. +- Tag message correction: the `parked/cultcache-soa` tag message says + `CultManagedDocument` was "deleted separately, not parked". This note + supersedes it: `CultManagedDocument` is parked under the same tag and is + restored alongside SoA. The tag itself is left as written. +- Consumers when parked: none outside tests. +- Restore: `git show parked/cultcache-soa:src/GameCult.Caching/CultManagedDocument.cs`, + take the SoA types and `CultManagedDocument`, restore both `Document` + openers from `CultCache.cs` and `CultNetDatabase.cs` at the tag, and re-hook a + `CultCacheSoaStore` into the cache's single + admission and eviction paths (`Admit`/`Evict`). diff --git a/docs/persisted-record-encoding.md b/docs/persisted-record-encoding.md new file mode 100644 index 00000000..b8e13f8a --- /dev/null +++ b/docs/persisted-record-encoding.md @@ -0,0 +1,68 @@ +# The encoding of a persisted CultCache record is part of its contract + +Changing how a stored record serializes is never a flag flip in this estate. +Consumers verify canonicality by **re-encoding what they decoded and comparing +bytes**, so a record that reads back correctly but re-encodes differently is +refused — and the refusal happens on live data, at startup or on the next read. + +Three guards, three separate stores, one idiom: + +| store | guard | refusal | +|---|---|---| +| Idunn runtime projection | `Idunn/src/drivers.rs:3611` | `service trust-anchor projection is noncanonical` | +| Idunn deployment brake | `Idunn/src/control_plane.rs` (`read_deployment_brake`) | `deployment brake is noncanonical or keyed by another authority` | +| Idunn trust bindings | `Idunn/src/provisioning.rs:546`, `:630` | `trust store contains a noncanonical or mismatched binding` | + +Idunn's control store applies the same rule to its own transactions. Assume the +guard exists and go looking for it before changing any persisted record's +serialization; three independent instances is a house pattern, not a +coincidence. + +## What this means for `#[cultcache(key = N, bytes)]` + +The attribute switches a `Vec` slot from a MessagePack array-of-integers +(serde's default) to a real `bin`. Measured, not assumed +(`packages/cultnet-rs/tests/byte_slot_migration.rs`): + +- **Reading stays compatible in both directions.** `serde_bytes::ByteBuf` + accepts an array marker, and a plain `Vec` accepts `bin`. No dual-read + path is needed, and reverting a binary is survivable. +- **Reproduction does not.** A record written before the flip can be read but + can never re-encode to the bytes it was stored as, and its `canonical_sha256` + moves with it. + +So the cost lands entirely on the guards above, not on readability. A field with +persisted records must flip the attribute **and** rewrite its store in the same +change, one target at a time, and never during a deployment — the projection +store is what the observer reads, and refusing it takes the observer down. + +A field with no live producer is free. Check for one before assuming a +migration: `IdunnAuthenticatedProviderHealthProjectionRecord` is constructed +only inside a test module, so its byte field costs nothing to convert. + +Size, for whether it is worth doing at all: 406 bytes as an array against 278 as +`bin` for a 256-byte payload, and 1234 against 851 for an 848-byte one. About +1.45x on every byte, which matters on a media path and is noise on a control +record. + +## `Option>` cannot take the attribute yet + +The derive emits `serde_bytes::Bytes::new(&self.field)`, which needs `&[u8]`, +and builds the field via `.into_vec()`, which yields `Vec`. Both are type +errors against an `Option>`, so the derive fails loudly rather than +collapsing "absent slot" into "unsigned". `IdunnDeploymentBrakeRecord.signature` +needs an `Option`-aware path in `cultcache-rs-derive` before it can move. + +## Fields validated for shape and produced by nothing + +`trust_binding_sha256` is declared on two records with different schemas +(`idunn.authenticated_provider_health_projection.v1` in cultnet-rs, +`idunn.authenticated_daemon_health_admission.v1` in odin-core), shape-validated +in both, and computed by nothing in CultLib, Idunn or Odin. + +Do not assume that means "omission". It may be a fossil — a contract whose +producer was deleted. The estate has form: Muninn's receiver runs a Reed-Solomon +audio FEC path against parity shards the Rust sender has never emitted, while an +acceptance ledger describes that FEC as having worked. Settling which requires +reading history, not source, and the answer changes whether the right move is to +implement the field or delete it. diff --git a/packages/cultcache-py/PARITY.md b/docs/python-runtime-parity.md similarity index 96% rename from packages/cultcache-py/PARITY.md rename to docs/python-runtime-parity.md index 18607d87..1cc222b7 100644 --- a/packages/cultcache-py/PARITY.md +++ b/docs/python-runtime-parity.md @@ -1,7 +1,7 @@ # Python Runtime Parity Map -This file is the current audit map for `cultcache-py` as a Python runtime for -CultCache, CultNet, and CultMesh. It is evidence, not confetti. +This file is the current audit map for the Python runtime packages +`cultcache-py`, `cultnet-py`, and `cultmesh-py`. It is evidence, not confetti. ## Proven Surfaces @@ -255,14 +255,14 @@ CultCache, CultNet, and CultMesh. It is evidence, not confetti. individual discovery, peer exchange, or simulation surfaces unserved without advertising disabled capabilities. - Python package health: - `cultcache-py-verify --json` checks the public CultCache/CultNet/CultMesh + `cultmesh-py-verify --json` checks the public CultCache/CultNet/CultMesh export surface declared by each package `__all__`, `py.typed` markers, a live local CultMesh framed-wire smoke over hello/schema/snapshot/shard catalog/shard log, typed peer-error export availability, peer-advertised mutation contracts, payload schema wire contracts, simulation capability truth, and benchmark sanity for Python-owned hot paths. - Python/C# public cache baseline: - `cultcache_py.compare_csharp` runs Python and C# `CultCache` upsert/get + `cultnet_py.compare_csharp` runs Python and C# `CultCache` upsert/get benchmarks with the same operation names and reports median Python-to-C# ratios from three samples by default. It also emits per-metric threshold checks and an overall parity status so performance claims have an explicit evidence gate. @@ -275,12 +275,14 @@ CultCache, CultNet, and CultMesh. It is evidence, not confetti. ## Local Python Gates ```powershell -$env:PYTHONPATH="$PWD\packages\cultcache-py\src" +$env:PYTHONPATH="$PWD\packages\cultcache-py\src;$PWD\packages\cultnet-py\src;$PWD\packages\cultmesh-py\src" python -m unittest discover -s packages\cultcache-py\tests -python -m cultcache_py.verify --json -python -m cultcache_py.benchmark --records 1000 --json -python -m cultcache_py.compare_csharp --records 1000 --samples 3 --json -python -m pip wheel --no-deps -w $env:TEMP packages\cultcache-py +python -m unittest discover -s packages\cultnet-py\tests +python -m unittest discover -s packages\cultmesh-py\tests +python -m cultmesh_py.verify --json +python -m cultnet_py.benchmark --records 1000 --json +python -m cultnet_py.compare_csharp --records 1000 --samples 3 --json +python -m pip wheel --no-deps -w $env:TEMP packages\cultcache-py packages\cultnet-py packages\cultmesh-py ``` ## Cross-Runtime Gate diff --git a/docs/runtime-parity-scope.md b/docs/runtime-parity-scope.md index cd3761d7..44715c7c 100644 --- a/docs/runtime-parity-scope.md +++ b/docs/runtime-parity-scope.md @@ -21,20 +21,23 @@ Cross-runtime parity has three layers: transport helpers instead of assembling raw maps and channel strings. SoA performance parity is deliberately not a cross-runtime requirement. C# -owns the CPU-local SoA table and shared-memory streaming body path. Other -runtimes may add native hot paths when their runtime role earns them, but they -do not need to mimic Unity/C# memory layout to speak CultNet or participate in -CultMesh. +owns the shared-memory streaming body path. The C# CultCache SoA table (ECS-style +structure-of-arrays columns over cached documents) is parked at the tag +`parked/cultcache-soa`: it was built for hot-loop performance, had no consumer +outside tests, and is restorable from that tag (`docs/parked-features.md`). +It is not a supported surface in the current tree. Other runtimes may add native hot paths when their +runtime role earns them, but they do not need to mimic Unity/C# memory layout +to speak CultNet or participate in CultMesh. ## Current Targeted Runtimes | Runtime | Role | Claimed parity | Not claimed | | --- | --- | --- | --- | -| C# | Reference implementation and production game/server substrate | CultCache typed documents and SoA, CultNet schema-v0, LiteNetLib production path with adapter profile, shared transport profiles, shared RUDP socket transport, C# multi-peer RUDP listener and schema service host, C# interop RUDP serve/dial path, connected RUDP CultMesh client helpers, RUDP-backed schema-client selection for `rudp://` service endpoints, RUDP reconnect policy/controller and reconnect-loop helper, LiteNetLib client reconnect-controller adoption, schema-client transport port for service request/response clients, CultMesh node/session/catalog/stream/simulation/RUDP surfaces, authorized peer lookup | No need to abandon LiteNetLib before the CultNet adapter fully owns production service adoption | +| C# | Reference implementation and production game/server substrate | CultCache typed documents, CultNet schema-v0, LiteNetLib production path with adapter profile, shared transport profiles, shared RUDP socket transport, C# multi-peer RUDP listener and schema service host, C# interop RUDP serve/dial path, connected RUDP CultMesh client helpers, RUDP-backed schema-client selection for `rudp://` service endpoints, RUDP reconnect policy/controller and reconnect-loop helper, LiteNetLib client reconnect-controller adoption, schema-client transport port for service request/response clients, CultMesh node/session/catalog/stream/simulation/RUDP surfaces, authorized peer lookup | No need to abandon LiteNetLib before the CultNet adapter fully owns production service adoption | | Python | Package runtime, local daemon endpoint, raw interop peer, typed state participant | CultCache/CultNet/CultMesh package surface, branded CultMesh node/schema/shard/Verse/peer/authority/stream/RUDP ergonomics, authorized peer lookup, shard catalogs/logs, schema-transport factory for raw client request/subscription flows, local TCP and RUDP CultMesh server listeners, Python interop RUDP serve/dial path, session/prediction/simulation fact helpers, RUDP codec/session/socket transport, RUDP reconnect policy/controller and reconnect-loop helper, schema-v0 MessagePack over RUDP evidence, benchmark evidence against C# cache paths | Universal C# throughput, full C# daemon/server stack, and RUDP as the only public service pipe | | TypeScript | Node/browser-adjacent projection and interop runtime | CultCache parity tests, CultNet schema-v0 peer, advertised schema/shard catalog wire contracts plus request/sync helpers, transport profiles, TCP-framed transport port and peer helper, shared RUDP socket transport, RUDP interop serve/dial path, RUDP reconnect policy/controller, RUDP reconnect-loop helper, branded CultMesh node/schema/shard/Verse/peer/authority/stream/RUDP ergonomics, authorized peer lookup | Full C# game session/server stack | -| Rust | Low-level cache/transport/runtime substrate | CultCache/CultNet interop lane, schema and shard catalog helpers, typed same-schema alias hydration from raw snapshots, transport profiles, TCP-framed port, shared RUDP codec/session/socket transport, Rust interop RUDP serve/dial path, connected RUDP CultMesh client helpers, RUDP reconnect policy/controller and reconnect-loop helper, schema-v0 MessagePack over RUDP evidence, small CultMesh facade for schema registry, shard catalog, peer cards, authority leases, authorized peer lookup, RUDP endpoint parsing, and peer-card/authorized-peer RUDP client construction | Full CultMesh game-session/server facade, stream catalog, or production game-session server | -| Kotlin | JVM/Android client substrate | Typed MessagePack cache, one-call typed snapshot document sync through local same-schema aliases, built-in schema catalog advertisement, schema/shard catalogs with shared schema-message transport fetch/sync helpers, authority leases, authorized peer lookup, stream catalog, branded `CultMesh` facade, WebSocket transport profile/connection helper, RUDP client/server sugar, connected RUDP CultMesh client helpers, Kotlin interop RUDP serve/dial path, RUDP reconnect policy/controller and reconnect-loop helper, schema-v0 MessagePack over RUDP evidence | C# SoA store, full C# server daemon, or Unity hot-loop memory layout | +| Rust | Low-level cache/transport/runtime substrate | CultCache/CultNet interop lane, schema and shard catalog helpers, typed same-schema alias hydration from raw snapshots, transport profiles, TCP-framed port, shared RUDP codec/session/socket transport, Rust interop RUDP serve/dial path, connected RUDP CultMesh client helpers, RUDP reconnect policy/controller and reconnect-loop helper, schema-v0 MessagePack over RUDP evidence, small CultMesh facade for schema registry, shard catalog, peer cards, authority leases, authorized peer lookup, RUDP endpoint parsing, and peer-card/authorized-peer RUDP client construction | Full CultMesh game-session/server facade or production game-session server. `packages/cultmesh-rs` now holds a Rust stream and session surface -- node, document sets, RUDP document publication, stream descriptors, negotiation, stream catalog, and shared-memory frame rings -- but no interop lane exercises it, so it carries no cross-runtime parity claim | +| Kotlin | JVM/Android client substrate | Typed MessagePack cache, one-call typed snapshot document sync through local same-schema aliases, built-in schema catalog advertisement, schema/shard catalogs with shared schema-message transport fetch/sync helpers, authority leases, authorized peer lookup, stream catalog, branded `CultMesh` facade, WebSocket transport profile/connection helper, RUDP client/server sugar, connected RUDP CultMesh client helpers, Kotlin interop RUDP serve/dial path, RUDP reconnect policy/controller and reconnect-loop helper, schema-v0 MessagePack over RUDP evidence | C# SoA store (parked at `parked/cultcache-soa`), full C# server daemon, or Unity hot-loop memory layout | ## Evidence Gates @@ -52,7 +55,7 @@ CultMesh. `reconnectPolicy` profile field from C#, Rust, Python, and Kotlin peers. Runtime-local tests cover the deterministic delay helper and reconnect controller in C#, TypeScript, Rust, Python, and Kotlin. -- `packages/cultcache-py/PARITY.md` is the Python-specific audit ledger. Its +- `docs/python-runtime-parity.md` is the Python-specific audit ledger. Its "Still Not Claimed" section is part of the contract, not an apology. - `docs/cultnet-transport-parity.md` is the transport ownership map. It tracks the remaining work to lower older LiteNetLib/WebSocket bodies behind the @@ -85,6 +88,16 @@ server body, is deliberately not made. backfilling C# server responsibilities it should not own. Kotlin WebSocket and RUDP adapters now share one `CultNetSchemaMessageTransport` helper shape for schema/shard catalog request and sync flows. +- Bring `packages/cultmesh-rs` under the wire claim. It arrived from a consumer repo where it grew + outside any conformance harness, and its stream, negotiation, and shared-memory frame-ring + surfaces have never been checked against the browser, Kotlin, and TypeScript CultMesh bodies + they must interoperate with. Runtime-local tests pass; that is not a parity claim. +- Carry the RUDP receive-path bounds to the other runtimes. Rust now caps tracked ordered + and sequenced channels, caps the out-of-order hold buffer by frames and bytes, refuses a + packet far ahead of the highest received so a forged sequence cannot starve a session, and + reports reliable sequence exhaustion instead of aborting. C#, TypeScript, Python and Kotlin + still accept all of it. These are refusals rather than wire-format changes, so the interop + lanes stay green either way -- which is exactly why the gap will not announce itself. - Keep performance claims evidence-bound: C# owns SoA; Python has benchmark comparison gates; other runtimes should advertise measured native hot paths only when those paths exist. diff --git a/package-lock.json b/package-lock.json index bc42f617..f027ecd5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8,8 +8,15 @@ "workspaces": [ "packages/cultcache-ts", "packages/cultnet-ts", - "packages/cultmesh-ts" - ] + "packages/cultmesh-ts", + "packages/cultmesh-browser" + ], + "devDependencies": { + "esbuild": "^0.28.1", + "fast-uri": "3.1.5", + "js-yaml": "4.3.1", + "playwright-core": "^1.62.1" + } }, "node_modules/@apidevtools/json-schema-ref-parser": { "version": "11.9.3", @@ -29,6 +36,452 @@ "url": "https://github.com/sponsors/philsturgeon" } }, + "node_modules/@esbuild/aix-ppc64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.28.1.tgz", + "integrity": "sha512-Svl7tq8k/08+p6CXPpRjQ1fKX+1odH/BQbb48fV6fj3CWHhsoIOoY87w1oHXm0qEpkIK3ZfVgp0hed3XBXzXMQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.28.1.tgz", + "integrity": "sha512-0k2F129Xdio1TdJfzJ8sy1Q47vUD2NnwdhiAf7drUN1EBTfPf4hsFCtmMgu/6m8JSzsBrlmVjudMBQqOfG8usQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.28.1.tgz", + "integrity": "sha512-34EGEbCIAgosYz6goLcopX6Mo7NyGv9tfwEM2/7Ce2VcVRk568iSvniGWcUXIy7wEDR1wzolcxcriFVrWYcwBg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.28.1.tgz", + "integrity": "sha512-dbwY7ltSMDWsRatcRpCnES4F+im88OCUgGZjy52shC7GqHRE/cYlxNbB4Z4UpJswpcc4Qxd2oE/ufM0p61IKng==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.28.1.tgz", + "integrity": "sha512-TZbWkQY7kvTAXbXUT7uVACR5cMHsDiSz9z7ZKAX/RTq/WJEk3QyRr0wZpNhBDX+/0CtdqUIJlOiodQcta6tY3Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.28.1.tgz", + "integrity": "sha512-zfdzgK9ACBNZLI/CyHTOx81SyNbM6YXn7rxSgX97VjyiPl9W1i4Ka4fgKECEoFCKGpvBj5qArWIGgQjOwkgskQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.28.1.tgz", + "integrity": "sha512-wG2EA8ENdEI0qhkSZMjfqrdY+ziCYCPMmtZjjIwOmXFjmyzEHn+UUxk5of+SYsjtfs3VpnlC7QLzSI5hY/rOAw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.28.1.tgz", + "integrity": "sha512-i7dZ9vQgnvSCzi/rYCXNgtF/U+eKZNJBzu3eTQbRgHnM7tNSizLOkRFAl3qzVc/Op/u5YkHHa4pf/3DOYHthLQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.28.1.tgz", + "integrity": "sha512-qVXBOHQS+d5Y722GwJzJUtOLlX7km3CraOaGormF1pDtPd2C/l1SHRPgjLunLGe51Sh5YYWKMFDyV4SxgMQYTQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.28.1.tgz", + "integrity": "sha512-yHs+0uc8+nvEAfAfxrWQKK5peSNzBc4PegcMO0EJ2hT71uA7vB8Ihg2e77R2P7SG5uYjPbHlLLmve4LLLRCf0g==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.28.1.tgz", + "integrity": "sha512-d1z4ZuP0ajrfz/FhGT4vv278rX8KnPPJx8i5+AtK7TYbx9Le9F1hyzurZpkEyjkGa9dUGhQow4C1NmeGvqxN2w==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.28.1.tgz", + "integrity": "sha512-M5sRjUVZrkm1OAPR3dlOYzNmN+loZKGVi1VUQGrwuqLcbR6qeAz+famMhjASeH3YVKvZz+zT1jlh/keC3Rj/lg==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.28.1.tgz", + "integrity": "sha512-mRObBZeHh2OxcBFPWE/FjylkRgZdYuiTR3vaTozquCGOH14iP9oN4x4Ge81CoIDYQrXmIxpFumJBu5MtZpnQJQ==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.28.1.tgz", + "integrity": "sha512-slScBsMAb3GFDcdrCgLwZtPYRoH2H/youv10QiZyRjmsP48fznoveWytSgCI/R0ZcUgpc0ZhIUEx6LHts8yrfQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.28.1.tgz", + "integrity": "sha512-kw0owk1o0GFETUJyW0jc0G4Yzs0BHZn0JDZ8JRT088vjJYX777BAs1fDGxAC+q831qOs2DTC96mNsG2opdfyyQ==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.28.1.tgz", + "integrity": "sha512-/lAIjX8aYFRByhh6L5rYtPEDRqa9de/4V/juOXcta5frjvzXO4/sqEtyytse0g3zZFuWu5cDN0MkLz2qRDD2Ag==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.28.1.tgz", + "integrity": "sha512-u/anNYF2mmVOEDwLtnQ1wOr3EZ9sTNGLWrsYGYwHWzGA3Si84IOkHXlbWTD1NB+9/1lcnweYKO54uhxZydNzfA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.28.1.tgz", + "integrity": "sha512-oks0DYbLwWMmaakTsCb+zL4E+aHRVLom9IJZOAthMQEPiQmydXHkziYEsGYRx0uNV/IjEKGAV941JzH02pflqw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.28.1.tgz", + "integrity": "sha512-aeL6lAnN89Hz43Mlh1G8ARasbuoYvSITDEx0tHh5b7jJnHcssqgjy9Yx430GDpmCa6OyrKoS0aNRjKundRizGg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.28.1.tgz", + "integrity": "sha512-MEFJe5C3R8pwXdZ5Y21oo6m7ePiS0d9pWucn99O/wvyJZChoIQKrQDxKrGeW8F5+T0okTHesAmDeiHDTIq0V/Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.28.1.tgz", + "integrity": "sha512-i/ZLIOafE0Z8cI/XANJAixoJL/uRAoS2xOA3rb0xN+KK0K177cMAsQYkzHtBrtMXAKuAc7HGgcWiZ/sRC1Nxgw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openharmony-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.28.1.tgz", + "integrity": "sha512-ge+Z7EXFNt2BO1oAMsVpiQ8EwndV9i1xXerAeTIK7AtPs3bKFXQM7nlRxDSIUIMeueR1CNXxqztLzdNeReKBJg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.28.1.tgz", + "integrity": "sha512-BEjgtECkL3vY+SaSQ6nzVfiALUeFxpawyp8Jmf5PtYhf1Ug40N1h/hxlhts+f1FvSvarEigdxS3BlSMI2PJLcQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.28.1.tgz", + "integrity": "sha512-lCv9eK/H6ZJWbE7bh2nw54CZ9M2nupBxJcTsdk/QQnWkdSjKGuxmmH8/GWrlT1eMmZfn4dGcCjRte397WqfQXA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.28.1.tgz", + "integrity": "sha512-zvb/mB2bSCoJOpoCBgYKKpX6YM6mJBlBUVUtVj41DlZJVEB6/0CKlRYxP5wWl1C1ILiCoAU5wZZ4q1P3qeS6Eg==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.28.1.tgz", + "integrity": "sha512-bm4Mowrv+GXMlpWX++EcXw/iLyd1o3+bJkC2DkWXYVvgZCqD/bSj9ctZeAMC3cIxgjRVR2Dufaiu4YPxr5gW1A==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@gamecult/cultcache-ts": { + "resolved": "packages/cultcache-ts", + "link": true + }, "node_modules/@jsdevtools/ono": { "version": "7.1.3", "resolved": "https://registry.npmjs.org/@jsdevtools/ono/-/ono-7.1.3.tgz", @@ -92,8 +545,8 @@ "dev": true, "license": "Python-2.0" }, - "node_modules/cultcache-ts": { - "resolved": "packages/cultcache-ts", + "node_modules/cultmesh-browser": { + "resolved": "packages/cultmesh-browser", "link": true }, "node_modules/cultmesh-ts": { @@ -104,6 +557,48 @@ "resolved": "packages/cultnet-ts", "link": true }, + "node_modules/esbuild": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.28.1.tgz", + "integrity": "sha512-HrJrvZv5ayxBzPfwphOoNzkzOIIlifzk0KJrGK2c8R4+LKpMtpYLQeUdjnwjWv/LZlkH2laZk+4w78pi99D4Vw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.28.1", + "@esbuild/android-arm": "0.28.1", + "@esbuild/android-arm64": "0.28.1", + "@esbuild/android-x64": "0.28.1", + "@esbuild/darwin-arm64": "0.28.1", + "@esbuild/darwin-x64": "0.28.1", + "@esbuild/freebsd-arm64": "0.28.1", + "@esbuild/freebsd-x64": "0.28.1", + "@esbuild/linux-arm": "0.28.1", + "@esbuild/linux-arm64": "0.28.1", + "@esbuild/linux-ia32": "0.28.1", + "@esbuild/linux-loong64": "0.28.1", + "@esbuild/linux-mips64el": "0.28.1", + "@esbuild/linux-ppc64": "0.28.1", + "@esbuild/linux-riscv64": "0.28.1", + "@esbuild/linux-s390x": "0.28.1", + "@esbuild/linux-x64": "0.28.1", + "@esbuild/netbsd-arm64": "0.28.1", + "@esbuild/netbsd-x64": "0.28.1", + "@esbuild/openbsd-arm64": "0.28.1", + "@esbuild/openbsd-x64": "0.28.1", + "@esbuild/openharmony-arm64": "0.28.1", + "@esbuild/sunos-x64": "0.28.1", + "@esbuild/win32-arm64": "0.28.1", + "@esbuild/win32-ia32": "0.28.1", + "@esbuild/win32-x64": "0.28.1" + } + }, "node_modules/fast-deep-equal": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", @@ -111,9 +606,9 @@ "license": "MIT" }, "node_modules/fast-uri": { - "version": "3.1.3", - "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.3.tgz", - "integrity": "sha512-i70LwGWUduXqzicKXWshooq+sWL1K3WUU5rKZNG/0i3a1OSoX3HqhH5WbWwTmqWfor4urUakGPiRQcleRZTwOg==", + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.5.tgz", + "integrity": "sha512-gHwA1O9LDIcKunMKhObS/HimwtehO1nPUECKAu5TpKgaO19fcWEl4bliWe1jWxVFvIXztJjjQ4L8XQ1EU9f7Jw==", "funding": [ { "type": "github", @@ -168,9 +663,9 @@ } }, "node_modules/js-yaml": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.3.0.tgz", - "integrity": "sha512-1td788aAnnZ5qs7V2QIRl1owjtYpbKt749Y3xauqQgwIIGF/xXWz1wMTEBx5O3LK3lXLVuqXPdPxj2BoFHaW9Q==", + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.3.1.tgz", + "integrity": "sha512-CY6crGq313MX8GkwvB7tzgp99vjQxY1++5y10/BKN/GUfHqWaOGQMNZkBvqSzsZKWk/ijwHlWzzkLulsGHhjWQ==", "dev": true, "funding": [ { @@ -250,6 +745,19 @@ "url": "https://github.com/sponsors/jonschlinkert" } }, + "node_modules/playwright-core": { + "version": "1.62.1", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.62.1.tgz", + "integrity": "sha512-wPYSwEBJY9GHraISXqyqtx0na0LpO3XEX7jNDhntbex7tzUS7kLnZsOlFruFJB4Hi/rhDMjXGqHewDZ68nYZVw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=20" + } + }, "node_modules/prettier": { "version": "3.9.5", "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.9.5.tgz", @@ -323,7 +831,8 @@ } }, "packages/cultcache-ts": { - "version": "0.13.5", + "name": "@gamecult/cultcache-ts", + "version": "0.14.0", "license": "MIT", "dependencies": { "@msgpack/msgpack": "^3.1.3", @@ -337,13 +846,28 @@ "node": ">=18" } }, + "packages/cultmesh-browser": { + "version": "0.3.0", + "license": "MIT", + "dependencies": { + "@msgpack/msgpack": "^3.1.3", + "cultnet-ts": "^0.2.0" + }, + "devDependencies": { + "@types/node": "^24.0.0", + "typescript": "^5.9.0" + }, + "engines": { + "node": ">=18" + } + }, "packages/cultmesh-ts": { "version": "0.1.1", "license": "MIT", "dependencies": { + "@gamecult/cultcache-ts": "^0.14.0", "@msgpack/msgpack": "^3.1.2", - "cultcache-ts": "^0.13.5", - "cultnet-ts": "^0.1.1" + "cultnet-ts": "^0.2.0" }, "devDependencies": { "@types/node": "^24.0.0", @@ -355,12 +879,12 @@ } }, "packages/cultnet-ts": { - "version": "0.1.1", + "version": "0.2.0", "license": "MIT", "dependencies": { + "@gamecult/cultcache-ts": "^0.14.0", "@msgpack/msgpack": "^3.1.3", - "ajv": "^8.17.1", - "cultcache-ts": "^0.13.5" + "ajv": "^8.17.1" }, "devDependencies": { "@types/node": "^24.0.0", diff --git a/package.json b/package.json index 09a80f89..c9b79e47 100644 --- a/package.json +++ b/package.json @@ -5,11 +5,18 @@ "packages/cultcache-ts", "packages/cultnet-ts", "packages/cultmesh-ts", - "packages/gamecult-geometry-ts" + "packages/cultmesh-browser" ], "scripts": { "build:ts": "node scripts/run-typescript-workspaces.mjs build", "test:ts": "node scripts/run-typescript-workspaces.mjs test", - "test:pack:ts": "node scripts/test-typescript-package-closure.mjs" + "test:pack:ts": "node scripts/test-typescript-package-closure.mjs", + "test:eve-browser-network": "node scripts/verify-eve-browser-network.mjs" + }, + "devDependencies": { + "esbuild": "^0.28.1", + "fast-uri": "3.1.5", + "js-yaml": "4.3.1", + "playwright-core": "^1.62.1" } } diff --git a/packages/cultcache-py/README.md b/packages/cultcache-py/README.md index 48c023ba..23effdbd 100644 --- a/packages/cultcache-py/README.md +++ b/packages/cultcache-py/README.md @@ -1,12 +1,16 @@ -# cultcache-py +# cultcache-py -`cultcache-py` is the Python runtime package for CultCache, CultNet, and -CultMesh. Callers work with registered domain documents while the cache owns -schema identity, routing, globals, name lookups, indexes, and backing-store -envelopes. +`cultcache-py` is the Python CultCache runtime. Callers work with registered +domain documents while the cache owns schema identity, routing, globals, name +lookups, indexes, and backing-store envelopes. It is intentionally small. It is not an ORM, a database, or distributed consensus wearing rented authority. +It is the bottom of the Python CultLib stack and depends only on `msgpack`. +`cultnet-py` (transport and wire contracts) and `cultmesh-py` (nodes, +discovery, sessions, local server) build on it, mirroring `cultcache-rs` / +`cultnet-rs` / `cultmesh-rs` and the npm packages. + ## Current Shape - document types are registered explicitly @@ -17,14 +21,14 @@ It is intentionally small. It is not an ORM, a database, or distributed consensu - payloads decode only through the registered document definition for that type - unknown persisted types fail closed - global documents are singleton-style per type -- type-specific backing stores beat generic backing stores +- one home store per document type: a store added with `add_backing_store` + owns its types, otherwise the generic store is the home; a type claimed by a + second store, or a second generic store, raises `CultCacheError`; a write + touches exactly one store and nothing is mirrored; once any store is + attached, a write whose type has no home raises - the JSONL store uses only the Python standard library for bootstrap consumers - `define_database_entry_type(...)` emits Rust/C#-style slot-indexed MessagePack array payloads for cross-runtime `DatabaseEntry` contracts -- `cultnet_py` exposes schema-v0 MessagePack helpers and 4-byte big-endian - frame helpers for raw document state, subscriptions, shard catch-up, - simulation observations, and witness artifact bundles -- `cultmesh_py` exposes a cache-backed local node surface for Python tools ## Example @@ -144,382 +148,28 @@ That matches the Rust `#[derive(DatabaseEntry)]` formatter shape and the C# `[Key(n)]` intent: schema evolution comes from stable field slots, not source member order. -## CultNet And CultMesh - -`cultnet_py` provides the Python schema-v0 wire helpers and an interop peer: - -```python -from datetime import UTC, datetime - -from cultnet_py import ( - CultNetSecret, - CultNetServerSecurityOptions, - CultNetRawClient, - CultNetSimulationObservation, - CultNetWitnessArtifactBundle, - apply_raw_snapshot, - apply_shard_log_response, - compute_simulation_claim_hash, - database_subscribe, - decode_frame, - encode_frame, - hello, - parse_message, - shard_catalog_request, - simulation_observation, - witness_artifact_bundle, -) -from cultmesh_py import CultMesh - -payload = hello(runtime_id="python-runtime").to_bytes() -message = parse_message(decode_frame(encode_frame(payload))) - -client = CultNetRawClient("127.0.0.1", 3075) -catalog = client.fetch_schema_catalog(kinds=["wire_message"]) -snapshot = client.fetch_snapshot_response(schema_ids=["cultnet.interop-note"]) -shard_catalog = client.fetch_shard_catalog(schema_ids=["cultnet.interop-note"]) -# With a local CultCache and the matching registered document definitions: -# applied = apply_raw_snapshot(cache, [note_doc], snapshot) - -subscription = database_subscribe(subscription_id="ui", schema_ids=["cultnet.interop-note"]) -shard_request = shard_catalog_request(message_id="shards", schema_ids=["cultnet.interop-note"]) -shard_log = client.fetch_shard_log_response(shard_id="interop", shard_epoch=1, after_sequence=0) -# log_changes = apply_shard_log_response(cache, [note_doc], shard_log) -with client.subscribe_database(subscription_id="ui", schema_ids=["cultnet.interop-note"]) as live: - initial_snapshot = live.read_next_snapshot_response() - -# The same raw-client request surface can ride RUDP when the peer advertises a -# rudp:// endpoint and shares the connection id for that lane. -rudp_client = CultMesh.create_client( - endpoint="rudp://127.0.0.1:3076", - connection_id=0x43554C54, -) -rudp_catalog = rudp_client.fetch_schema_catalog(kinds=["wire_message"]) - -claim_hash = compute_simulation_claim_hash("frame:42", "subject:player-1", "hit") -observation = simulation_observation( - message_id="obs-1", - witness_runtime_id="python-runtime", - shard_id="interop", - shard_epoch=1, - frame=42, - subject_id="player-1", - claim_kind="hit", - claim_hash=claim_hash, -) -typed_observation = CultNetSimulationObservation.from_wire(observation.to_wire()) -witness = witness_artifact_bundle( - bundle_id="bundle-1", - witness_kind="interop-proof", - captured_at="2026-06-13T00:00:00Z", - subject={"documentType": "cultnet.interop-note", "subjectId": "note:python"}, - contracts=[{"role": "payload", "schemaId": "cultnet.interop-note"}], - artifacts=[{"role": "log", "uri": "cultcache://bundle-1/log", "mediaType": "text/plain"}], - provenance={"pipelineId": "interop", "runId": "run-1", "runtimeId": "python-runtime"}, -) -typed_witness = CultNetWitnessArtifactBundle.from_wire(witness) -witness_payload = typed_witness.to_payload() - -security = CultNetServerSecurityOptions.development() -token = CultNetSecret.create_session_token( - "318fb4b6-ff5e-4c4f-b911-d81807de53a8", - datetime(2035, 1, 1, tzinfo=UTC), - security, - session_version=1, -) -session = CultNetSecret.try_validate_session_token(token, security) -``` - -AES-GCM string/byte encryption helpers are available through the same -`CultNetSecret` surface when the optional `crypto` extra is installed. - -The peer can serve, dial, and probe the same raw-state interop lane used by the -TypeScript, Rust, and C# test peers: - -```powershell -python -m cultnet_py.interop_peer serve --runtime-id python-peer --runtime-kind python --display-name "Python Peer" --agent-id python-agent --advertise-host 127.0.0.1 --tcp-port 3075 --discovery-port 4075 --discovery-group 239.77.44.11 --schema-path ..\cultnet-ts\integration\contracts\cultnet.interop-note.schema.json -python -m cultnet_py.interop_peer dial --runtime-id python-client --runtime-kind python --display-name "Python Client" --agent-id python-client --target-host 127.0.0.1 --target-port 3075 --schema-path ..\cultnet-ts\integration\contracts\cultnet.interop-note.schema.json -python -m cultnet_py.interop_peer probe --runtime-id python-prober --discovery-port 4075 --discovery-group 239.77.44.11 -``` - -`cultmesh_py` includes a local cache-backed node, schema-v0 helpers for the -CultMesh Verse catalog and peer exchange wire messages, local and verified -authority lease checks, stream transport negotiation, committed simulation fact -documents, and local prediction reconciliation: - -```python -from datetime import UTC, datetime, timedelta - -from cultcache_py import define_database_entry_type -from cultnet_py import ( - CultNetClientAuthorityScope, - CultNetFileShardReplicaCursorStore, - CultNetRawClient, - CultNetSchemaShardLogFetcher, - CultNetSchemaShardSnapshotFetcher, - CultNetShardDescriptor, - CultNetShardReplicator, - CultNetShardReplicatorOptions, -) -from cultmesh_py import ( - CultMesh, - CultMeshAuthorityLease, - CultMeshGameSessionOptions, - CultMeshHmacAuthorityLeaseVerifier, - CultMeshNodeOptions, - CultMeshSnapshotFanout, - peer_exchange_request, -) - -note_doc = define_database_entry_type( - "mesh.note", - [("body", 0), ("name", 1), ("kind", 2)], - name="name", - indexes={"kind": "kind"}, -) -node = CultMesh.start_node( - "mesh.cc", - runtime_id="python-runtime", - options=CultMeshNodeOptions(enable_durable_shard_logs=True), -) -node.database.register_document(note_doc) -unsubscribe = node.database.watch_record(note_doc, "note:1", lambda change: print(change.change_kind)) -unsubscribe_named = node.database.watch_by_name(note_doc, "intro", lambda change: print(change.record_key)) -unsubscribe_kind = node.database.watch_by_index(note_doc, "kind", "guide", lambda change: print(change.record_key)) -node.database.put(note_doc, "note:1", {"body": "hello", "name": "intro", "kind": "guide"}) -live_note = node.database.get_required(note_doc, "note:1") -put_message = node.database.put_raw_message( - note_doc, - "note:2", - {"body": "wire me", "name": "wire", "kind": "interop"}, - shard_id="interop", - shard_epoch=1, -) -snapshot_response = node.database.build_snapshot_response(schema_ids=[note_doc.catalog_entry().schema_id]) -snapshot_wire = node.database.create_snapshot_response(schema_ids=[note_doc.catalog_entry().schema_id]) -delete_message = node.database.delete_raw_message(note_doc, "note:2", shard_id="interop", shard_epoch=1) -shard_log_response = node.database.build_shard_log_response(shard_id="interop", shard_epoch=1) -shard_log_wire = node.database.create_shard_log_response(shard_id="interop", shard_epoch=1) - -station_stock_doc = define_database_entry_type( - "aetheria.station_stock", - [("missiles", 0), ("coolant", 1)], - schema_id="gamecult.aetheria.station_stock.v1", -) -station_stock_ui_doc = define_database_entry_type( - "aetheria.station_stock.ui", - [("missiles", 0), ("coolant", 1)], - schema_id="gamecult.aetheria.station_stock.v1", -) -node.database.register_document(station_stock_doc) -node.database.put(station_stock_ui_doc, "station:starbridge:stock", {"missiles": 24, "coolant": 80}) -stock = node.database.get_required(station_stock_ui_doc, "station:starbridge:stock") - -peers = CultMesh.create_peer_catalog() -response = peers.create_response(peer_exchange_request("pex-1", verse_id="local")) -lease_verifier = CultMeshHmacAuthorityLeaseVerifier({"odin": b"shared-lease-key"}) -lease_catalog = CultMesh.create_authority_lease_catalog( - signature_verifier=lease_verifier.verify, - require_verified_signatures=True, -) -lease_valid_from = datetime.now(UTC) -lease = lease_verifier.issue(CultMeshAuthorityLease( - lease_id="lease:python-runtime", - verse_id="local", - peer_id="python-runtime", - roles=("shard-primary",), - valid_from=lease_valid_from, - expires_at=lease_valid_from + timedelta(minutes=5), - shard_ids=("interop",), - issuer_runtime_id="odin", -)) -lease_catalog.upsert(lease) -session = CultMesh.create_game_session( - node, - CultMeshGameSessionOptions( - client_authority_scopes=( - CultNetClientAuthorityScope("python-runtime", schema_ids=(note_doc.catalog_entry().schema_id,), key_prefix="input:python"), - ), - ), -) -server = CultMesh.serve_node( - node, - peer_catalog=peers, - observation_hub=session.observation_hub, - port=3075, - max_snapshot_documents=1000, - max_snapshot_bytes=4 * 1024 * 1024, -) -``` - -The same local server can be launched as a package daemon when another runtime -or operator process needs a CultNet endpoint without embedding Python glue: - -```powershell -python -m cultmesh_py.daemon --runtime-id python-runtime --display-name "Python Runtime" --host 127.0.0.1 --port 3075 --cache-file mesh.cc --enable-durable-shard-logs --seed-interop-note --seed-shard-id interop --verse-id local --role shard-primary --ready-file mesh.ready.json -``` - -The ready file and stdout line are process readiness hints. The served state and -capability truth still live on the framed CultNet/CultMesh endpoint. Use -`--register-interop-note` to expose the package interop-note schema without -seeding a note, or `--seed-interop-note` to register it and publish one local -record for wire probes. Seeded records are committed through the raw mutation -path, so peers can read them through both snapshot and shard-log catch-up. Add -`--verse-id` and one or more `--role` values to publish the launched endpoint in -its own Verse and peer catalogs for discovery clients. When the daemon is -launched with `--cache-file`, `--enable-durable-shard-logs`, and -`--shard-log-file`, a later daemon process can rehydrate and serve the persisted -snapshot and shard log without reseeding. - -`CultMeshPeerHealthMonitor` probes peer-card endpoints with `cultnet.hello.v0` -and preserves the runtime id, display name, document types, message versions, -and mutation contracts reported by the peer. - -```python - -client = CultMesh.create_verse_discovery_client("127.0.0.1", 3075) -verses = client.fetch_verses(transport_version="cultmesh.v0") -mesh_peers = client.fetch_peers(verse_id="python-interop", roles=["read-replica"]) -client.sync_peer_catalog(peers, verse_id="python-interop", roles=["read-replica"]) -client.fanout_peer_catalog(peers, verse_id="python-interop", roles=["read-replica"]) -replica_notes = client.sync_documents( - node.database, - peers, - verse_id="python-interop", - roles=["read-replica"], - documents=[(note_doc, "note:remote")], -) -snapshot_fanout = CultMeshSnapshotFanout( - client, - node.database, - peers, - verse_id="python-interop", - roles=["read-replica"], - documents=[(note_doc, "note:remote")], - on_document=lambda note: print(note), -) -snapshot_fanout.sync_once() - -raw_client = CultNetRawClient("127.0.0.1", 3075) -synced_note = node.database.sync_document(raw_client, note_doc, "note:remote") -synced_stock = CultMesh.sync_document( - node, - raw_client, - station_stock_ui_doc, - "station:starbridge:stock", -) -with CultMesh.subscribe_document( - node, - raw_client, - station_stock_ui_doc, - "station:starbridge:stock", -) as stock_subscription: - stock = stock_subscription.sync_initial() - next_change = stock_subscription.read_next_change() -node.database.sync_shard_log(raw_client, shard_id="interop", shard_epoch=1) -replicator = CultNetShardReplicator( - node.database, - CultNetShardReplicatorOptions( - fetcher=CultNetSchemaShardLogFetcher(), - snapshot_fetcher=CultNetSchemaShardSnapshotFetcher(), - cursor_store=CultNetFileShardReplicaCursorStore("mesh.replica-cursors.msgpack"), - poll_interval_seconds=1.0, - on_error=lambda error: print(error), - ), -) -replica_shard = CultNetShardDescriptor( - shard_id="interop", - owner_runtime_id="primary-runtime", - epoch=1, - schema_ids=(note_doc.catalog_entry().schema_id,), - primary_endpoints=("cultnet://127.0.0.1:3075",), -) -replicator.pull_once(replica_shard) -replicator.start([replica_shard]) -replicator.stop() -server.stop() - -streams = CultMesh.create_stream_catalog() - -claim_hash = "accepted-claim-hash" -facts = CultMesh.create_simulation_fact_committer(node) -committed = facts.commit({ - "shardId": "arena", - "shardEpoch": 4, - "frame": 100, - "subjectId": "bob", - "claimKind": "hit", - "claimHash": claim_hash, - "witnessCount": 2, - "supportWeight": 2.0, - "totalWeight": 2.0, - "confidence": 1.0, - "hasQuorum": True, -}) -session_commits = session.submit_and_commit(typed_observation) -prediction = session.predict(note_doc, "input:python:move", {"body": "predicted input"}) -unsubscribe() -``` - ## Wire Parity -See [PARITY.md](PARITY.md) for the current evidence map, gates, and things this -package still does not claim. +See [docs/python-runtime-parity.md](../../docs/python-runtime-parity.md) for +the evidence map across the three Python packages. -The Python interop peer is `cultcache_py.interop`: +The store interop peer is `cultcache_py.interop`: ```powershell python -m cultcache_py.interop write --file cache.cc --runtime-id python python -m cultcache_py.interop read --file cache.cc ``` -Current receipts: +`packages/cultcache-ts/test/cult-cache.test.ts` includes Python in the shared +CultCache v1 parity matrix with TypeScript, Rust, and C#. `cultcache_py` ships +a `py.typed` marker so downstream type checkers can inspect the package surface. -- `packages/cultcache-ts/test/cult-cache.test.ts` includes Python in the shared - CultCache v1 parity matrix with TypeScript, Rust, and C#. -- `packages/cultnet-ts/test/interop/cultnet-interop.test.ts` includes Python in - the live TS/Rust/C#/Python schema-v0 peer ring: discovery, hello, schema - catalog, wire-message catalog discovery, raw snapshot, raw document - put/delete, mutation receipt, fire-command receipt, database subscription - changes, shard catalog, shard log catch-up, simulation consensus candidates, - committed simulation fact snapshots, Python session prediction reconciliation, - and witness artifact bundle round-trips. The same test asks the Python peer - for CultMesh Verse catalog and peer exchange responses over the CultNet pipe, - then verifies the public `CultMeshDiscoveryClient` can fetch typed Python - Verse and peer descriptors from that live endpoint. It also verifies - `CultNetRawClient` can fetch the live Python peer's schema catalog, raw - snapshot, and shard catalog. -- `packages/cultcache-py/tests/test_cultcache.py` covers Python CultMesh - Verse catalog, peer exchange, authority lease, stream negotiation, CultNet - helper shapes, raw snapshot/shard-log application, simulation claim hashing, - consensus aggregation, game-session fact commits, local prediction - reconciliation, committed simulation fact payload slots, the local CultMesh - server's TCP and RUDP schema request paths, and witness artifact bundle - payload slots. -- `cultcache_py`, `cultnet_py`, and `cultmesh_py` ship `py.typed` markers so - downstream type checkers can inspect the package surface instead of treating - the runtime as an untyped xenos swamp. - -## Performance Baseline - -The package ships a lightweight benchmark for Python-owned hot paths. In a -CultLib source checkout, it can also run the C# reference benchmark beside it: +## Tests ```powershell -python -m cultcache_py.benchmark --records 1000 --json -python -m cultcache_py.compare_csharp --records 1000 --json -python -m cultcache_py.verify --json +$env:PYTHONPATH="$PWD\packages\cultcache-py\src" +python -m unittest discover -s packages\cultcache-py\tests ``` -It reports slot-indexed `DatabaseEntry` encode/decode throughput, framed -CultNet MessagePack parse throughput, raw snapshot application into a -registered cache, and public `CultCache` upsert/get throughput. The C# compare -command also runs `packages/cultcache-py/tools/GameCult.Caching.Benchmark` and -reports median Python-to-C# ratios for the shared public cache operations. It -uses three samples by default; pass `--samples` when you need a different -evidence shape. Both runtimes precompute record keys before measured cache -upsert/get loops so the comparison is about public cache access, not string -construction noise. The public in-memory cache owner stores records in -type-partitioned dictionaries so hot `get(...)` calls avoid per-call -composite-key allocation. Treat the benchmark numbers as a local baseline, not -a claim that Python matches the C# reference in every workload. +The suite runs with only this package on the path. If a test here needs +`cultnet_py` or `cultmesh_py`, it belongs in that package. diff --git a/packages/cultcache-py/pyproject.toml b/packages/cultcache-py/pyproject.toml index 93ebf849..6c46d102 100644 --- a/packages/cultcache-py/pyproject.toml +++ b/packages/cultcache-py/pyproject.toml @@ -4,11 +4,11 @@ build-backend = "setuptools.build_meta" [project] name = "cultcache-py" -version = "0.1.0" -description = "Python CultCache, CultNet, and CultMesh runtime package for GameCult typed state." +version = "0.3.0" +description = "Python CultCache runtime for GameCult typed state." readme = "README.md" requires-python = ">=3.10" -license = { text = "Private GameCult infrastructure" } +license = "MIT" authors = [{ name = "GameCult" }] classifiers = [ "Programming Language :: Python :: 3", @@ -17,23 +17,11 @@ classifiers = [ "Programming Language :: Python :: 3.12", ] dependencies = ["msgpack>=1.0"] - -[project.optional-dependencies] -msgpack = ["msgpack>=1.0"] -crypto = ["cryptography>=42"] - [project.scripts] cultcache-py-interop = "cultcache_py.interop:main" -cultnet-py-interop = "cultnet_py.interop_peer:main" -cultcache-py-benchmark = "cultcache_py.benchmark:main" -cultcache-py-compare-csharp = "cultcache_py.compare_csharp:main" -cultcache-py-verify = "cultcache_py.verify:main" -cultmesh-py-daemon = "cultmesh_py.daemon:main" [tool.setuptools.packages.find] where = ["src"] [tool.setuptools.package-data] cultcache_py = ["py.typed"] -cultnet_py = ["py.typed"] -cultmesh_py = ["py.typed"] diff --git a/packages/cultcache-py/src/cultcache_py/cache.py b/packages/cultcache-py/src/cultcache_py/cache.py index f7ad1c12..0da2c04a 100644 --- a/packages/cultcache-py/src/cultcache_py/cache.py +++ b/packages/cultcache-py/src/cultcache_py/cache.py @@ -1,15 +1,52 @@ from __future__ import annotations -from dataclasses import dataclass, field -from typing import Any, Generic, TypeVar +import functools +import threading +from dataclasses import dataclass, field, replace +from typing import Any, Callable, Generic, TypeVar from .backing_store import BackingStore, CultCacheEnvelope from .documents import DocumentDefinition, extract_value T = TypeVar("T") +F = TypeVar("F", bound=Callable[..., Any]) GLOBAL_KEY = "__global__" +def _locked(method: F) -> F: + """Runs a read holding the cache's lock. Reads are reentrant and may run inside a mutation.""" + + @functools.wraps(method) + def wrapper(self: "CultCache", *args: Any, **kwargs: Any) -> Any: + with self._lock: + return method(self, *args, **kwargs) + + return wrapper # type: ignore[return-value] + + +def _mutating(method: F) -> F: + """Runs an attach, pull, registration, write or delete holding the cache's lock, so its validation, + store I/O and apply cannot interleave with another thread's. One started on a thread that is already + inside a mutation of this cache (from a store call, decoder, encoder, extractor or updater) raises + before touching anything.""" + + @functools.wraps(method) + def wrapper(self: "CultCache", *args: Any, **kwargs: Any) -> Any: + if self._mutating_thread == threading.get_ident(): + raise CultCacheError( + f"CultCache refuses a re-entrant {method.__name__}: a mutation cannot start from inside a store call, " + "decoder, encoder, extractor or updater of another mutation on the same cache" + ) + with self._lock: + self._mutating_thread = threading.get_ident() + try: + return method(self, *args, **kwargs) + finally: + self._mutating_thread = None + + return wrapper # type: ignore[return-value] + + class CultCacheError(RuntimeError): pass @@ -20,12 +57,31 @@ class _State: documents_by_schema_name: dict[str, DocumentDefinition[Any]] = field(default_factory=dict) values: dict[str, dict[str, Any]] = field(default_factory=dict) envelopes: dict[str, dict[str, CultCacheEnvelope]] = field(default_factory=dict) - stores_by_type: dict[str, list[BackingStore]] = field(default_factory=dict) - generic_stores: list[BackingStore] = field(default_factory=list) + stores_by_type: dict[str, BackingStore] = field(default_factory=dict) + generic_store: BackingStore | None = None name_extractors: dict[str, str | Any] = field(default_factory=dict) index_extractors: dict[str, dict[str, str | Any]] = field(default_factory=dict) names: dict[tuple[str, str], str] = field(default_factory=dict) indexes: dict[tuple[str, str, str], str] = field(default_factory=dict) + # The name and index values each held record produced when it was admitted. Removing a + # record reads these and runs no extractor, so removal has no fallible step. + lookup_keys: dict[str, dict[str, "_LookupKeys"]] = field(default_factory=dict) + # Load compatibility shim: a global persisted under a key other than GLOBAL_KEY is held as that + # type's global under GLOBAL_KEY, and its persisted key is kept here so the first write or delete + # of that global removes it from the home store. Remove once no store holds a global under a key + # other than GLOBAL_KEY. + legacy_global_keys: dict[str, str] = field(default_factory=dict) + + +@dataclass(frozen=True) +class _LookupKeys: + name: str | None + indexes: dict[str, str] + + +_Names = dict[tuple[str, str], str] +_Indexes = dict[tuple[str, str, str], str] +_LookupKeysByType = dict[str, dict[str, _LookupKeys]] class CultCacheBuilder: @@ -69,46 +125,135 @@ class CultCache: def __init__(self) -> None: self._state = _State() + # Mutations hold this lock and refuse to start on the thread already inside one (see _mutating). + # It is reentrant so a mutation's extractors, decoders and updaters may still read the cache. + # Single-structure reads (get, get_all, get_envelope, key lookups) stay unlocked. The guarantee is + # per call: each reads one dict under the GIL and sees it wholly before or after a write's install. + # Two unlocked calls can straddle a write or pull, and see different states. + self._lock = threading.RLock() + self._mutating_thread: int | None = None + @_mutating def register_document_type(self, document: DocumentDefinition[Any]) -> None: + self._register_document_type(document) + + def _register_document_type(self, document: DocumentDefinition[Any]) -> None: if document.type in self._state.documents: raise CultCacheError(f"Document type already registered: {document.type}") schema_name = document.catalog_entry().schema_name if schema_name in self._state.documents_by_schema_name: raise CultCacheError(f"Document schema name already registered: {schema_name}") + name_extractors = dict(self._state.name_extractors) + index_extractors = {type: dict(indexes) for type, indexes in self._state.index_extractors.items()} + if document.name is not None: + name_extractors[document.type] = document.name + if document.indexes: + index_extractors.setdefault(document.type, {}).update(document.indexes) + self._install_lookups(name_extractors, index_extractors) self._state.documents[document.type] = document self._state.documents_by_schema_name[schema_name] = document - if document.name is not None: - self.register_name_lookup(document, document.name) - for index, extractor in document.indexes.items(): - self.register_index(document, index, extractor) + @_mutating def register_registry(self, documents: list[DocumentDefinition[Any]] | tuple[DocumentDefinition[Any], ...]) -> None: for document in documents: - self.register_document_type(document) + self._register_document_type(document) + @_mutating def register_name_lookup(self, document: DocumentDefinition[Any], extractor: str | Any) -> None: self._assert_registered(document) - self._state.name_extractors[document.type] = extractor - self._rebuild_indexes() + name_extractors = dict(self._state.name_extractors) + name_extractors[document.type] = extractor + self._install_lookups(name_extractors, self._state.index_extractors) + @_mutating def register_index(self, document: DocumentDefinition[Any], index: str, extractor: str | Any) -> None: self._assert_registered(document) - self._state.index_extractors.setdefault(document.type, {})[index] = extractor - self._rebuild_indexes() + index_extractors = {type: dict(indexes) for type, indexes in self._state.index_extractors.items()} + index_extractors.setdefault(document.type, {})[index] = extractor + self._install_lookups(self._state.name_extractors, index_extractors) + def _install_lookups( + self, name_extractors: dict[str, str | Any], index_extractors: dict[str, dict[str, str | Any]] + ) -> None: + """Derives every lookup under the candidate extractors, then installs extractors and lookups + together. An extractor that raises on a held value installs nothing.""" + names, indexes, lookup_keys = self._derive_lookups(self._state.values, name_extractors, index_extractors) + ( + self._state.name_extractors, + self._state.index_extractors, + self._state.names, + self._state.indexes, + self._state.lookup_keys, + ) = (name_extractors, index_extractors, names, indexes, lookup_keys) + + @_mutating def add_backing_store(self, store: BackingStore, types: list[str] | tuple[str, ...] | set[str]) -> None: + """Makes the store home to the given types, or the generic store when there are none. + + A type has exactly one home store, and attaching cannot move the home of a record the cache holds. + """ + types = list(types) + if not types: + self._add_generic_store(store) + return for type in types: - self._state.stores_by_type.setdefault(type, []).append(store) + if type in self._state.stores_by_type: + raise CultCacheError( + f"Document type {type} is already routed to another backing store; it cannot also route to this one" + ) + routes = dict(self._state.stores_by_type) + routes.update({type: store for type in types}) + self._refuse_home_moves(routes, self._state.generic_store) + self._state.stores_by_type = routes + @_mutating def add_generic_store(self, store: BackingStore) -> None: - self._state.generic_stores.append(store) + """Makes the store home to every type no other store claims. A cache has at most one.""" + self._add_generic_store(store) + def _add_generic_store(self, store: BackingStore) -> None: + if self._state.generic_store is not None: + raise CultCacheError( + "Backing store would be a second generic store; name the types it is home to" + ) + self._refuse_home_moves(self._state.stores_by_type, store) + self._state.generic_store = store + + def _refuse_home_moves(self, routes: dict[str, BackingStore], generic: BackingStore | None) -> None: + for type, values in self._state.values.items(): + if not values: + continue + before = self._state.stores_by_type.get(type, self._state.generic_store) + after = routes.get(type, generic) + if before is not after: + raise CultCacheError( + f"Attaching this backing store would move {type} from {self._describe(before, routes, generic)} " + f"to {self._describe(after, routes, generic)}; attach routed stores before the generic store" + ) + + @staticmethod + def _describe(store: BackingStore | None, routes: dict[str, BackingStore], generic: BackingStore | None) -> str: + if store is None: + return "memory" + if store is generic: + return "the generic store" + return "the store routed to " + ", ".join(sorted(type for type, routed in routes.items() if routed is store)) + + @_mutating def pull_all_backing_stores(self) -> None: - self._state.values.clear() - self._state.envelopes.clear() + """Builds the complete next state (values, envelopes, name and index lookups) in local + structures, running every check and every user extractor; the cache's state is replaced + only once that build finishes, so a refused load admits nothing.""" + loaded_values: dict[str, dict[str, Any]] = {} + loaded_envelopes: dict[str, dict[str, CultCacheEnvelope]] = {} + loaded_legacy_global_keys: dict[str, str] = {} + routes, generic = self._state.stores_by_type, self._state.generic_store seen_globals: set[str] = set() - for store in [*self._all_specific_stores(), *self._state.generic_stores]: + stores: list[BackingStore] = [] + for candidate in [*self._state.stores_by_type.values(), self._state.generic_store]: + if candidate is not None and all(candidate is not store for store in stores): + stores.append(candidate) + for store in stores: for envelope in store.pull_all(): document = self._resolve_document_for_envelope(envelope) if document is None: @@ -122,14 +267,34 @@ class CultCache: schema_id=envelope.schema_id, catalog_entry=envelope.catalog_entry, ) + home = routes.get(envelope.type, generic) + if home is not store: + raise CultCacheError( + f"{envelope.type} record {envelope.key} was loaded from {self._describe(store, routes, generic)}, " + f"but its home is {self._describe(home, routes, generic)}" + ) if document.global_document: - if envelope.type in seen_globals and envelope.key == GLOBAL_KEY: + if envelope.type in seen_globals: raise CultCacheError(f"Duplicate global document for type: {envelope.type}") seen_globals.add(envelope.type) + if envelope.key != GLOBAL_KEY: + loaded_legacy_global_keys[envelope.type] = envelope.key + envelope = replace(envelope, key=GLOBAL_KEY) value = document.decode_payload(envelope.payload) - self._state.values.setdefault(envelope.type, {})[envelope.key] = value - self._state.envelopes.setdefault(envelope.type, {})[envelope.key] = envelope - self._rebuild_indexes() + loaded_values.setdefault(envelope.type, {})[envelope.key] = value + loaded_envelopes.setdefault(envelope.type, {})[envelope.key] = envelope + loaded_names, loaded_indexes, loaded_lookup_keys = self._derive_lookups( + loaded_values, self._state.name_extractors, self._state.index_extractors + ) + # Plain attribute assignments: nothing fallible runs once the swap begins. + ( + self._state.values, + self._state.envelopes, + self._state.names, + self._state.indexes, + self._state.lookup_keys, + self._state.legacy_global_keys, + ) = (loaded_values, loaded_envelopes, loaded_names, loaded_indexes, loaded_lookup_keys, loaded_legacy_global_keys) def get(self, document: DocumentDefinition[T], key: str) -> T | None: self._assert_registered(document) @@ -169,6 +334,9 @@ class CultCache: self._assert_registered(document) return self._state.names.get((document.type, name)) + # Name and index resolution read the lookup and then the values, two structures a pull swaps and a + # write updates in sequence, so they hold the lock to resolve a key and its value from one state. + @_locked def get_by_name(self, document: DocumentDefinition[T], name: str) -> T | None: key = self.get_key_by_name(document, name) return None if key is None else self.get(document, key) @@ -177,17 +345,19 @@ class CultCache: self._assert_registered(document) return self._state.indexes.get((document.type, index, value)) + @_locked def get_by_index(self, document: DocumentDefinition[T], index: str, value: str) -> T | None: key = self.get_key_by_index(document, index, value) return None if key is None else self.get(document, key) + @_mutating def put(self, document: DocumentDefinition[T], key: str, value: T) -> None: + self._put(document, key, value) + + def _put(self, document: DocumentDefinition[T], key: str, value: T) -> None: self._assert_registered(document) if document.global_document and key != GLOBAL_KEY: raise CultCacheError(f"Global document {document.type} must use key {GLOBAL_KEY}") - values = self._state.values.setdefault(document.type, {}) - envelopes = self._state.envelopes.setdefault(document.type, {}) - old_value = values.get(key) catalog_entry = document.catalog_entry() envelope = CultCacheEnvelope.create( key=key, @@ -196,100 +366,139 @@ class CultCache: schema_id=catalog_entry.schema_id, catalog_entry=catalog_entry, ) - stores = self._stores_for_type(document.type) - for store in stores: - store.push(envelope) - if old_value is not None: - self._remove_value_indexes(document.type, key, old_value) - values[key] = value - envelopes[key] = envelope - self._add_value_indexes(document.type, key, value) + self._write(document, [(envelope, value)], batch=False) + @_mutating def put_envelope(self, document: DocumentDefinition[T], envelope: CultCacheEnvelope) -> T: self._assert_registered(document) - if envelope.type != document.type: - raise CultCacheError( - f"Envelope type {envelope.type} does not match document type {document.type}" - ) - values = self._state.values.setdefault(document.type, {}) - envelopes = self._state.envelopes.setdefault(document.type, {}) - old_value = values.get(envelope.key) + self._check_envelope(document, envelope) value = document.decode_payload(envelope.payload) - for store in self._stores_for_type(document.type): - store.push(envelope) - if old_value is not None: - self._remove_value_indexes(document.type, envelope.key, old_value) - values[envelope.key] = value - envelopes[envelope.key] = envelope - self._add_value_indexes(document.type, envelope.key, value) + self._write(document, [(envelope, value)], batch=False) return value + @_mutating def put_envelopes(self, document: DocumentDefinition[T], envelopes: list[CultCacheEnvelope]) -> list[T]: self._assert_registered(document) values: list[T] = [] for envelope in envelopes: - if envelope.type != document.type: - raise CultCacheError( - f"Envelope type {envelope.type} does not match document type {document.type}" - ) - if document.global_document and envelope.key != GLOBAL_KEY: - raise CultCacheError(f"Global document {document.type} must use key {GLOBAL_KEY}") + self._check_envelope(document, envelope) values.append(document.decode_payload(envelope.payload)) - - stores = self._stores_for_type(document.type) - for store in stores: - store.push_all(envelopes) - values_by_key = self._state.values.setdefault(document.type, {}) - envelopes_by_key = self._state.envelopes.setdefault(document.type, {}) - for envelope, value in zip(envelopes, values): - old_value = values_by_key.get(envelope.key) - if old_value is not None: - self._remove_value_indexes(document.type, envelope.key, old_value) - values_by_key[envelope.key] = value - envelopes_by_key[envelope.key] = envelope - self._add_value_indexes(document.type, envelope.key, value) + self._write(document, list(zip(envelopes, values)), batch=True) return values + @staticmethod + def _check_envelope(document: DocumentDefinition[Any], envelope: CultCacheEnvelope) -> None: + if envelope.type != document.type: + raise CultCacheError( + f"Envelope type {envelope.type} does not match document type {document.type}" + ) + if not envelope.key or not envelope.key.strip(): + raise CultCacheError(f"Envelope key for document type {document.type} must be non-empty") + if document.global_document and envelope.key != GLOBAL_KEY: + raise CultCacheError(f"Global document {document.type} must use key {GLOBAL_KEY}") + + def _write( + self, document: DocumentDefinition[Any], records: list[tuple[CultCacheEnvelope, Any]], *, batch: bool + ) -> None: + """Validates every record (home and name and index extractors included) before the store is + touched; once the store accepts, applying to the cache has no fallible step.""" + type = document.type + name_extractor = self._state.name_extractors.get(type) + index_extractors = self._state.index_extractors.get(type, {}) + admitted = [ + (envelope, value, self._lookup_keys_for(value, name_extractor, index_extractors)) + for envelope, value in records + ] + store = self._store_for_type(type) + legacy_key = self._state.legacy_global_keys.get(type) + if store is not None: + if batch: + store.push_all([envelope for envelope, _ in records]) + else: + store.push(records[0][0]) + if legacy_key is not None: + self._retire_legacy_global(store, type, legacy_key) + self._state.legacy_global_keys.pop(type, None) + values = self._state.values.setdefault(type, {}) + envelopes = self._state.envelopes.setdefault(type, {}) + lookup_keys = self._state.lookup_keys.setdefault(type, {}) + for envelope, value, keys in admitted: + self._remove_lookups(type, envelope.key) + values[envelope.key] = value + envelopes[envelope.key] = envelope + lookup_keys[envelope.key] = keys + if keys.name is not None: + self._state.names[(type, keys.name)] = envelope.key + for index, index_value in keys.indexes.items(): + self._state.indexes[(type, index, index_value)] = envelope.key + + @staticmethod + def _retire_legacy_global(store: BackingStore, type: str, legacy_key: str) -> None: + """The first write of a global loaded from a legacy key removes that record in the same write. If + the removal fails, the new record is taken back out, so the store keeps only the legacy record.""" + try: + store.delete(type, legacy_key) + except BaseException: + try: + store.delete(type, GLOBAL_KEY) + except Exception: # noqa: BLE001 - the removal failure is the error reported + pass + raise + + @_mutating def put_global(self, document: DocumentDefinition[T], value: T) -> None: self._assert_global(document) - self.put(document, GLOBAL_KEY, value) + self._put(document, GLOBAL_KEY, value) + @_mutating def update(self, document: DocumentDefinition[T], key: str, updater: Any) -> T: + return self._update(document, key, updater) + + def _update(self, document: DocumentDefinition[T], key: str, updater: Any) -> T: current = self.get_required(document, key) updated = updater(current) - self.put(document, key, updated) + self._put(document, key, updated) return updated + @_mutating def update_global(self, document: DocumentDefinition[T], updater: Any) -> T: self._assert_global(document) - return self.update(document, GLOBAL_KEY, updater) + return self._update(document, GLOBAL_KEY, updater) + @_mutating def delete(self, document: DocumentDefinition[Any], key: str) -> None: - self._assert_registered(document) - for store in self._stores_for_type(document.type): - store.delete(document.type, key) - values = self._state.values.get(document.type) - old_value = None if values is None else values.pop(key, None) - if values == {}: - self._state.values.pop(document.type, None) - envelopes = self._state.envelopes.get(document.type) - if envelopes is not None: - envelopes.pop(key, None) - if envelopes == {}: - self._state.envelopes.pop(document.type, None) - if old_value is not None: - self._remove_value_indexes(document.type, key, old_value) + self._delete(document, key) + def _delete(self, document: DocumentDefinition[Any], key: str) -> None: + self._assert_registered(document) + store = self._store_for_type(document.type) + legacy_key = self._state.legacy_global_keys.get(document.type) if key == GLOBAL_KEY else None + if store is not None: + store.delete(document.type, key if legacy_key is None else legacy_key) + if legacy_key is not None: + del self._state.legacy_global_keys[document.type] + self._remove_lookups(document.type, key) + for by_type in (self._state.values, self._state.envelopes, self._state.lookup_keys): + held = by_type.get(document.type) + if held is not None: + held.pop(key, None) + if not held: + by_type.pop(document.type, None) + + @_mutating def delete_global(self, document: DocumentDefinition[Any]) -> None: self._assert_global(document) - self.delete(document, GLOBAL_KEY) + self._delete(document, GLOBAL_KEY) + # Snapshots iterate dicts a concurrent write would resize mid-iteration, so they hold the lock. + @_locked def snapshot(self) -> dict[str, dict[str, Any]]: out: dict[str, dict[str, Any]] = {} for type, values in self._state.values.items(): out[type] = dict(values) return out + @_locked def snapshot_envelopes(self) -> list[CultCacheEnvelope]: return [ envelope @@ -297,75 +506,70 @@ class CultCache: for envelope in envelopes.values() ] - def _stores_for_type(self, type: str) -> list[BackingStore]: - specific = self._state.stores_by_type.get(type) - return specific if specific else self._state.generic_stores + def _store_for_type(self, type: str) -> BackingStore | None: + """The store that claims the type, else the generic store. - def _all_specific_stores(self) -> list[BackingStore]: - stores: list[BackingStore] = [] - seen: set[int] = set() - for routed in self._state.stores_by_type.values(): - for store in routed: - marker = id(store) - if marker not in seen: - stores.append(store) - seen.add(marker) - return stores + Returns None only for a cache with no stores, which is an in-memory cache. + """ + store = self._state.stores_by_type.get(type, self._state.generic_store) + if store is None and self._state.stores_by_type: + raise CultCacheError(f"No backing store is home to document type: {type}") + return store - def _rebuild_indexes(self) -> None: - self._state.names.clear() - self._state.indexes.clear() - for type, values in self._state.values.items(): + @staticmethod + def _lookup_keys_for( + value: Any, name_extractor: str | Any | None, index_extractors: dict[str, str | Any] + ) -> _LookupKeys: + name = None if name_extractor is None else extract_value(value, name_extractor) + indexes: dict[str, str] = {} + for index, extractor in index_extractors.items(): + index_value = extract_value(value, extractor) + if index_value is not None: + indexes[index] = str(index_value) + return _LookupKeys(None if name is None else str(name), indexes) + + def _derive_lookups( + self, + values_by_type: dict[str, dict[str, Any]], + name_extractors: dict[str, str | Any], + index_extractors: dict[str, dict[str, str | Any]], + ) -> tuple[_Names, _Indexes, _LookupKeysByType]: + """Builds name and index lookups for the given values in fresh dicts, touching no cache state.""" + names: _Names = {} + indexes: _Indexes = {} + lookup_keys: _LookupKeysByType = {} + for type, values in values_by_type.items(): + name_extractor = name_extractors.get(type) + type_index_extractors = index_extractors.get(type, {}) for key, value in values.items(): - self._add_value_indexes(type, key, value) + keys = self._lookup_keys_for(value, name_extractor, type_index_extractors) + lookup_keys.setdefault(type, {})[key] = keys + if keys.name is not None: + names[(type, keys.name)] = key + for index, index_value in keys.indexes.items(): + indexes[(type, index, index_value)] = key + return names, indexes, lookup_keys - def _add_value_indexes(self, type: str, key: str, value: Any) -> None: - name_extractor = self._state.name_extractors.get(type) - if name_extractor is not None: - name = extract_value(value, name_extractor) - if name is not None: - self._state.names[(type, str(name))] = key - for index, extractor in self._state.index_extractors.get(type, {}).items(): - index_value = extract_value(value, extractor) - if index_value is not None: - self._state.indexes[(type, index, str(index_value))] = key - - def _remove_value_indexes(self, type: str, key: str, value: Any) -> None: - name_extractor = self._state.name_extractors.get(type) - if name_extractor is not None: - name = extract_value(value, name_extractor) - if name is not None: - self._remove_name_index(type, key, str(name)) - for index, extractor in self._state.index_extractors.get(type, {}).items(): - index_value = extract_value(value, extractor) - if index_value is not None: - self._remove_secondary_index(type, key, index, str(index_value)) - - def _remove_name_index(self, type: str, key: str, name: str) -> None: - lookup_key = (type, name) - if self._state.names.get(lookup_key) != key: + def _remove_lookups(self, type: str, key: str) -> None: + """Removes the lookups the held record produced at admission, handing a name or index value to + another held record that produced it too. Reads stored lookup keys only; runs no extractor.""" + held = self._state.lookup_keys.get(type, {}) + keys = held.get(key) + if keys is None: return - self._state.names.pop(lookup_key, None) - name_extractor = self._state.name_extractors.get(type) - if name_extractor is None: - return - for candidate_key, candidate in self._state.values.get(type, {}).items(): - candidate_name = extract_value(candidate, name_extractor) - if candidate_key != key and candidate_name is not None and str(candidate_name) == name: - self._state.names[lookup_key] = candidate_key - - def _remove_secondary_index(self, type: str, key: str, index: str, value: str) -> None: - lookup_key = (type, index, value) - if self._state.indexes.get(lookup_key) != key: - return - self._state.indexes.pop(lookup_key, None) - extractor = self._state.index_extractors.get(type, {}).get(index) - if extractor is None: - return - for candidate_key, candidate in self._state.values.get(type, {}).items(): - candidate_value = extract_value(candidate, extractor) - if candidate_key != key and candidate_value is not None and str(candidate_value) == value: - self._state.indexes[lookup_key] = candidate_key + if keys.name is not None and self._state.names.get((type, keys.name)) == key: + del self._state.names[(type, keys.name)] + for candidate_key, candidate in held.items(): + if candidate_key != key and candidate.name == keys.name: + self._state.names[(type, keys.name)] = candidate_key + for index, index_value in keys.indexes.items(): + lookup_key = (type, index, index_value) + if self._state.indexes.get(lookup_key) != key: + continue + del self._state.indexes[lookup_key] + for candidate_key, candidate in held.items(): + if candidate_key != key and candidate.indexes.get(index) == index_value: + self._state.indexes[lookup_key] = candidate_key def _assert_registered(self, document: DocumentDefinition[Any]) -> None: if self._state.documents.get(document.type) is not document: diff --git a/packages/cultcache-py/src/cultmesh_py/simulation.py b/packages/cultcache-py/src/cultmesh_py/simulation.py deleted file mode 100644 index 4b727fd5..00000000 --- a/packages/cultcache-py/src/cultmesh_py/simulation.py +++ /dev/null @@ -1,135 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass -from datetime import UTC, datetime -import hashlib -from typing import Any - -from cultcache_py import define_database_entry_type -from cultcache_py.documents import DocumentDefinition - -from .node import CultMeshNode - -SIMULATION_FACT_DOCUMENT_TYPE = "gamecult.mesh.simulation_fact" -SIMULATION_FACT_SCHEMA_VERSION = "gamecult.mesh.simulation_fact.v1" - - -@dataclass(frozen=True) -class CultMeshSimulationFact: - fact_id: str - shard_id: str - shard_epoch: int - frame: int - subject_id: str - claim_kind: str - claim_hash: str - claim_summary: str | None - witness_count: int - support_weight: float - total_weight: float - confidence: float - committed_at: str - - @staticmethod - def compute_fact_id(candidate: dict[str, Any]) -> str: - canonical = "\x1f".join([ - _candidate_string(candidate, "shardId", "shard_id"), - str(_candidate_int(candidate, "shardEpoch", "shard_epoch")), - str(_candidate_int(candidate, "frame", "frame")), - _candidate_string(candidate, "subjectId", "subject_id"), - _candidate_string(candidate, "claimKind", "claim_kind"), - ]) - return hashlib.sha256(canonical.encode("utf-8")).hexdigest() - - @staticmethod - def create_record_key(candidate: dict[str, Any]) -> str: - return f"simulation:{CultMeshSimulationFact.compute_fact_id(candidate)}" - - @staticmethod - def from_candidate(candidate: dict[str, Any], *, committed_at: str | None = None) -> "CultMeshSimulationFact": - return CultMeshSimulationFact( - fact_id=CultMeshSimulationFact.compute_fact_id(candidate), - shard_id=_candidate_string(candidate, "shardId", "shard_id"), - shard_epoch=_candidate_int(candidate, "shardEpoch", "shard_epoch"), - frame=_candidate_int(candidate, "frame", "frame"), - subject_id=_candidate_string(candidate, "subjectId", "subject_id"), - claim_kind=_candidate_string(candidate, "claimKind", "claim_kind"), - claim_hash=_candidate_string(candidate, "claimHash", "claim_hash"), - claim_summary=_candidate_optional_string(candidate, "claimSummary", "claim_summary"), - witness_count=_candidate_int(candidate, "witnessCount", "witness_count"), - support_weight=_candidate_float(candidate, "supportWeight", "support_weight"), - total_weight=_candidate_float(candidate, "totalWeight", "total_weight"), - confidence=_candidate_float(candidate, "confidence", "confidence"), - committed_at=committed_at or datetime.now(UTC).isoformat(), - ) - - -simulation_fact_document: DocumentDefinition[CultMeshSimulationFact] = define_database_entry_type( - SIMULATION_FACT_DOCUMENT_TYPE, - [ - ("fact_id", 0), - ("shard_id", 1), - ("shard_epoch", 2), - ("frame", 3), - ("subject_id", 4), - ("claim_kind", 5), - ("claim_hash", 6), - ("claim_summary", 7, None), - ("witness_count", 8), - ("support_weight", 9), - ("total_weight", 10), - ("confidence", 11), - ("committed_at", 12), - ], - cls=CultMeshSimulationFact, - name="fact_id", - indexes={"shard": "shard_id", "frame": "frame", "subject": "subject_id", "claim_kind": "claim_kind"}, - schema_name=SIMULATION_FACT_DOCUMENT_TYPE, - schema_version=SIMULATION_FACT_SCHEMA_VERSION, -) - - -@dataclass(frozen=True) -class CultMeshSimulationFactCommit: - key: str - fact: CultMeshSimulationFact - - -@dataclass -class CultMeshSimulationFactCommitter: - node: CultMeshNode - document: DocumentDefinition[CultMeshSimulationFact] = simulation_fact_document - - def __post_init__(self) -> None: - if all(document.type != self.document.type for document in self.node.documents): - self.node.register_document(self.document) - - def commit(self, candidate: dict[str, Any], *, committed_at: str | None = None) -> CultMeshSimulationFactCommit: - if not bool(candidate.get("hasQuorum") if "hasQuorum" in candidate else candidate.get("has_quorum")): - raise ValueError("Simulation candidate cannot be committed before quorum") - key = CultMeshSimulationFact.create_record_key(candidate) - fact = CultMeshSimulationFact.from_candidate(candidate, committed_at=committed_at) - self.node.put(self.document, key, fact) - return CultMeshSimulationFactCommit(key=key, fact=fact) - - -def _candidate_string(candidate: dict[str, Any], camel: str, snake: str) -> str: - value = candidate.get(camel, candidate.get(snake, "")) - if value is None: - return "" - return str(value) - - -def _candidate_optional_string(candidate: dict[str, Any], camel: str, snake: str) -> str | None: - value = candidate.get(camel, candidate.get(snake)) - if value is None: - return None - return str(value) - - -def _candidate_int(candidate: dict[str, Any], camel: str, snake: str) -> int: - return int(candidate.get(camel, candidate.get(snake, 0))) - - -def _candidate_float(candidate: dict[str, Any], camel: str, snake: str) -> float: - return float(candidate.get(camel, candidate.get(snake, 0.0))) diff --git a/packages/cultcache-py/tests/test_cultcache.py b/packages/cultcache-py/tests/test_cultcache.py index 3ecb763f..ac81cc6d 100644 --- a/packages/cultcache-py/tests/test_cultcache.py +++ b/packages/cultcache-py/tests/test_cultcache.py @@ -1,23 +1,12 @@ -from __future__ import annotations +from __future__ import annotations -import json -import os -import socket -import subprocess -import sys import tempfile import threading -import time import unittest -import hashlib -import hmac -import io -from dataclasses import asdict, dataclass -from datetime import UTC, datetime, timedelta +from dataclasses import asdict, dataclass, replace from pathlib import Path -from typing import Callable -from uuid import uuid4 - +from cultcache_py.backing_store import CultCacheEnvelope +from cultcache_py.cache import CultCacheError from cultcache_py import ( CultCache, JsonLinesBackingStore, @@ -25,155 +14,7 @@ from cultcache_py import ( define_database_entry_type, define_document_type, ) -from cultcache_py.benchmark import run_benchmark -from cultcache_py.compare_csharp import ( - DEFAULT_PARITY_THRESHOLD, - DEFAULT_SAMPLE_COUNT, - _compare_common_metrics, - _median_result, - _parity_status, -) -from cultcache_py.interop import INTEROP_SCHEMA_VERSION, interop_note_document, read_note, write_note -from cultcache_py.verify import verify -from cultnet_py.interop_peer import append_shard_log_put, build_state, raw_snapshot_response -from cultnet_py import ( - compute_simulation_claim_hash, - CultNetDatabaseChange, - CultNetClientAuthorityScope, - CultNetRawClient, - CultNetRawDocumentRecord, - CultNetRawSnapshotResponse, - CultNetSchemaCatalog, - CultNetSchemaDescriptor, - CultNetShardCatalog, - CultNetShardDescriptor, - CultNetShardLogEntry, - CultNetShardLogResponse, - CultNetFileShardReplicaCursorStore, - CultNetFileShardMutationLogStore, - CultNetInMemoryShardReplicaCursorStore, - CultNetSchemaWriteForwarder, - CultNetSchemaShardLogFetcher, - CultNetSchemaShardSnapshotFetcher, - CultNetPeerError, - CultNetClientSecurityOptions, - CultNetSecret, - CultNetServerSecurityOptions, - CultNetShardReplicator, - CultNetShardReplicatorOptions, - CultNetShardMutationLogStore, - CultNetShardWriteForwarder, - CultNetSimulationConsensusOptions, - CultNetSimulationConsensusCandidate, - CultNetSimulationObservation, - CultNetSimulationObservationHub, - CultNetRudpPacket, - CultNetRudpPacketType, - CultNetRudpSendOptions, - CultNetRudpSession, - CultNetRudpSessionOptions, - CultNetRudpSocketMode, - CultNetRudpReconnectLoop, - CultNetRudpSocketTransportConnection, - CultNetRudpSocketTransportOptions, - TcpFramedTransportConnection, - CultNetWitnessArtifactBundle, - apply_raw_document_record, - apply_raw_snapshot, - apply_shard_log_response, - create_rudp_transport_profile, - create_tcp_framed_transport_profile, - create_reconnect_policy, - CultNetReconnectController, - database_subscribe, - database_unsubscribe, - decode_frame, - decode_rudp_packet, - decode_witness_artifact_bundle_payload, - document_delete, - document_put_raw, - encode_frame, - encode_rudp_packet, - encode_witness_artifact_bundle_payload, - hello, - login, - login_success, - parse_message, - register, - schema_catalog_request, - schema_document_map, - shard_catalog_request, - shard_log_request, - simulation_observation, - snapshot_request, - verify_session, - wire_message_schema_catalog, - wire_message_schema_descriptors, - witness_artifact_bundle, - compute_reconnect_delay_ms, -) -from cultmesh_py import create_node -from cultmesh_py import ( - CultMesh, - CultMeshDatabase, - CultMeshDatabaseChange, - CultMeshGameSessionOptions, - CultMeshNodeOptions, - CultMeshReactiveDocumentOptions, - CultMeshPeerCard, - CultMeshPeerCatalog, - CultMeshSimulationFact, - CultMeshAuthorityLease, - CultMeshAuthorityLeaseCatalog, - CultMeshHmacAuthorityLeaseVerifier, - CultMeshDiscoveryClient, - CultMeshPeerExchangeClient, - CultMeshPeerHealthMonitor, - CultMeshSimulationObservationFanout, - CultMeshSnapshotFanout, - CultMeshStreamCatalog, - CultMeshStreamConsumerProfile, - CultMeshStreamDescriptor, - CultMeshStreamFrameHandle, - CultMeshVerseCatalog, - CultMeshVerseCompatibility, - CultMeshVerseDescriptor, - CultMeshVerseDiscoveryClient, - READY_SCHEMA_VERSION, - peer_exchange_request, - simulation_fact_document, - verse_catalog_request, -) - - -def bind_udp_socket() -> socket.socket: - sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - sock.bind(("127.0.0.1", 0)) - sock.settimeout(0.02) - return sock - - -def pump_rudp_handshake( - client: CultNetRudpSocketTransportConnection, - server: CultNetRudpSocketTransportConnection, -) -> None: - for _ in range(20): - server.receive_once() - client.receive_once() - server.receive_once() - if client.connected and server.connected: - return - time.sleep(0.005) - raise AssertionError("RUDP socket handshake did not complete") - - -def receive_rudp_frame(transport: CultNetRudpSocketTransportConnection): - for _ in range(20): - frame = transport.receive_once() - if frame is not None: - return frame - time.sleep(0.005) - raise AssertionError("RUDP socket frame was not delivered") +from cultcache_py.interop import read_note, write_note @dataclass @@ -194,14 +35,6 @@ def item_doc(): class CultCacheTests(unittest.TestCase): - def _wait_until(self, predicate: Callable[[], bool], *, timeout_seconds: float = 2.0) -> None: - deadline = time.monotonic() + timeout_seconds - while time.monotonic() < deadline: - if predicate(): - return - time.sleep(0.01) - self.fail("Timed out waiting for condition") - def test_round_trips_registered_documents(self) -> None: with tempfile.TemporaryDirectory() as tmp: document = item_doc() @@ -408,100 +241,549 @@ class CultCacheTests(unittest.TestCase): self.assertEqual(loaded["authorRuntimeId"], "python-test") self.assertIn("interop", loaded["tags"]) - def test_benchmark_reports_core_hot_path_metrics(self) -> None: - result = run_benchmark(8) - metric_names = {metric["name"] for metric in result["metrics"]} - self.assertEqual(result["records"], 8) - self.assertEqual(result["wireMessageCount"], 8) - self.assertEqual(metric_names, { - "database_entry_encode", - "database_entry_decode", - "cultnet_frame_parse", - "raw_snapshot_apply", - "cache_upsert", - "cache_get", - }) - self.assertTrue(all(metric["opsPerSecond"] > 0 for metric in result["metrics"])) + def test_second_generic_store_rejected(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + cache = CultCache() + cache.add_generic_store(SingleFileMessagePackBackingStore(Path(tmp) / "a.cc")) + with self.assertRaisesRegex(CultCacheError, "second generic store"): + cache.add_generic_store(SingleFileMessagePackBackingStore(Path(tmp) / "b.cc")) - def test_compare_csharp_median_result_summarizes_samples(self) -> None: - self.assertEqual(DEFAULT_SAMPLE_COUNT, 3) - self.assertEqual(DEFAULT_PARITY_THRESHOLD, 0.90) - samples = [ - { - "runtime": "python", - "records": 3, - "metrics": [ - {"name": "cache_get", "operations": 3, "elapsedMs": 30.0, "opsPerSecond": 100.0}, - {"name": "cache_upsert", "operations": 3, "elapsedMs": 60.0, "opsPerSecond": 50.0}, - ], - }, - { - "runtime": "python", - "records": 3, - "metrics": [ - {"name": "cache_get", "operations": 3, "elapsedMs": 10.0, "opsPerSecond": 300.0}, - {"name": "cache_upsert", "operations": 3, "elapsedMs": 20.0, "opsPerSecond": 150.0}, - ], - }, - { - "runtime": "python", - "records": 3, - "metrics": [ - {"name": "cache_get", "operations": 3, "elapsedMs": 20.0, "opsPerSecond": 200.0}, - {"name": "cache_upsert", "operations": 3, "elapsedMs": 40.0, "opsPerSecond": 75.0}, - ], - }, - ] + def test_type_claimed_twice_rejected(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + cache = CultCache() + cache.add_backing_store(SingleFileMessagePackBackingStore(Path(tmp) / "a.cc"), ["item"]) + with self.assertRaisesRegex(CultCacheError, "already routed"): + cache.add_backing_store(SingleFileMessagePackBackingStore(Path(tmp) / "b.cc"), ["settings", "item"]) + # A refused registration claims nothing, so "settings" is still free. + cache.add_backing_store(SingleFileMessagePackBackingStore(Path(tmp) / "c.cc"), ["settings"]) - summarized = _median_result(samples) + def test_types_route_to_home_store(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + item = item_doc() + settings = define_database_entry_type("settings", [("theme", 0)]) + generic_path = Path(tmp) / "generic.cc" + settings_path = Path(tmp) / "settings.cc" - self.assertEqual(summarized["sampleCount"], 3) - self.assertEqual(summarized["metrics"][0]["name"], "cache_get") - self.assertEqual(summarized["metrics"][0]["elapsedMs"], 20.0) - self.assertEqual(summarized["metrics"][0]["opsPerSecond"], 200.0) - self.assertEqual(summarized["metrics"][1]["opsPerSecond"], 75.0) + def build() -> CultCache: + return ( + CultCache.builder() + .register_document_type(item) + .register_document_type(settings) + .add_backing_store(SingleFileMessagePackBackingStore(settings_path), ["settings"]) + .add_generic_store(SingleFileMessagePackBackingStore(generic_path)) + .build() + ) - def test_compare_csharp_marks_threshold_status_per_metric(self) -> None: - python_result = { - "metrics": [ - {"name": "cache_get", "operations": 3, "elapsedMs": 2.0, "opsPerSecond": 50.0}, - {"name": "cache_upsert", "operations": 3, "elapsedMs": 1.0, "opsPerSecond": 95.0}, - ], - } - csharp_result = { - "metrics": [ - {"name": "cache_get", "operations": 3, "elapsedMs": 1.0, "opsPerSecond": 100.0}, - {"name": "cache_upsert", "operations": 3, "elapsedMs": 1.0, "opsPerSecond": 100.0}, - ], - } + cache = build() + cache.put(item, "item:potion", Item(name="Potion", category="Consumable", value=50)) + cache.put(settings, "app", {"theme": "ash"}) - comparison = _compare_common_metrics(python_result, csharp_result, parity_threshold=0.90) - by_name = {item["name"]: item for item in comparison} + generic = SingleFileMessagePackBackingStore(generic_path).pull_all() + routed = SingleFileMessagePackBackingStore(settings_path).pull_all() + self.assertEqual([(e.key, e.type) for e in generic], [("item:potion", "item")]) + self.assertEqual([(e.key, e.type) for e in routed], [("app", "settings")]) - self.assertFalse(by_name["cache_get"]["meetsParityThreshold"]) - self.assertTrue(by_name["cache_upsert"]["meetsParityThreshold"]) - self.assertEqual(_parity_status(comparison, "ok"), "below-threshold") - self.assertEqual(_parity_status([by_name["cache_upsert"]], "ok"), "meets-threshold") - self.assertEqual(_parity_status(comparison, "failed"), "unknown") + cache.delete(settings, "app") + self.assertEqual(SingleFileMessagePackBackingStore(settings_path).pull_all(), []) + self.assertEqual(len(SingleFileMessagePackBackingStore(generic_path).pull_all()), 1) - def test_compare_csharp_marks_meets_threshold_when_all_common_metrics_pass(self) -> None: - comparison = _compare_common_metrics( - { - "metrics": [ - {"name": "cache_get", "operations": 3, "elapsedMs": 1.0, "opsPerSecond": 95.0}, - {"name": "cache_upsert", "operations": 3, "elapsedMs": 1.0, "opsPerSecond": 120.0}, - ], - }, - { - "metrics": [ - {"name": "cache_get", "operations": 3, "elapsedMs": 1.0, "opsPerSecond": 100.0}, - {"name": "cache_upsert", "operations": 3, "elapsedMs": 1.0, "opsPerSecond": 100.0}, - ], - }, - parity_threshold=0.90, + reloaded = build() + reloaded.pull_all_backing_stores() + self.assertEqual(reloaded.get_required(item, "item:potion").value, 50) + self.assertIsNone(reloaded.get(settings, "app")) + + def test_attach_that_would_move_a_held_record_home_is_refused(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + settings = define_database_entry_type("settings", [("theme", 0)]) + generic_path = Path(tmp) / "generic.cc" + settings_path = Path(tmp) / "settings.cc" + cache = ( + CultCache.builder() + .register_document_type(settings) + .add_generic_store(SingleFileMessagePackBackingStore(generic_path)) + .build() + ) + cache.put(settings, "app", {"theme": "v1-generic"}) + + with self.assertRaisesRegex( + CultCacheError, "would move settings from the generic store to the store routed to settings" + ): + cache.add_backing_store(SingleFileMessagePackBackingStore(settings_path), ["settings"]) + # Nothing attached: the next write still lands in the generic store. + cache.put(settings, "app", {"theme": "v2-generic"}) + generic = SingleFileMessagePackBackingStore(generic_path).pull_all() + self.assertEqual([(e.key, e.type) for e in generic], [("app", "settings")]) + self.assertFalse(settings_path.exists()) + + def test_load_from_a_store_that_is_not_the_record_home_is_refused(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + settings = define_database_entry_type("settings", [("theme", 0)]) + generic_path = Path(tmp) / "generic.cc" + settings_path = Path(tmp) / "settings.cc" + writer = CultCache.builder().register_document_type(settings).add_generic_store( + SingleFileMessagePackBackingStore(generic_path) + ).build() + writer.put(settings, "app", {"theme": "stray"}) + generic_bytes = generic_path.read_bytes() + + cache = ( + CultCache.builder() + .register_document_type(settings) + .add_backing_store(SingleFileMessagePackBackingStore(settings_path), ["settings"]) + .add_generic_store(SingleFileMessagePackBackingStore(generic_path)) + .build() + ) + # The cache already holds a record, in its home store, before the refused load. + cache.put(settings, "held", {"theme": "home"}) + before = cache.snapshot_envelopes() + settings_bytes = settings_path.read_bytes() + with self.assertRaisesRegex( + CultCacheError, + "settings record app was loaded from the generic store, but its home is the store routed to settings", + ): + cache.pull_all_backing_stores() + self.assertIsNone(cache.get(settings, "app")) + self.assertEqual(cache.snapshot_envelopes(), before) + self.assertEqual(cache.get_required(settings, "held"), {"theme": "home"}) + self.assertEqual(generic_path.read_bytes(), generic_bytes) + self.assertEqual(settings_path.read_bytes(), settings_bytes) + + def test_pull_with_duplicate_global_leaves_cache_unchanged(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + item = item_doc() + settings = define_document_type("settings", global_document=True) + store_path = Path(tmp) / "generic.cc" + cache = ( + CultCache.builder() + .register_document_type(item) + .register_document_type(settings) + .add_generic_store(SingleFileMessagePackBackingStore(store_path)) + .build() + ) + cache.put(item, "item:potion", Item(name="Potion", category="Consumable", value=50)) + cache.put_global(settings, {"theme": "ash"}) + before = cache.snapshot_envelopes() + + # A second record of the global type lands in the store file behind the cache's back. + stray = replace(cache.get_required_envelope(settings, CultCache.GLOBAL_KEY), key="stray") + SingleFileMessagePackBackingStore(store_path).push(stray) + + with self.assertRaisesRegex(CultCacheError, "Duplicate global document for type: settings"): + cache.pull_all_backing_stores() + self.assertEqual(cache.snapshot_envelopes(), before) + self.assertEqual(cache.get_required(item, "item:potion").value, 50) + self.assertEqual(cache.get_key_by_name(item, "Potion"), "item:potion") + self.assertEqual(cache.get_required_global(settings)["theme"], "ash") + + def test_pull_with_throwing_index_accessor_leaves_cache_unchanged(self) -> None: + def category(item: Item) -> str: + if item.name == "Bomb": + raise ValueError("category accessor refuses Bomb") + return item.category + + with tempfile.TemporaryDirectory() as tmp: + item = define_document_type( + "item", + encode=lambda value: asdict(value), + decode=lambda raw: Item(**raw), + name="name", + indexes={"category": category}, + ) + settings = define_document_type("settings", global_document=True) + store_path = Path(tmp) / "generic.cc" + cache = ( + CultCache.builder() + .register_document_type(item) + .register_document_type(settings) + .add_generic_store(SingleFileMessagePackBackingStore(store_path)) + .build() + ) + cache.put(item, "item:potion", Item(name="Potion", category="Consumable", value=50)) + cache.put_global(settings, {"theme": "ash"}) + before_envelopes = cache.snapshot_envelopes() + before_values = cache.snapshot() + + # A record whose index accessor throws lands in the store file behind the cache's back. + bomb = replace( + cache.get_required_envelope(item, "item:potion"), + key="item:bomb", + payload=item.encode_payload(Item(name="Bomb", category="Explosive", value=1)), + ) + SingleFileMessagePackBackingStore(store_path).push(bomb) + + with self.assertRaisesRegex(ValueError, "category accessor refuses Bomb"): + cache.pull_all_backing_stores() + self.assertEqual(cache.snapshot_envelopes(), before_envelopes) + self.assertEqual(cache.snapshot(), before_values) + self.assertEqual(cache.get_required(item, "item:potion").value, 50) + self.assertIsNone(cache.get(item, "item:bomb")) + self.assertEqual(cache.get_key_by_name(item, "Potion"), "item:potion") + self.assertIsNone(cache.get_key_by_name(item, "Bomb")) + self.assertEqual(cache.get_key_by_index(item, "category", "Consumable"), "item:potion") + self.assertEqual(cache.get_required_global(settings)["theme"], "ash") + + def _store_state(self, path: Path) -> list[tuple[str, str, bytes]]: + return sorted((e.type, e.key, e.payload) for e in SingleFileMessagePackBackingStore(path).pull_all()) + + def test_put_with_throwing_name_extractor_changes_neither_store_nor_cache(self) -> None: + def name(value: dict) -> str: + if value["n"] == "bad": + raise ValueError("extractor refuses bad") + return value["n"] + + with tempfile.TemporaryDirectory() as tmp: + item = define_document_type("item", name=name) + path = Path(tmp) / "generic.cc" + cache = CultCache.builder().register_document_type(item).add_generic_store( + SingleFileMessagePackBackingStore(path) + ).build() + cache.put(item, "k", {"n": "good"}) + store_before, cache_before = self._store_state(path), cache.snapshot_envelopes() + + with self.assertRaisesRegex(ValueError, "extractor refuses bad"): + cache.put(item, "k", {"n": "bad"}) + self.assertEqual(self._store_state(path), store_before) + self.assertEqual(cache.snapshot_envelopes(), cache_before) + self.assertEqual(cache.get_required(item, "k"), {"n": "good"}) + self.assertEqual(cache.get_key_by_name(item, "good"), "k") + + def test_overwrite_under_a_registered_extractor_that_now_raises_changes_neither_store_nor_cache(self) -> None: + refusing = {"on": False} + + def category(value: dict) -> str: + if refusing["on"]: + raise ValueError("extractor refuses now") + return value["c"] + + with tempfile.TemporaryDirectory() as tmp: + item = define_document_type("item") + path = Path(tmp) / "generic.cc" + cache = CultCache.builder().register_document_type(item).add_generic_store( + SingleFileMessagePackBackingStore(path) + ).build() + cache.put(item, "k", {"c": "old"}) + cache.register_index(item, "category", category) + refusing["on"] = True + store_before, cache_before = self._store_state(path), cache.snapshot_envelopes() + + with self.assertRaisesRegex(ValueError, "extractor refuses now"): + cache.put(item, "k", {"c": "new"}) + self.assertEqual(self._store_state(path), store_before) + self.assertEqual(cache.snapshot_envelopes(), cache_before) + self.assertEqual(cache.get_key_by_index(item, "category", "old"), "k") + # Deleting runs no extractor: it uses the lookup keys stored when the record was admitted. + cache.delete(item, "k") + self.assertIsNone(cache.get_key_by_index(item, "category", "old")) + + def test_registering_an_extractor_that_raises_on_a_held_value_installs_nothing(self) -> None: + def name(value: dict) -> str: + if value["n"] == "bad": + raise ValueError("extractor refuses bad") + return value["n"] + + with tempfile.TemporaryDirectory() as tmp: + item = define_document_type("item") + path = Path(tmp) / "generic.cc" + cache = CultCache.builder().register_document_type(item).add_generic_store( + SingleFileMessagePackBackingStore(path) + ).build() + cache.put(item, "old", {"n": "bad"}) + with self.assertRaisesRegex(ValueError, "extractor refuses bad"): + cache.register_name_lookup(item, name) + cache.put(item, "old", {"n": "fine"}) + self.assertEqual(cache.get_required(item, "old"), {"n": "fine"}) + self.assertIsNone(cache.get_key_by_name(item, "fine")) + self.assertEqual([item.decode_payload(e.payload) for e in SingleFileMessagePackBackingStore(path).pull_all()], [{"n": "fine"}]) + + def test_put_envelopes_with_throwing_extractor_changes_neither_store_nor_cache(self) -> None: + def name(value: dict) -> str: + if value["n"] == "bad": + raise ValueError("extractor refuses bad") + return value["n"] + + with tempfile.TemporaryDirectory() as tmp: + item = define_document_type("item", name=name) + path = Path(tmp) / "generic.cc" + cache = CultCache.builder().register_document_type(item).add_generic_store( + SingleFileMessagePackBackingStore(path) + ).build() + cache.put(item, "seed", {"n": "seed"}) + seed = cache.get_required_envelope(item, "seed") + first = replace(seed, key="a", payload=item.encode_payload({"n": "a"})) + second = replace(seed, key="b", payload=item.encode_payload({"n": "bad"})) + store_before, cache_before = self._store_state(path), cache.snapshot_envelopes() + + with self.assertRaisesRegex(ValueError, "extractor refuses bad"): + cache.put_envelopes(item, [first, second]) + self.assertEqual(self._store_state(path), store_before) + self.assertEqual(cache.snapshot_envelopes(), cache_before) + self.assertIsNone(cache.get_key_by_name(item, "a")) + + def test_global_replace_with_failing_store_push_keeps_old_global(self) -> None: + class FailingPushStore(SingleFileMessagePackBackingStore): + fail = False + + def push(self, envelope): + if self.fail: + raise OSError("disk full") + super().push(envelope) + + with tempfile.TemporaryDirectory() as tmp: + settings = define_document_type("settings", global_document=True) + path = Path(tmp) / "generic.cc" + store = FailingPushStore(path) + cache = CultCache.builder().register_document_type(settings).add_generic_store(store).build() + cache.put_global(settings, {"theme": "base"}) + store_before, cache_before = self._store_state(path), cache.snapshot_envelopes() + store.fail = True + + with self.assertRaisesRegex(OSError, "disk full"): + cache.put_global(settings, {"theme": "new"}) + with self.assertRaisesRegex(CultCacheError, "must use key __global__"): + cache.put(settings, "other", {"theme": "new"}) + self.assertEqual(self._store_state(path), store_before) + self.assertEqual(cache.snapshot_envelopes(), cache_before) + self.assertEqual(cache.get_required_global(settings), {"theme": "base"}) + + @staticmethod + def _pausing_store(path: Path, barrier: threading.Barrier, resume: threading.Event): + """A store whose first push writes, meets the other thread at the barrier, then waits until that + thread's call has returned (or a timeout: a locked cache makes the other thread wait on us).""" + + class PausingStore(SingleFileMessagePackBackingStore): + pushes = 0 + paused = False + other_reached_store_while_paused = False + + def push(self, envelope): + self.other_reached_store_while_paused |= self.paused + self.pushes += 1 + super().push(envelope) + if self.pushes == 1: + self.paused = True + barrier.wait(timeout=5) + resume.wait(timeout=0.5) + self.paused = False + + return PausingStore(path) + + @staticmethod + def _run_racing(first, second, resume: threading.Event) -> list[BaseException]: + errors: list[BaseException] = [] + + def run(action, signal): + try: + action() + except BaseException as error: # noqa: BLE001 - reported to the test + errors.append(error) + finally: + if signal: + resume.set() + + threads = [threading.Thread(target=run, args=(first, False)), threading.Thread(target=run, args=(second, True))] + for thread in threads: + thread.start() + for thread in threads: + thread.join(timeout=10) + return errors + + def test_concurrent_put_and_attach_cannot_land_record_in_non_home_store(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + settings = define_database_entry_type("settings", [("theme", 0)]) + generic_path, settings_path = Path(tmp) / "generic.cc", Path(tmp) / "settings.cc" + barrier, resume = threading.Barrier(2), threading.Event() + generic = self._pausing_store(generic_path, barrier, resume) + cache = CultCache.builder().register_document_type(settings).add_generic_store(generic).build() + + def attach() -> None: + barrier.wait(timeout=5) + cache.add_backing_store(SingleFileMessagePackBackingStore(settings_path), ["settings"]) + + errors = self._run_racing(lambda: cache.put(settings, "app", {"theme": "t"}), attach, resume) + + # Either the attach was refused (the record is held, so its home cannot move) or it won; + # in every case each held record must be in the store that is its home. + home = SingleFileMessagePackBackingStore(generic_path if errors else settings_path) + self.assertEqual([(e.key, e.type) for e in home.pull_all()], [("app", "settings")]) + self.assertTrue(all(isinstance(error, CultCacheError) for error in errors), errors) + + def test_concurrent_global_puts_cannot_both_reach_disk(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + settings = define_document_type("settings", global_document=True) + path = Path(tmp) / "generic.cc" + barrier, resume = threading.Barrier(2), threading.Event() + store = self._pausing_store(path, barrier, resume) + cache = CultCache.builder().register_document_type(settings).add_generic_store(store).build() + + def second() -> None: + barrier.wait(timeout=5) + cache.put_global(settings, {"theme": "second"}) + + errors = self._run_racing(lambda: cache.put_global(settings, {"theme": "first"}), second, resume) + + self.assertEqual(errors, []) + # While the first write was between its store push and its apply, the second never reached disk. + self.assertFalse(store.other_reached_store_while_paused) + on_disk = SingleFileMessagePackBackingStore(path).pull_all() + self.assertEqual(len(on_disk), 1) + self.assertEqual(on_disk[0].payload, cache.get_required_envelope(settings, CultCache.GLOBAL_KEY).payload) + self.assertEqual(cache.get_required_global(settings), {"theme": "second"}) + + @staticmethod + def _envelope(document, key: str, value) -> CultCacheEnvelope: + catalog_entry = document.catalog_entry() + return CultCacheEnvelope.create( + key=key, + type=document.type, + payload=document.encode_payload(value), + schema_id=catalog_entry.schema_id, + catalog_entry=catalog_entry, ) - self.assertEqual(_parity_status(comparison, "ok"), "meets-threshold") + def test_legacy_key_global_loads_without_writing_and_first_write_leaves_one_global(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + settings = define_document_type("settings", global_document=True) + path = Path(tmp) / "generic.cc" + + def build() -> CultCache: + return CultCache.builder().register_document_type(settings).add_generic_store( + SingleFileMessagePackBackingStore(path) + ).build() + + SingleFileMessagePackBackingStore(path).push(self._envelope(settings, "legacy", {"theme": "old"})) + stored_bytes = path.read_bytes() + + cache = build() + cache.pull_all_backing_stores() + self.assertEqual(cache.get_global(settings), {"theme": "old"}) + self.assertEqual(cache.get(settings, CultCache.GLOBAL_KEY), {"theme": "old"}) + self.assertEqual(path.read_bytes(), stored_bytes) + + cache.put_global(settings, {"theme": "new"}) + self.assertEqual([e.key for e in SingleFileMessagePackBackingStore(path).pull_all()], [CultCache.GLOBAL_KEY]) + reloaded = build() + reloaded.pull_all_backing_stores() + self.assertEqual(reloaded.get_required_global(settings), {"theme": "new"}) + + def test_deleting_a_legacy_key_global_removes_the_legacy_record(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + settings = define_document_type("settings", global_document=True) + path = Path(tmp) / "generic.cc" + SingleFileMessagePackBackingStore(path).push(self._envelope(settings, "legacy", {"theme": "old"})) + cache = CultCache.builder().register_document_type(settings).add_generic_store( + SingleFileMessagePackBackingStore(path) + ).build() + cache.pull_all_backing_stores() + + cache.delete_global(settings) + self.assertEqual(SingleFileMessagePackBackingStore(path).pull_all(), []) + self.assertIsNone(cache.get_global(settings)) + + def test_two_globals_of_one_type_on_disk_are_refused_under_any_keys(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + settings = define_document_type("settings", global_document=True) + for keys in (["one", "two"], ["legacy", CultCache.GLOBAL_KEY]): + path = Path(tmp) / f"{keys[0]}.cc" + store = SingleFileMessagePackBackingStore(path) + store.push_all([self._envelope(settings, key, {"theme": key}) for key in keys]) + cache = CultCache.builder().register_document_type(settings).add_generic_store(store).build() + with self.assertRaisesRegex(CultCacheError, "Duplicate global document for type: settings"): + cache.pull_all_backing_stores() + self.assertIsNone(cache.get_global(settings)) + + def test_decoder_writing_during_pull_raises_and_changes_neither_store_nor_cache(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "generic.cc" + holder: dict[str, CultCache] = {} + + def decode(raw): + holder["cache"].put(item, "side", {"name": "side"}) + return raw + + item = define_document_type("item", decode=decode, encode=lambda value: value) + store = SingleFileMessagePackBackingStore(path) + store.push(self._envelope(item, "a", {"name": "a"})) + stored_bytes = path.read_bytes() + cache = holder["cache"] = CultCache.builder().register_document_type(item).add_generic_store(store).build() + + with self.assertRaisesRegex(CultCacheError, "re-entrant put"): + cache.pull_all_backing_stores() + self.assertEqual(path.read_bytes(), stored_bytes) + self.assertEqual(cache.snapshot_envelopes(), []) + self.assertIsNone(cache.get(item, "side")) + # The refusal released the cache: an ordinary write still works. + cache.put(item, "b", {"name": "b"}) + self.assertEqual(cache.get(item, "b"), {"name": "b"}) + + def test_index_extractor_writing_during_register_index_raises_and_installs_nothing(self) -> None: + thing = define_document_type("thing") + cache = CultCache.builder().register_document_type(thing).build() + cache.put(thing, "k1", {"n": "one", "cat": "x"}) + + def category(value: dict) -> str: + cache.put(thing, "k2", {"n": "two", "cat": "y"}) + return value["cat"] + + with self.assertRaisesRegex(CultCacheError, "re-entrant put"): + cache.register_index(thing, "cat", category) + self.assertIsNone(cache.get(thing, "k2")) + self.assertNotIn("thing", cache._state.index_extractors) + self.assertIsNone(cache.get_key_by_index(thing, "cat", "x")) + + def test_put_envelope_refuses_a_global_under_another_key(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + settings = define_document_type("settings", global_document=True) + path = Path(tmp) / "generic.cc" + cache = CultCache.builder().register_document_type(settings).add_generic_store( + SingleFileMessagePackBackingStore(path) + ).build() + cache.put_global(settings, {"theme": "base"}) + store_before, cache_before = self._store_state(path), cache.snapshot_envelopes() + stray = replace(cache.get_required_envelope(settings, CultCache.GLOBAL_KEY), key="stray") + + with self.assertRaisesRegex(CultCacheError, "must use key __global__"): + cache.put_envelope(settings, stray) + with self.assertRaisesRegex(CultCacheError, "must be non-empty"): + cache.put_envelope(settings, replace(stray, key="")) + self.assertEqual(self._store_state(path), store_before) + self.assertEqual(cache.snapshot_envelopes(), cache_before) + + def test_refused_put_leaves_no_empty_type_entry(self) -> None: + def refuse(value: dict) -> dict: + raise ValueError("encode refuses") + + with tempfile.TemporaryDirectory() as tmp: + other = define_document_type("other", encode=refuse) + path = Path(tmp) / "generic.cc" + cache = CultCache.builder().register_document_type(other).add_generic_store( + SingleFileMessagePackBackingStore(path) + ).build() + with self.assertRaisesRegex(ValueError, "encode refuses"): + cache.put(other, "x", {"a": 1}) + self.assertEqual(cache.snapshot(), {}) + self.assertFalse(path.exists()) + + def test_attach_with_empty_type_list_is_the_generic_store(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + settings = define_database_entry_type("settings", [("theme", 0)]) + store_path = Path(tmp) / "generic.cc" + cache = CultCache.builder().register_document_type(settings).build() + cache.add_backing_store(SingleFileMessagePackBackingStore(store_path), []) + cache.put(settings, "y", {"theme": "t"}) + self.assertEqual( + [(e.key, e.type) for e in SingleFileMessagePackBackingStore(store_path).pull_all()], + [("y", "settings")], + ) + with self.assertRaisesRegex(CultCacheError, "second generic store"): + cache.add_backing_store(SingleFileMessagePackBackingStore(Path(tmp) / "b.cc"), []) + + def test_zero_store_cache_writes_and_deletes_in_memory(self) -> None: + settings = define_database_entry_type("settings", [("theme", 0)]) + cache = CultCache.builder().register_document_type(settings).build() + cache.put(settings, "x", {"theme": "mem"}) + self.assertEqual(cache.get_required(settings, "x"), {"theme": "mem"}) + cache.delete(settings, "x") + self.assertIsNone(cache.get(settings, "x")) def test_cache_put_without_backing_store_keeps_in_memory_value(self) -> None: document = define_database_entry_type( @@ -519,5893 +801,6 @@ class CultCacheTests(unittest.TestCase): self.assertEqual(cache.get(document, "item:one"), {"name": "one", "value": 1}) self.assertEqual(cache.get_required_envelope(document, "item:one").key, "item:one") - def test_verify_reports_python_runtime_surface_health(self) -> None: - result = verify(records=4) - - self.assertEqual(result["status"], "ok") - self.assertEqual( - {check["name"] for check in result["checks"]}, - { - "public_exports", - "typed_markers", - "local_cultmesh_wire_smoke", - "cultmesh_capability_truth", - "benchmark_sanity", - }, - ) - by_name = {check["name"]: check for check in result["checks"]} - self.assertEqual(by_name["public_exports"]["missing"], {}) - self.assertEqual(by_name["cultmesh_capability_truth"]["failures"], []) - - def test_cultnet_security_options_match_cultlib_defaults_and_environment_rules(self) -> None: - client = CultNetClientSecurityOptions.development() - server = CultNetServerSecurityOptions.development() - - self.assertEqual(client.connection_key, "gamecult-dev-connection-key") - self.assertEqual(server.connection_key, "gamecult-dev-connection-key") - self.assertTrue(server.is_development) - self.assertEqual(server.to_client_options().connection_key, client.connection_key) - self.assertEqual(client.encryption_key(), hashlib.sha256(b"gamecult-dev-connection-key").digest()) - self.assertEqual(server.session_signing_key(), hashlib.sha256(b"gamecult-dev-session-signing-secret").digest()) - self.assertEqual( - CultNetServerSecurityOptions.from_environment({}, allow_development_defaults=True), - server, - ) - with self.assertRaisesRegex(ValueError, "not configured"): - CultNetServerSecurityOptions.from_environment({}) - with self.assertRaisesRegex(ValueError, "GAMECULT_SESSION_SIGNING_SECRET"): - CultNetServerSecurityOptions.from_environment({"GAMECULT_CONNECTION_KEY": "key"}) - configured = CultNetServerSecurityOptions.from_environment({ - "GAMECULT_CONNECTION_KEY": "prod-key", - "GAMECULT_SESSION_SIGNING_SECRET": "prod-secret", - }) - self.assertEqual(configured.connection_key, "prod-key") - self.assertFalse(configured.is_development) - - def test_cultnet_secret_helpers_validate_csharp_versioned_session_tokens(self) -> None: - security = CultNetServerSecurityOptions("connection-key", "session-secret") - user_id = uuid4() - expires = datetime(2035, 1, 2, 3, 4, 5, tzinfo=UTC) - - token = CultNetSecret.create_session_token(user_id, expires, security, session_version=7) - payload_b64, signature_b64 = token.split(".") - self.assertEqual( - CultNetSecret.from_base64url(payload_b64).decode("utf-8"), - f"{user_id.hex}|{int(expires.timestamp())}|7", - ) - self.assertEqual( - CultNetSecret.from_base64url(signature_b64), - hmac.new( - hashlib.sha256(b"session-secret").digest(), - CultNetSecret.from_base64url(payload_b64), - hashlib.sha256, - ).digest(), - ) - - validated = CultNetSecret.try_validate_session_token( - token, - security, - now_utc=datetime(2035, 1, 2, 3, 4, 4, tzinfo=UTC), - ) - self.assertIsNotNone(validated) - assert validated is not None - self.assertEqual(validated.user_id, user_id) - self.assertEqual(validated.expires_at_utc, expires) - self.assertEqual(validated.session_version, 7) - - tampered_payload = CultNetSecret.to_base64url(CultNetSecret.from_base64url(payload_b64) + b"tamper") - tampered = f"{tampered_payload}.{signature_b64}" - self.assertIsNone(CultNetSecret.try_validate_session_token(tampered, security)) - self.assertIsNone(CultNetSecret.try_validate_session_token(token, security, now_utc=expires)) - - def test_cultnet_secret_helpers_validate_legacy_two_field_session_tokens(self) -> None: - security = CultNetServerSecurityOptions.development() - user_id = uuid4() - expires_at_seconds = int(datetime(2036, 1, 1, tzinfo=UTC).timestamp()) - payload = f"{user_id.hex}|{expires_at_seconds}".encode("utf-8") - signature = hmac.new(security.session_signing_key(), payload, hashlib.sha256).digest() - token = f"{CultNetSecret.to_base64url(payload)}.{CultNetSecret.to_base64url(signature)}" - - validated = CultNetSecret.try_validate_session_token( - token, - security, - now_utc=datetime(2035, 12, 31, tzinfo=UTC), - ) - - self.assertIsNotNone(validated) - assert validated is not None - self.assertEqual(validated.user_id, user_id) - self.assertEqual(validated.session_version, 0) - - def test_cultnet_secret_encryption_helpers_are_optional_without_crypto_dependency(self) -> None: - nonce = b"123456789012" - security = CultNetServerSecurityOptions.development() - try: - encrypted = CultNetSecret.encrypt_string("hello", nonce, security.to_client_options()) - except ImportError as exc: - self.assertIn("cryptography", str(exc)) - return - self.assertEqual(CultNetSecret.decrypt_string(encrypted, nonce, security), "hello") - - def test_cultnet_schema_message_frame_round_trip(self) -> None: - message = hello( - runtime_id="python-test", - supported_schema_versions=["cultnet.hello.v0"], - transport_profiles=[ - { - "schemaVersion": "cultnet.transport_profile.v0", - "runtimeId": "python-test", - "transports": [ - { - "transportId": "test", - "protocol": "tcp_framed", - "channels": [{"channelId": "schema", "delivery": "reliable", "ordering": "ordered"}], - } - ], - } - ], - ) - parsed = parse_message(decode_frame(encode_frame(message.to_bytes()))) - self.assertEqual(parsed.schema_version, "cultnet.hello.v0") - self.assertEqual(parsed.body["runtimeId"], "python-test") - self.assertEqual(parsed.body["transportProfiles"][0]["transports"][0]["protocol"], "tcp_framed") - - def test_cultnet_tcp_framed_transport_carries_schema_payloads_with_stats(self) -> None: - payload = b"cultnet-payload" - output = io.BytesIO() - sender = TcpFramedTransportConnection( - output, - profile=create_tcp_framed_transport_profile("sender", transport_id="test-tcp"), - ) - sender.send("schema", payload) - self.assertEqual(sender.stats.frames_sent, 1) - self.assertEqual(sender.stats.bytes_sent, len(payload) + 4) - with self.assertRaisesRegex(ValueError, "only supports the schema channel"): - sender.send("unreliable", b"") - - receiver = TcpFramedTransportConnection( - io.BytesIO(output.getvalue()), - profile=create_tcp_framed_transport_profile("receiver", transport_id="test-tcp"), - ) - frame = receiver.receive() - self.assertEqual(frame.channel_id, "schema") - self.assertEqual(frame.payload, payload) - self.assertEqual(receiver.stats.frames_received, 1) - self.assertEqual(receiver.stats.bytes_received, len(payload) + 4) - self.assertEqual(receiver.profile["transports"][0]["protocol"], "tcp_framed") - - def test_cultnet_raw_client_uses_schema_transport_factory(self) -> None: - import msgpack # type: ignore - - sent_messages: list[dict[str, object]] = [] - test_case = self - - class FakeSchemaTransport: - profile = create_tcp_framed_transport_profile("fake-client") - - def __enter__(self) -> "FakeSchemaTransport": - return self - - def __exit__(self, exc_type: object, exc: object, traceback: object) -> None: - self.close() - - def send(self, channel_id: str, payload: bytes) -> None: - test_case.assertEqual(channel_id, "schema") - sent_messages.append(msgpack.unpackb(payload, raw=False)) - - def receive(self) -> object: - response = { - "schemaVersion": "cultnet.schema_catalog_response.v0", - "messageId": "transport-factory", - "schemas": [], - } - return type("Frame", (), {"channel_id": "schema", "payload": msgpack.packb(response, use_bin_type=True)})() - - def close(self) -> None: - pass - - client = CultNetRawClient( - "unused.example.test", - 1, - create_transport=FakeSchemaTransport, - ) - - response = client.fetch_schema_catalog(message_id="transport-factory") - - self.assertEqual(response["schemaVersion"], "cultnet.schema_catalog_response.v0") - self.assertEqual(sent_messages[0]["schemaVersion"], "cultnet.schema_catalog_request.v0") - self.assertEqual(sent_messages[0]["messageId"], "transport-factory") - - def test_cultmesh_client_uses_rudp_schema_transport_endpoint(self) -> None: - import msgpack # type: ignore - - server_socket = bind_udp_socket() - server_host, server_port = server_socket.getsockname()[:2] - connection_id = 0x43554C54 - server = CultNetRudpSocketTransportConnection( - CultNetRudpSocketTransportOptions( - runtime_id="python-rudp-schema-server", - socket=server_socket, - mode=CultNetRudpSocketMode.SERVER, - connection_id=connection_id, - initial_sequence=100, - resend_delay_ms=25, - ) - ) - received_messages: list[dict[str, object]] = [] - - def serve_once() -> None: - deadline = time.monotonic() + 2.0 - try: - while time.monotonic() < deadline: - frame = server.receive_once() - server.poll_resends() - if frame is None: - time.sleep(0.005) - continue - received = msgpack.unpackb(frame.payload, raw=False) - received_messages.append(received) - response = { - "schemaVersion": "cultnet.schema_catalog_response.v0", - "messageId": received["messageId"], - "schemas": [], - } - server.send("schema", msgpack.packb(response, use_bin_type=True)) - return - raise AssertionError("RUDP schema server did not receive a request") - finally: - server.close() - - server_thread = threading.Thread(target=serve_once, daemon=True) - server_thread.start() - client = CultMesh.create_client( - endpoint=f"rudp://{server_host}:{server_port}", - timeout_seconds=2.0, - connection_id=connection_id, - runtime_id="python-rudp-schema-client", - ) - - response = client.fetch_schema_catalog(message_id="rudp-transport-factory") - server_thread.join(timeout=2.0) - - self.assertFalse(server_thread.is_alive()) - self.assertEqual(response["schemaVersion"], "cultnet.schema_catalog_response.v0") - self.assertEqual(response["messageId"], "rudp-transport-factory") - self.assertEqual(received_messages[0]["schemaVersion"], "cultnet.schema_catalog_request.v0") - self.assertEqual(received_messages[0]["messageId"], "rudp-transport-factory") - - def test_cultnet_rudp_packet_codec_uses_deterministic_reliable_ordered_fixture(self) -> None: - encoded = encode_rudp_packet( - CultNetRudpPacket( - packet_type=CultNetRudpPacketType.DATA, - connection_id=0x01020304, - sequence=0x0000002A, - ack=0x00000029, - ack_mask=0x80000001, - channel_id="schema", - reliable=True, - ordered=True, - fragment_id=7, - fragment_index=1, - fragment_count=3, - payload=b"hello", - ) - ) - - self.assertEqual( - encoded.hex(), - "434e523000030b2a010203040000002a0000002980000001000700010003000000050600736368656d6168656c6c6f", - ) - - decoded = decode_rudp_packet(encoded) - self.assertEqual(decoded.packet_type, CultNetRudpPacketType.DATA) - self.assertEqual(decoded.connection_id, 0x01020304) - self.assertEqual(decoded.sequence, 0x0000002A) - self.assertEqual(decoded.ack, 0x00000029) - self.assertEqual(decoded.ack_mask, 0x80000001) - self.assertEqual(decoded.channel_id, "schema") - self.assertTrue(decoded.reliable) - self.assertTrue(decoded.ordered) - self.assertFalse(decoded.sequenced) - self.assertEqual(decoded.fragment_id, 7) - self.assertEqual(decoded.fragment_index, 1) - self.assertEqual(decoded.fragment_count, 3) - self.assertEqual(decoded.payload, b"hello") - - def test_cultnet_rudp_transport_profile_advertises_state_and_realtime_channels(self) -> None: - profile = create_rudp_transport_profile( - "python-rudp", - transport_id="public-rudp", - host="127.0.0.1", - port=7777, - max_payload_bytes=1200, - max_fragment_bytes=1000, - ) - - self.assertEqual(profile["transports"][0]["protocol"], "rudp") - self.assertEqual(profile["transports"][0]["reconnectPolicy"]["schemaVersion"], "cultnet.reconnect_policy.v0") - self.assertEqual(profile["transports"][0]["reconnectPolicy"]["baseDelayMs"], 1_000) - self.assertEqual( - [ - (channel["channelId"], channel["delivery"], channel["ordering"]) - for channel in profile["transports"][0]["channels"] - ], - [ - ("schema", "reliable", "ordered"), - ("latest", "unreliable", "sequenced"), - ("realtime", "unreliable", "unordered"), - ], - ) - - def test_cultnet_reconnect_policy_exposes_portable_delay_contract(self) -> None: - policy = create_reconnect_policy(policy_id="rudp-default", max_attempts=8) - - self.assertEqual(policy.schema_version, "cultnet.reconnect_policy.v0") - self.assertEqual(policy.policy_id, "rudp-default") - self.assertEqual(policy.max_attempts, 8) - self.assertEqual(policy.to_wire()["policyId"], "rudp-default") - self.assertEqual(policy.to_wire()["maxAttempts"], 8) - self.assertEqual(compute_reconnect_delay_ms(policy, 1), 1_000) - self.assertEqual(compute_reconnect_delay_ms(policy, 3, 17), 4_017) - self.assertEqual(compute_reconnect_delay_ms(policy, 9, 999), 30_250) - self.assertEqual(compute_reconnect_delay_ms(policy, 0, -5), 1_000) - - def test_cultnet_reconnect_controller_schedules_attempts_and_reset(self) -> None: - controller = CultNetReconnectController(create_reconnect_policy(max_attempts=2)) - - first = controller.record_failure(10_000) - self.assertEqual(first.attempt, 1) - self.assertTrue(first.should_retry) - self.assertEqual(first.delay_ms, 1_000) - self.assertEqual(first.next_attempt_at_ms, 11_000) - self.assertFalse(first.exhausted) - self.assertFalse(controller.can_attempt(10_999)) - self.assertTrue(controller.can_attempt(11_000)) - - second = controller.record_failure(11_000, 17) - self.assertEqual(second.attempt, 2) - self.assertEqual(second.delay_ms, 2_017) - self.assertEqual(second.next_attempt_at_ms, 13_017) - self.assertTrue(second.should_retry) - - exhausted = controller.record_failure(13_017) - self.assertEqual(exhausted.attempt, 2) - self.assertFalse(exhausted.should_retry) - self.assertEqual(exhausted.delay_ms, 0) - self.assertIsNone(exhausted.next_attempt_at_ms) - self.assertTrue(exhausted.exhausted) - self.assertFalse(controller.can_attempt(99_000)) - - controller.reset() - self.assertEqual(controller.attempt, 0) - self.assertIsNone(controller.next_attempt_at_ms) - self.assertFalse(controller.exhausted) - self.assertTrue(controller.can_attempt(99_000)) - - def test_cultnet_rudp_reconnect_loop_consumes_shared_controller(self) -> None: - class FakeRudpReconnectTransport: - def __init__(self) -> None: - self.connect_calls = 0 - self.close_calls = 0 - self.last_payload: bytes | None = None - - def connect(self, payload: bytes = b"") -> None: - self.connect_calls += 1 - self.last_payload = bytes(payload) - - def close(self) -> None: - self.close_calls += 1 - - now_ms = 10_000 - captured_callback: list[Callable[[], None]] = [] - transports: list[FakeRudpReconnectTransport] = [] - - def create_transport() -> FakeRudpReconnectTransport: - transport = FakeRudpReconnectTransport() - transports.append(transport) - return transport - - def scheduler(delay_ms: int, callback: Callable[[], None]) -> Callable[[], None]: - self.assertEqual(delay_ms, 1_017) - captured_callback[:] = [callback] - - def cancel() -> None: - captured_callback.clear() - - return cancel - - loop = CultNetRudpReconnectLoop( - create_transport, - reconnect_policy=create_reconnect_policy(max_attempts=2), - connect_payload=b"join", - now_ms=lambda: now_ms, - jitter_ms=lambda: 17, - scheduler=scheduler, - ) - - first = loop.start() - self.assertEqual(first.connect_calls, 1) - self.assertEqual(first.last_payload, b"join") - self.assertIs(loop.transport, first) - - decision = loop.handle_closed() - self.assertIsNotNone(decision) - self.assertEqual(decision.attempt, 1) - self.assertTrue(decision.should_retry) - self.assertEqual(decision.delay_ms, 1_017) - self.assertEqual(decision.next_attempt_at_ms, 11_017) - self.assertEqual(loop.reconnect_controller.attempt, 1) - self.assertEqual(loop.reconnect_controller.next_attempt_at_ms, 11_017) - self.assertEqual(len(captured_callback), 1) - - now_ms = 11_017 - captured_callback[0]() - self.assertEqual(len(transports), 2) - self.assertEqual(transports[1].connect_calls, 1) - self.assertIs(loop.transport, transports[1]) - - loop.mark_connected() - self.assertEqual(loop.reconnect_controller.attempt, 0) - - loop.stop() - self.assertEqual(transports[1].close_calls, 1) - self.assertIsNone(loop.transport) - - def test_cultnet_rudp_session_handshake_acks_reliable_connect_and_accept_packets(self) -> None: - client = CultNetRudpSession( - CultNetRudpSessionOptions(connection_id=0x0A0B0C0D, initial_sequence=1, resend_delay_ms=50) - ) - server = CultNetRudpSession( - CultNetRudpSessionOptions(connection_id=0x0A0B0C0D, initial_sequence=100, resend_delay_ms=50) - ) - - connect = client.create_connect(0, b"join") - self.assertEqual(connect.packet_type, CultNetRudpPacketType.CONNECT) - self.assertEqual(connect.sequence, 1) - self.assertEqual(client.pending_reliable_sequences, (1,)) - - accept = server.accept_connect(connect, 10, b"ok") - self.assertEqual(accept.packet_type, CultNetRudpPacketType.ACCEPT) - self.assertEqual(accept.ack, 1) - self.assertTrue(server.connected) - self.assertEqual(server.pending_reliable_sequences, (100,)) - - client.receive(accept, 20) - self.assertTrue(client.connected) - self.assertEqual(client.pending_reliable_sequences, ()) - - ack = client.create_ack() - self.assertEqual(ack.ack, 100) - server.receive(ack, 30) - self.assertEqual(server.pending_reliable_sequences, ()) - - def test_cultnet_rudp_session_computes_ack_masks_and_clears_pending_reliable_packets(self) -> None: - sender = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=7, initial_sequence=10, resend_delay_ms=100)) - receiver = CultNetRudpSession( - CultNetRudpSessionOptions(connection_id=7, initial_sequence=200, resend_delay_ms=100) - ) - sender.receive( - CultNetRudpPacket(CultNetRudpPacketType.ACCEPT, 7, 1, 0, 0, "control") - ) - receiver.receive( - CultNetRudpPacket(CultNetRudpPacketType.ACCEPT, 7, 2, 0, 0, "control") - ) - - first = sender.send( - "schema", - b"first", - CultNetRudpSendOptions(reliable=True, ordered=True, now_ms=0), - ) - second = sender.send( - "schema", - b"second", - CultNetRudpSendOptions(reliable=True, ordered=True, now_ms=0), - ) - third = sender.send( - "schema", - b"third", - CultNetRudpSendOptions(reliable=True, ordered=True, now_ms=0), - ) - self.assertEqual(sender.pending_reliable_sequences, (10, 11, 12)) - - receiver.receive(first) - receiver.receive(third) - ack_with_gap = receiver.create_ack() - self.assertEqual(ack_with_gap.ack, 12) - self.assertEqual(ack_with_gap.ack_mask, 0b10 | (1 << 9)) - sender.receive(ack_with_gap) - self.assertEqual(sender.pending_reliable_sequences, (11,)) - - receiver.receive(second) - full_ack = receiver.create_ack() - self.assertEqual(full_ack.ack, 12) - self.assertEqual(full_ack.ack_mask, 0b11 | (1 << 9)) - sender.receive(full_ack) - self.assertEqual(sender.pending_reliable_sequences, ()) - - def test_cultnet_rudp_session_schedules_reliable_resends_until_acked(self) -> None: - session = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=99, initial_sequence=1, resend_delay_ms=100)) - session.receive( - CultNetRudpPacket(CultNetRudpPacketType.ACCEPT, 99, 50, 0, 0, "control") - ) - sent = session.send( - "schema", - b"payload", - CultNetRudpSendOptions(reliable=True, ordered=True, now_ms=10), - ) - - self.assertEqual(session.due_resends(90), ()) - self.assertEqual(tuple(packet.sequence for packet in session.due_resends(110)), (sent.sequence,)) - self.assertEqual(session.due_resends(150), ()) - - session.receive( - CultNetRudpPacket(CultNetRudpPacketType.ACK, 99, 51, sent.sequence, 0, "control") - ) - self.assertEqual(session.due_resends(250), ()) - - def test_cultnet_rudp_session_pings_and_detects_receive_timeout(self) -> None: - client = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=101, initial_sequence=1)) - server = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=101, initial_sequence=100)) - connect = client.create_connect(0, b"join") - accept = server.accept_connect(connect, 10) - client.receive(accept, 20) - - ping = client.create_ping(b"pulse") - ping_result = server.receive(ping, 30) - self.assertIsNotNone(ping_result.reply) - self.assertEqual(ping_result.reply.packet_type, CultNetRudpPacketType.PONG) - self.assertEqual(ping_result.reply.payload, b"pulse") - - pong_result = client.receive(ping_result.reply, 40) - self.assertTrue(pong_result.pong) - self.assertEqual(pong_result.pong_payload, b"pulse") - self.assertFalse(client.check_timeout(90, 50)) - self.assertTrue(client.check_timeout(91, 50)) - self.assertFalse(client.connected) - - def test_cultnet_rudp_session_bounds_pending_reliable_packets_before_enqueue(self) -> None: - session = CultNetRudpSession( - CultNetRudpSessionOptions(connection_id=102, initial_sequence=1, max_pending_reliable_packets=2) - ) - session.receive(CultNetRudpPacket(CultNetRudpPacketType.ACCEPT, 102, 50, 0, 0, "control")) - session.send("schema", b"first", CultNetRudpSendOptions(reliable=True, ordered=True)) - session.send("schema", b"second", CultNetRudpSendOptions(reliable=True, ordered=True)) - with self.assertRaisesRegex(ValueError, "reliable send queue is full"): - session.send("schema", b"third", CultNetRudpSendOptions(reliable=True, ordered=True)) - self.assertEqual(session.pending_reliable_sequences, (1, 2)) - - fragmented = CultNetRudpSession( - CultNetRudpSessionOptions(connection_id=103, initial_sequence=1, max_pending_reliable_packets=3) - ) - fragmented.receive(CultNetRudpPacket(CultNetRudpPacketType.ACCEPT, 103, 50, 0, 0, "control")) - with self.assertRaisesRegex(ValueError, "reliable send queue is full"): - fragmented.send_many( - "schema", - b"fragment-me", - CultNetRudpSendOptions(reliable=True, ordered=True), - max_fragment_bytes=3, - ) - self.assertEqual(fragmented.pending_reliable_sequences, ()) - - def test_cultnet_rudp_session_suppresses_duplicates_and_delivers_reliable_ordered_payloads(self) -> None: - sender = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=123, initial_sequence=1)) - receiver = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=123, initial_sequence=100)) - sender.receive( - CultNetRudpPacket(CultNetRudpPacketType.ACCEPT, 123, 90, 0, 0, "control") - ) - receiver.receive( - CultNetRudpPacket(CultNetRudpPacketType.ACCEPT, 123, 91, 0, 0, "control") - ) - - first = sender.send("schema", b"first", CultNetRudpSendOptions(reliable=True, ordered=True)) - second = sender.send("schema", b"second", CultNetRudpSendOptions(reliable=True, ordered=True)) - third = sender.send("schema", b"third", CultNetRudpSendOptions(reliable=True, ordered=True)) - - self.assertEqual([frame.payload.decode("utf-8") for frame in receiver.receive(first).delivered], ["first"]) - self.assertEqual(receiver.receive(third).delivered, ()) - self.assertEqual(receiver.receive(first).delivered, ()) - self.assertEqual( - [frame.payload.decode("utf-8") for frame in receiver.receive(second).delivered], - ["second", "third"], - ) - - def test_cultnet_rudp_session_skips_control_packets_while_ordering_schema_payloads(self) -> None: - sender = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=124, initial_sequence=1)) - receiver = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=124, initial_sequence=100)) - sender.receive( - CultNetRudpPacket(CultNetRudpPacketType.ACCEPT, 124, 90, 0, 0, "control") - ) - receiver.receive( - CultNetRudpPacket(CultNetRudpPacketType.ACCEPT, 124, 91, 0, 0, "control") - ) - - first = sender.send("schema", b"first", CultNetRudpSendOptions(reliable=True, ordered=True)) - control = sender.create_ack() - second = sender.send("schema", b"second", CultNetRudpSendOptions(reliable=True, ordered=True)) - - self.assertEqual([frame.payload.decode("utf-8") for frame in receiver.receive(first).delivered], ["first"]) - self.assertEqual(receiver.receive(control).delivered, ()) - self.assertEqual([frame.payload.decode("utf-8") for frame in receiver.receive(second).delivered], ["second"]) - - def test_cultnet_rudp_session_fragments_and_reassembles_reliable_ordered_payloads(self) -> None: - sender = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=456, initial_sequence=1)) - receiver = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=456, initial_sequence=100)) - sender.receive( - CultNetRudpPacket(CultNetRudpPacketType.ACCEPT, 456, 90, 0, 0, "control") - ) - receiver.receive( - CultNetRudpPacket(CultNetRudpPacketType.ACCEPT, 456, 91, 0, 0, "control") - ) - - packets = sender.send_many( - "schema", - b"fragment-me-please", - CultNetRudpSendOptions(reliable=True, ordered=True, now_ms=10), - max_fragment_bytes=5, - ) - self.assertEqual(len(packets), 4) - self.assertEqual(tuple(packet.fragment_count for packet in packets), (4, 4, 4, 4)) - self.assertEqual(tuple(packet.fragment_index for packet in packets), (0, 1, 2, 3)) - self.assertTrue(all(packet.fragment_id == packets[0].fragment_id for packet in packets)) - - self.assertEqual(receiver.receive(packets[0]).delivered, ()) - self.assertEqual(receiver.receive(packets[1]).delivered, ()) - self.assertEqual(receiver.receive(packets[2]).delivered, ()) - delivered = receiver.receive(packets[3]).delivered - self.assertEqual(len(delivered), 1) - self.assertEqual(delivered[0].payload, b"fragment-me-please") - self.assertEqual(delivered[0].sequence, packets[0].sequence) - - def test_cultnet_rudp_socket_transport_handshakes_and_carries_reliable_ordered_schema_frames(self) -> None: - server_socket = bind_udp_socket() - client_socket = bind_udp_socket() - connection_id = 0x10203040 - server = CultNetRudpSocketTransportConnection( - CultNetRudpSocketTransportOptions( - runtime_id="python-rudp-server", - socket=server_socket, - mode=CultNetRudpSocketMode.SERVER, - connection_id=connection_id, - initial_sequence=100, - resend_delay_ms=25, - ) - ) - client = CultNetRudpSocketTransportConnection( - CultNetRudpSocketTransportOptions( - runtime_id="python-rudp-client", - socket=client_socket, - mode=CultNetRudpSocketMode.CLIENT, - remote_addr=server_socket.getsockname(), - connection_id=connection_id, - initial_sequence=1, - resend_delay_ms=25, - ) - ) - - try: - client.connect(b"join") - pump_rudp_handshake(client, server) - self.assertTrue(client.connected) - self.assertTrue(server.connected) - - client.send("schema", b"client-state") - server_frame = receive_rudp_frame(server) - self.assertEqual(server_frame.channel_id, "schema") - self.assertEqual(server_frame.payload, b"client-state") - - server.send("schema", b"server-state") - client_frame = receive_rudp_frame(client) - self.assertEqual(client_frame.channel_id, "schema") - self.assertEqual(client_frame.payload, b"server-state") - self.assertEqual(server.profile["transports"][0]["protocol"], "rudp") - self.assertEqual(client.stats.frames_sent, 1) - self.assertEqual(server.stats.frames_received, 1) - finally: - client.close() - server.close() - - def test_cultnet_rudp_socket_transport_carries_fragmented_reliable_ordered_schema_frames(self) -> None: - server_socket = bind_udp_socket() - client_socket = bind_udp_socket() - connection_id = 0x10203041 - server = CultNetRudpSocketTransportConnection( - CultNetRudpSocketTransportOptions( - runtime_id="python-rudp-fragment-server", - socket=server_socket, - mode=CultNetRudpSocketMode.SERVER, - connection_id=connection_id, - initial_sequence=100, - resend_delay_ms=25, - max_fragment_bytes=8, - ) - ) - client = CultNetRudpSocketTransportConnection( - CultNetRudpSocketTransportOptions( - runtime_id="python-rudp-fragment-client", - socket=client_socket, - mode=CultNetRudpSocketMode.CLIENT, - remote_addr=server_socket.getsockname(), - connection_id=connection_id, - initial_sequence=1, - resend_delay_ms=25, - max_fragment_bytes=8, - ) - ) - - try: - payload = b"this-schema-frame-is-larger-than-one-rudp-fragment" - client.connect(b"join") - pump_rudp_handshake(client, server) - client.send("schema", payload) - server_frame = receive_rudp_frame(server) - self.assertEqual(server_frame.channel_id, "schema") - self.assertEqual(server_frame.payload, payload) - self.assertEqual(client.stats.frames_sent, 1) - self.assertEqual(server.stats.frames_received, 1) - finally: - client.close() - server.close() - - def test_cultmesh_facade_creates_rudp_client_from_peer_endpoint(self) -> None: - connection_id = 0x10203042 - server = CultMesh.create_rudp_server( - "python-cultmesh-rudp-server", - connection_id, - initial_sequence=100, - resend_delay_ms=25, - max_fragment_bytes=1024, - max_pending_reliable_packets=16, - ) - server_host, server_port = server.socket.getsockname()[:2] - endpoint = CultMesh.parse_rudp_endpoint(f"rudp://{server_host}:{server_port}") - self.assertEqual(endpoint.host, server_host) - self.assertEqual(endpoint.port, server_port) - peer = CultMeshPeerCard( - peer_id="python-cultmesh-rudp-server", - verse_id="local", - endpoints=(endpoint.uri,), - roles=("schema",), - authority_lease_id="lease:python-cultmesh-rudp-server", - ) - peers = CultMesh.create_peer_catalog() - leases = CultMesh.create_authority_lease_catalog() - peers.upsert(peer) - with self.assertRaisesRegex(ValueError, "No authorized RUDP peer"): - CultMesh.create_rudp_client_for_authorized_peer( - "python-cultmesh-rudp-client", - connection_id, - peers, - leases, - "local", - "schema", - resend_delay_ms=25, - max_fragment_bytes=1024, - max_pending_reliable_packets=16, - ) - leases.upsert( - CultMeshAuthorityLease( - lease_id="lease:python-cultmesh-rudp-server", - verse_id="local", - peer_id="python-cultmesh-rudp-server", - roles=("schema",), - valid_from=datetime.now(UTC) - timedelta(seconds=1), - expires_at=datetime.now(UTC) + timedelta(seconds=30), - issuer_runtime_id="python-authority", - ) - ) - client = CultMesh.create_rudp_client_for_authorized_peer( - "python-cultmesh-rudp-client", - connection_id, - peers, - leases, - "local", - "schema", - resend_delay_ms=25, - max_fragment_bytes=1024, - max_pending_reliable_packets=16, - ) - - try: - client.connect(b"join") - pump_rudp_handshake(client, server) - self.assertTrue(client.connected) - self.assertTrue(server.connected) - - client.send("schema", b"client-state") - server_frame = receive_rudp_frame(server) - self.assertEqual(server_frame.channel_id, "schema") - self.assertEqual(server_frame.payload, b"client-state") - self.assertEqual(server.profile["transports"][0]["protocol"], "rudp") - self.assertEqual(client.profile["transports"][0]["protocol"], "rudp") - finally: - client.close() - server.close() - - def test_cultnet_database_subscription_helpers_match_schema_v0_shape(self) -> None: - subscribe = database_subscribe( - message_id="sub-message", - subscription_id="sub-1", - schema_ids=["schema-a"], - record_keys=["record-a"], - include_snapshot=False, - ).to_wire() - self.assertEqual(subscribe["schemaVersion"], "cultnet.database_subscribe.v0") - self.assertEqual(subscribe["messageId"], "sub-message") - self.assertEqual(subscribe["subscriptionId"], "sub-1") - self.assertEqual(subscribe["schemaIds"], ["schema-a"]) - self.assertEqual(subscribe["recordKeys"], ["record-a"]) - self.assertFalse(subscribe["includeSnapshot"]) - - unsubscribe = database_unsubscribe(message_id="unsub-message", subscription_id="sub-1").to_wire() - self.assertEqual(unsubscribe["schemaVersion"], "cultnet.database_unsubscribe.v0") - self.assertEqual(unsubscribe["messageId"], "unsub-message") - self.assertEqual(unsubscribe["subscriptionId"], "sub-1") - - def test_cultnet_database_change_parses_put_and_delete_shapes(self) -> None: - put_change = CultNetDatabaseChange.from_wire({ - "schemaVersion": "cultnet.database_change_raw.v0", - "messageId": "change-put", - "subscriptionId": "sub-1", - "changeKind": "added", - "document": { - "schemaId": "schema-note", - "recordKey": "note:1", - "payload": b"payload", - }, - }) - self.assertEqual(put_change.schema_id, "schema-note") - self.assertEqual(put_change.record_key, "note:1") - self.assertIsInstance(put_change.raw_document, CultNetRawDocumentRecord) - self.assertEqual(put_change.raw_document.payload, b"payload") - self.assertEqual(put_change.to_wire()["document"]["recordKey"], "note:1") - - delete_change = CultNetDatabaseChange.from_wire({ - "schemaVersion": "cultnet.database_change_raw.v0", - "messageId": "change-delete", - "subscriptionId": "sub-1", - "changeKind": "removed", - "schemaId": "schema-note", - "recordKey": "note:1", - }) - self.assertIsNone(delete_change.document) - self.assertIsNone(delete_change.raw_document) - self.assertEqual(delete_change.schema_id, "schema-note") - self.assertEqual(delete_change.to_wire()["recordKey"], "note:1") - - def test_cultnet_raw_put_helper_carries_message_id(self) -> None: - put = document_put_raw( - message_id="put-1", - key="record-a", - schema_id="schema-a", - stored_at="2026-06-13T00:00:00Z", - payload=b"payload", - source_runtime_id="python-test", - shard_id="interop", - shard_epoch=1, - ).to_wire() - self.assertEqual(put["schemaVersion"], "cultnet.document_put_raw.v0") - self.assertEqual(put["messageId"], "put-1") - self.assertEqual(put["document"]["recordKey"], "record-a") - self.assertEqual(put["document"]["sourceRuntimeId"], "python-test") - self.assertEqual(put["shardId"], "interop") - self.assertEqual(put["shardEpoch"], 1) - - def test_cultnet_catalog_and_snapshot_helpers_accept_filters(self) -> None: - catalog = schema_catalog_request( - message_id="catalog-1", - include_schema_json=True, - schema_ids=["schema-a"], - kinds=["wire_message"], - ).to_wire() - self.assertEqual(catalog["schemaVersion"], "cultnet.schema_catalog_request.v0") - self.assertEqual(catalog["messageId"], "catalog-1") - self.assertTrue(catalog["includeSchemaJson"]) - self.assertEqual(catalog["schemaIds"], ["schema-a"]) - self.assertEqual(catalog["kinds"], ["wire_message"]) - - snapshot = snapshot_request( - message_id="snapshot-1", - schema_ids=["schema-a"], - record_keys=["record-a"], - shard_id="interop", - shard_epoch=1, - ).to_wire() - self.assertEqual(snapshot["schemaVersion"], "cultnet.snapshot_request.v0") - self.assertEqual(snapshot["messageId"], "snapshot-1") - self.assertEqual(snapshot["schemaIds"], ["schema-a"]) - self.assertEqual(snapshot["recordKeys"], ["record-a"]) - self.assertEqual(snapshot["shardId"], "interop") - self.assertEqual(snapshot["shardEpoch"], 1) - - def test_cultnet_raw_snapshot_response_filters_and_roundtrips_records(self) -> None: - response = CultNetRawSnapshotResponse.from_wire({ - "schemaVersion": "cultnet.snapshot_response_raw.v0", - "messageId": "snapshot-1", - "shardId": "notes", - "shardEpoch": 3, - "shardLogSequence": 8, - "documents": [ - CultNetRawDocumentRecord( - schema_id="schema-note", - record_key="note:1", - stored_at="2026-06-14T00:00:00Z", - payload=b"note", - source_runtime_id="python-test", - tags=("snapshot",), - ).to_wire(), - { - "schemaId": "schema-fact", - "recordKey": "fact:1", - "payloadEncoding": "messagepack", - "payload": b"fact", - }, - ], - }) - self.assertEqual(response.shard_id, "notes") - self.assertEqual(response.shard_epoch, 3) - self.assertEqual(response.shard_log_sequence, 8) - self.assertEqual(response.documents[0].source_runtime_id, "python-test") - self.assertEqual(response.filter(schema_ids=["schema-note"])[0].record_key, "note:1") - self.assertEqual(response.filter(record_keys=["fact:1"])[0].schema_id, "schema-fact") - self.assertEqual(response.to_wire()["documents"][0]["tags"], ["snapshot"]) - - def test_cultnet_raw_client_fetches_schema_snapshot_and_shard_reads(self) -> None: - import msgpack # type: ignore - from cultnet_py import read_frame, write_frame - - received_versions: list[str] = [] - ready = threading.Event() - server_error: list[BaseException] = [] - port_holder: list[int] = [] - - def serve_requests() -> None: - try: - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as server: - server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - server.bind(("127.0.0.1", 0)) - port_holder.append(server.getsockname()[1]) - server.listen(4) - ready.set() - for _ in range(4): - connection, _ = server.accept() - with connection: - stream = connection.makefile("rwb") - request = msgpack.unpackb(read_frame(stream), raw=False) - received_versions.append(request["schemaVersion"]) - if request["schemaVersion"] == "cultnet.schema_catalog_request.v0": - self.assertEqual(request["kinds"], ["wire_message"]) - response = {"schemaVersion": "cultnet.schema_catalog_response.v0", "messageId": request["messageId"], "schemas": []} - elif request["schemaVersion"] == "cultnet.snapshot_request.v0": - self.assertEqual(request["shardId"], "interop") - response = {"schemaVersion": "cultnet.snapshot_response_raw.v0", "messageId": request["messageId"], "documents": []} - elif request["schemaVersion"] == "cultnet.shard_catalog_request.v0": - response = { - "schemaVersion": "cultnet.shard_catalog_response.v0", - "messageId": request["messageId"], - "shards": [{ - "shardId": "interop", - "ownerRuntimeId": "socket-test", - "epoch": 1, - "schemaIds": request["schemaIds"], - "keyPrefix": "record-", - "primaryEndpoints": ["cultnet://127.0.0.1:1"], - }], - } - elif request["schemaVersion"] == "cultnet.shard_log_request.v0": - self.assertEqual(request["afterSequence"], 7) - response = { - "schemaVersion": "cultnet.shard_log_response.v0", - "messageId": request["messageId"], - "shardId": request["shardId"], - "shardEpoch": request["shardEpoch"], - "entries": [], - "resyncRequired": False, - } - else: - raise AssertionError(f"unexpected request {request['schemaVersion']}") - write_frame(stream, msgpack.packb(response, use_bin_type=True)) - stream.flush() - except BaseException as error: - server_error.append(error) - ready.set() - - thread = threading.Thread(target=serve_requests, daemon=True) - thread.start() - self.assertTrue(ready.wait(2.0)) - self.assertFalse(server_error) - - client = CultNetRawClient("127.0.0.1", port_holder[0], timeout_seconds=2.0) - self.assertEqual(client.fetch_schema_catalog(kinds=["wire_message"])["schemaVersion"], "cultnet.schema_catalog_response.v0") - typed_snapshot = client.fetch_snapshot_response(shard_id="interop", shard_epoch=1) - self.assertEqual(typed_snapshot.shard_id, None) - self.assertEqual(typed_snapshot.documents, ()) - shard_catalog = CultNetShardCatalog() - synced_shards = client.sync_shard_catalog(shard_catalog, schema_ids=["schema-a"]) - self.assertEqual(synced_shards[0].shard_id, "interop") - self.assertEqual(shard_catalog.get("interop"), synced_shards[0]) - shard_log = client.fetch_shard_log_response(shard_id="interop", shard_epoch=1, after_sequence=7) - self.assertEqual(shard_log.shard_id, "interop") - self.assertFalse(shard_log.resync_required) - self.assertEqual(shard_log.last_sequence, 0) - - thread.join(2.0) - self.assertFalse(server_error) - self.assertEqual(received_versions, [ - "cultnet.schema_catalog_request.v0", - "cultnet.snapshot_request.v0", - "cultnet.shard_catalog_request.v0", - "cultnet.shard_log_request.v0", - ]) - - def test_cultnet_raw_client_raises_typed_peer_error_response(self) -> None: - import msgpack # type: ignore - from cultnet_py import read_frame, write_frame - - ready = threading.Event() - port_holder: list[int] = [] - - def serve_error() -> None: - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as server: - server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - server.bind(("127.0.0.1", 0)) - port_holder.append(server.getsockname()[1]) - server.listen(1) - ready.set() - connection, _ = server.accept() - with connection: - stream = connection.makefile("rwb") - read_frame(stream) - write_frame(stream, msgpack.packb({ - "schemaVersion": "cultnet.error.v0", - "messageId": "error-1", - "error": "Snapshot document limit exceeded", - }, use_bin_type=True)) - stream.flush() - - thread = threading.Thread(target=serve_error, daemon=True) - thread.start() - self.assertTrue(ready.wait(2.0)) - - client = CultNetRawClient("127.0.0.1", port_holder[0], timeout_seconds=2.0) - with self.assertRaisesRegex(CultNetPeerError, "Snapshot document limit exceeded") as raised: - client.fetch_snapshot() - - self.assertEqual(raised.exception.peer_error, "Snapshot document limit exceeded") - self.assertEqual(raised.exception.response["schemaVersion"], "cultnet.error.v0") - thread.join(2.0) - - def test_cultnet_database_subscription_reads_snapshot_and_change(self) -> None: - import msgpack # type: ignore - from cultnet_py import read_frame, write_frame - - document = define_database_entry_type( - "sub.item", - [ - ("name", 0), - ("category", 1), - ("value", 2, 0), - ], - cls=Item, - ) - schema_id = document.catalog_entry().schema_id - ready = threading.Event() - server_error: list[BaseException] = [] - port_holder: list[int] = [] - received_versions: list[str] = [] - - def serve_subscription() -> None: - try: - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as server: - server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - server.bind(("127.0.0.1", 0)) - port_holder.append(server.getsockname()[1]) - server.listen(1) - ready.set() - connection, _ = server.accept() - with connection: - stream = connection.makefile("rwb") - subscribe = msgpack.unpackb(read_frame(stream), raw=False) - received_versions.append(subscribe["schemaVersion"]) - self.assertEqual(subscribe["subscriptionId"], "sub-1") - snapshot = { - "schemaVersion": "cultnet.snapshot_response_raw.v0", - "messageId": subscribe["messageId"], - "documents": [], - } - write_frame(stream, msgpack.packb(snapshot, use_bin_type=True)) - stream.flush() - - put = msgpack.unpackb(read_frame(stream), raw=False) - received_versions.append(put["schemaVersion"]) - change = { - "schemaVersion": "cultnet.database_change_raw.v0", - "messageId": "change-1", - "subscriptionId": "sub-1", - "changeKind": "added", - "document": put["document"], - } - write_frame(stream, msgpack.packb(change, use_bin_type=True)) - stream.flush() - - unsubscribe = msgpack.unpackb(read_frame(stream), raw=False) - received_versions.append(unsubscribe["schemaVersion"]) - except BaseException as error: - server_error.append(error) - ready.set() - - thread = threading.Thread(target=serve_subscription, daemon=True) - thread.start() - self.assertTrue(ready.wait(2.0)) - self.assertFalse(server_error) - - client = CultNetRawClient("127.0.0.1", port_holder[0], timeout_seconds=2.0) - put = document_put_raw( - message_id="put-sub", - key="item:sub", - schema_id=schema_id, - stored_at="2026-06-14T00:00:00Z", - payload=document.encode_payload(Item("orb", "gear", 8)), - ) - with client.subscribe_database(subscription_id="sub-1", schema_ids=[schema_id]) as subscription: - snapshot = subscription.read_next_snapshot_response() - subscription.send(put) - change = subscription.read_next_change() - - thread.join(2.0) - self.assertFalse(server_error) - self.assertIsInstance(snapshot, CultNetRawSnapshotResponse) - self.assertEqual(snapshot.message_id, "cultnet-python-subscribe") - self.assertEqual(snapshot.to_wire()["schemaVersion"], "cultnet.snapshot_response_raw.v0") - self.assertEqual(change.change_kind, "added") - self.assertEqual(change.record_key, "item:sub") - self.assertIsInstance(change.raw_document, CultNetRawDocumentRecord) - self.assertEqual(change.raw_document.record_key, "item:sub") - self.assertEqual(received_versions, [ - "cultnet.database_subscribe.v0", - "cultnet.document_put_raw.v0", - "cultnet.database_unsubscribe.v0", - ]) - - def test_cultnet_database_subscription_raises_typed_peer_error(self) -> None: - import msgpack # type: ignore - from cultnet_py import read_frame, write_frame - - ready = threading.Event() - port_holder: list[int] = [] - - def serve_subscription_error() -> None: - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as server: - server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - server.bind(("127.0.0.1", 0)) - port_holder.append(server.getsockname()[1]) - server.listen(1) - ready.set() - connection, _ = server.accept() - with connection: - stream = connection.makefile("rwb") - read_frame(stream) - write_frame(stream, msgpack.packb({ - "schemaVersion": "cultnet.error.v0", - "messageId": "subscription-error", - "error": "subscription rejected", - }, use_bin_type=True)) - stream.flush() - - thread = threading.Thread(target=serve_subscription_error, daemon=True) - thread.start() - self.assertTrue(ready.wait(2.0)) - - client = CultNetRawClient("127.0.0.1", port_holder[0], timeout_seconds=2.0) - with client.subscribe_database(subscription_id="sub-error") as subscription: - with self.assertRaisesRegex(CultNetPeerError, "subscription rejected") as raised: - subscription.read_next() - self.assertEqual(raised.exception.response["messageId"], "subscription-error") - thread.join(2.0) - - def test_cultnet_replication_helpers_apply_raw_snapshot_and_shard_log(self) -> None: - document = define_database_entry_type( - "replica.item", - [ - ("name", 0), - ("category", 1), - ("value", 2, 0), - ], - cls=Item, - ) - cache = CultCache() - cache.register_document_type(document) - schema_id = document.catalog_entry().schema_id - - snapshot = { - "schemaVersion": "cultnet.snapshot_response_raw.v0", - "messageId": "snapshot-1", - "documents": [ - { - "schemaId": schema_id, - "recordKey": "item:1", - "storedAt": "2026-06-13T00:00:00Z", - "payloadEncoding": "messagepack", - "payload": document.encode_payload(Item("sword", "gear", 3)), - } - ], - } - applied_snapshot = apply_raw_snapshot(cache, [document], snapshot) - self.assertEqual(applied_snapshot[0].record_key, "item:1") - self.assertEqual(cache.get_required(document, "item:1").value, 3) - typed_snapshot = CultNetRawSnapshotResponse.from_wire(snapshot) - direct_applied = apply_raw_document_record(cache, schema_document_map([document]), typed_snapshot.documents[0]) - self.assertEqual(direct_applied.schema_id, schema_id) - self.assertEqual(direct_applied.record_key, "item:1") - typed_snapshot_applied = apply_raw_snapshot(cache, [document], typed_snapshot) - self.assertEqual(typed_snapshot_applied[0].record_key, "item:1") - - shard_log = { - "schemaVersion": "cultnet.shard_log_response.v0", - "messageId": "log-1", - "shardId": "interop", - "shardEpoch": 1, - "resyncRequired": False, - "entries": [ - { - "sequence": 0, - "changeKind": "updated", - "put": { - "schemaVersion": "cultnet.document_put_raw.v0", - "messageId": "put-unknown", - "document": { - "schemaId": "replica.unknown.v1", - "recordKey": "unknown:1", - "storedAt": "2026-06-13T00:00:01Z", - "payloadEncoding": "messagepack", - "payload": document.encode_payload(Item("ignored", "unknown", 0)), - }, - "shardId": "interop", - "shardEpoch": 1, - }, - }, - { - "sequence": 1, - "changeKind": "updated", - "put": { - "schemaVersion": "cultnet.document_put_raw.v0", - "messageId": "put-1", - "document": { - "schemaId": schema_id, - "recordKey": "item:1", - "storedAt": "2026-06-13T00:00:01Z", - "payloadEncoding": "messagepack", - "payload": document.encode_payload(Item("sword", "gear", 12)), - }, - "shardId": "interop", - "shardEpoch": 1, - }, - }, - { - "sequence": 2, - "changeKind": "removed", - "delete": { - "schemaVersion": "cultnet.document_delete.v0", - "messageId": "delete-1", - "schemaId": schema_id, - "recordKey": "item:1", - "shardId": "interop", - "shardEpoch": 1, - }, - }, - ], - } - applied_log = apply_shard_log_response(cache, [document], shard_log) - self.assertEqual([change.change_kind for change in applied_log], ["updated", "removed"]) - self.assertIsNone(cache.get(document, "item:1")) - - cache.put(document, "item:1", Item("sword", "gear", 3)) - typed_shard_log = CultNetShardLogResponse.from_wire({ - **shard_log, - "entries": shard_log["entries"][1:], - }) - typed_applied_log = apply_shard_log_response(cache, [document], typed_shard_log) - self.assertEqual([change.change_kind for change in typed_applied_log], ["updated", "removed"]) - self.assertIsNone(cache.get(document, "item:1")) - - def test_cultnet_replication_helpers_recover_schema_stamped_raw_records(self) -> None: - document = define_database_entry_type( - "replica.runtime-policy", - [ - ("schema_version", 0), - ("name", 1), - ("value", 2), - ], - schema_id="replica.runtime-policy.current", - schema_name="replica.runtime_policy", - schema_version="replica.runtime_policy.v1", - ) - cache = CultCache() - cache.register_document_type(document) - stale_schema_id = "sha256:stale-replica-runtime-policy" - payload = document.encode_payload({ - "schema_version": "replica.runtime_policy.v1", - "name": "policy", - "value": "still synced", - }) - record = { - "schemaId": stale_schema_id, - "recordKey": "policy:1", - "storedAt": "2026-06-25T00:00:00Z", - "payloadEncoding": "messagepack", - "payload": payload, - } - snapshot = { - "schemaVersion": "cultnet.snapshot_response_raw.v0", - "messageId": "snapshot-stale", - "documents": [record], - } - - applied_snapshot = apply_raw_snapshot(cache, [document], snapshot) - local_schema_id = document.catalog_entry().schema_id - self.assertEqual(applied_snapshot[0].schema_id, local_schema_id) - self.assertEqual(cache.get_required(document, "policy:1")["value"], "still synced") - self.assertEqual(cache.get_required_envelope(document, "policy:1").schema_id, local_schema_id) - direct_applied = apply_raw_document_record(cache, schema_document_map([document]), record) - self.assertEqual(direct_applied.schema_id, local_schema_id) - self.assertEqual(cache.get_required_envelope(document, "policy:1").schema_id, local_schema_id) - - shard_log = { - "schemaVersion": "cultnet.shard_log_response.v0", - "messageId": "log-stale", - "shardId": "interop", - "shardEpoch": 1, - "resyncRequired": False, - "entries": [ - { - "sequence": 0, - "changeKind": "updated", - "put": { - "schemaVersion": "cultnet.document_put_raw.v0", - "messageId": "put-stale", - "document": { - **record, - "payload": document.encode_payload({ - "schema_version": "replica.runtime_policy.v1", - "name": "policy", - "value": "updated through log", - }), - }, - "shardId": "interop", - "shardEpoch": 1, - }, - } - ], - } - - applied_log = apply_shard_log_response(cache, [document], shard_log) - self.assertEqual(applied_log[0].schema_id, local_schema_id) - self.assertEqual(cache.get_required(document, "policy:1")["value"], "updated through log") - self.assertEqual(cache.get_required_envelope(document, "policy:1").schema_id, local_schema_id) - - delete_log = { - "schemaVersion": "cultnet.shard_log_response.v0", - "messageId": "log-delete-alias", - "shardId": "interop", - "shardEpoch": 1, - "resyncRequired": False, - "entries": [ - { - "sequence": 1, - "changeKind": "removed", - "delete": { - "schemaVersion": "cultnet.document_delete.v0", - "messageId": "delete-version-alias", - "schemaId": "replica.runtime_policy.v1", - "recordKey": "policy:1", - "shardId": "interop", - "shardEpoch": 1, - }, - } - ], - } - applied_delete = apply_shard_log_response(cache, [document], delete_log) - self.assertEqual(applied_delete[0].schema_id, local_schema_id) - self.assertEqual(applied_delete[0].change_kind, "removed") - self.assertIsNone(cache.get(document, "policy:1")) - - def test_cultmesh_node_applies_foreign_schema_snapshot_as_local_update(self) -> None: - document = define_database_entry_type( - "mesh.runtime-policy", - [ - ("schema_version", 0), - ("name", 1), - ("value", 2), - ], - schema_id="mesh.runtime-policy.current", - schema_name="mesh.runtime_policy", - schema_version="mesh.runtime_policy.v1", - ) - node = create_node(runtime_id="python-node") - node.register_document(document) - node.put(document, "policy:1", { - "schema_version": "mesh.runtime_policy.v1", - "name": "policy", - "value": "old", - }) - observed: list[CultMeshDatabaseChange] = [] - unsubscribe = node.database.watch_record(document, "policy:1", observed.append) - try: - applied = node.database.apply_snapshot_response({ - "schemaVersion": "cultnet.snapshot_response_raw.v0", - "messageId": "foreign-schema-snapshot", - "documents": [ - { - "schemaId": "sha256:foreign-mesh-runtime-policy", - "recordKey": "policy:1", - "storedAt": "2026-06-25T00:00:00Z", - "payloadEncoding": "messagepack", - "payload": document.encode_payload({ - "schema_version": "mesh.runtime_policy.v1", - "name": "policy", - "value": "new", - }), - }, - ], - }) - finally: - unsubscribe() - - local_schema_id = document.catalog_entry().schema_id - self.assertEqual(applied[0].schema_id, local_schema_id) - self.assertEqual(observed[0].schema_id, local_schema_id) - self.assertEqual(observed[0].change_kind, "updated") - self.assertEqual(observed[0].previous_value["value"], "old") - self.assertEqual(observed[0].value["value"], "new") - self.assertEqual(node.cache.get_required_envelope(document, "policy:1").schema_id, local_schema_id) - - def test_cultmesh_node_syncs_snapshot_and_shard_log_through_cultnet_client(self) -> None: - import msgpack # type: ignore - from cultnet_py import read_frame, write_frame - - document = define_database_entry_type( - "mesh.sync_item", - [ - ("name", 0), - ("category", 1), - ("value", 2, 0), - ], - cls=Item, - ) - node = create_node(runtime_id="python-node") - node.register_document(document) - schema_id = document.catalog_entry().schema_id - ready = threading.Event() - server_error: list[BaseException] = [] - port_holder: list[int] = [] - - def serve_requests() -> None: - try: - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as server: - server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - server.bind(("127.0.0.1", 0)) - port_holder.append(server.getsockname()[1]) - server.listen(2) - ready.set() - for _ in range(2): - connection, _ = server.accept() - with connection: - stream = connection.makefile("rwb") - request = msgpack.unpackb(read_frame(stream), raw=False) - if request["schemaVersion"] == "cultnet.snapshot_request.v0": - response = { - "schemaVersion": "cultnet.snapshot_response_raw.v0", - "messageId": request["messageId"], - "documents": [ - { - "schemaId": schema_id, - "recordKey": "item:node", - "storedAt": "2026-06-13T00:00:00Z", - "payloadEncoding": "messagepack", - "payload": document.encode_payload(Item("node", "mesh", 1)), - } - ], - } - elif request["schemaVersion"] == "cultnet.shard_log_request.v0": - response = { - "schemaVersion": "cultnet.shard_log_response.v0", - "messageId": request["messageId"], - "shardId": request["shardId"], - "shardEpoch": request["shardEpoch"], - "resyncRequired": False, - "entries": [ - { - "sequence": 1, - "changeKind": "updated", - "put": { - "schemaVersion": "cultnet.document_put_raw.v0", - "messageId": "put-node", - "document": { - "schemaId": schema_id, - "recordKey": "item:node", - "storedAt": "2026-06-13T00:00:01Z", - "payloadEncoding": "messagepack", - "payload": document.encode_payload(Item("node", "mesh", 9)), - }, - "shardId": request["shardId"], - "shardEpoch": request["shardEpoch"], - }, - } - ], - } - else: - raise AssertionError(f"unexpected request {request['schemaVersion']}") - write_frame(stream, msgpack.packb(response, use_bin_type=True)) - stream.flush() - except BaseException as error: - server_error.append(error) - ready.set() - - thread = threading.Thread(target=serve_requests, daemon=True) - thread.start() - self.assertTrue(ready.wait(2.0)) - self.assertFalse(server_error) - - client = CultNetRawClient("127.0.0.1", port_holder[0], timeout_seconds=2.0) - snapshot_changes = node.sync_snapshot(client, schema_ids=[schema_id]) - log_changes = node.sync_shard_log(client, shard_id="mesh", shard_epoch=1) - - thread.join(2.0) - self.assertFalse(server_error) - self.assertEqual(snapshot_changes[0].record_key, "item:node") - self.assertEqual(log_changes[0].change_kind, "updated") - self.assertEqual(node.get_required(document, "item:node").value, 9) - - def test_cultmesh_node_sync_document_returns_requested_alias_type(self) -> None: - @dataclass - class CanonicalNote: - body: str - - @dataclass - class UiNote: - body: str - - document = define_database_entry_type( - "mesh.sync_alias_note", - [("body", 0)], - cls=CanonicalNote, - schema_id="mesh.sync_alias_note.v1", - schema_name="mesh.sync_alias_note", - schema_version="mesh.sync_alias_note.v1", - ) - alias = define_database_entry_type( - "mesh.sync_alias_note.ui", - [("body", 0)], - cls=UiNote, - schema_id="mesh.sync_alias_note.v1", - schema_name="mesh.sync_alias_note", - schema_version="mesh.sync_alias_note.v1", - ) - node = create_node(runtime_id="python-node") - node.register_document(document) - requests: list[dict[str, Any]] = [] - - class SnapshotClient: - def fetch_snapshot_response( - self, - *, - schema_ids: list[str] | None = None, - record_keys: list[str] | None = None, - shard_id: str | None = None, - shard_epoch: int | None = None, - ) -> dict[str, Any]: - requests.append({ - "schema_ids": schema_ids, - "record_keys": record_keys, - "shard_id": shard_id, - "shard_epoch": shard_epoch, - }) - return { - "schemaVersion": "cultnet.snapshot_response_raw.v0", - "messageId": "sync-document", - "documents": [ - { - "schemaId": document.catalog_entry().schema_id, - "recordKey": "note:remote", - "storedAt": "2026-06-27T00:00:00Z", - "payloadEncoding": "messagepack", - "payload": document.encode_payload(CanonicalNote("synced-alias")), - } - ], - } - - synced = node.sync_document(SnapshotClient(), alias, "note:remote") - facade_synced = CultMesh.sync_document(node, SnapshotClient(), alias, "note:remote") - - self.assertIsInstance(synced, UiNote) - self.assertEqual(synced.body, "synced-alias") - self.assertIsInstance(facade_synced, UiNote) - self.assertEqual(facade_synced.body, "synced-alias") - self.assertEqual(node.get_required(document, "note:remote").body, "synced-alias") - self.assertIsInstance(node.get_required(alias, "note:remote"), UiNote) - self.assertEqual(requests, [ - { - "schema_ids": ["mesh.sync_alias_note.v1"], - "record_keys": ["note:remote"], - "shard_id": None, - "shard_epoch": None, - }, - { - "schema_ids": ["mesh.sync_alias_note.v1"], - "record_keys": ["note:remote"], - "shard_id": None, - "shard_epoch": None, - }, - ]) - - def test_cultmesh_node_sync_document_from_publication_returns_requested_alias_type(self) -> None: - @dataclass - class CanonicalNote: - body: str - - @dataclass - class UiNote: - body: str - - document = define_database_entry_type( - "mesh.publication_alias_note", - [("body", 0)], - cls=CanonicalNote, - schema_id="mesh.publication_alias_note.v1", - schema_name="mesh.publication_alias_note", - schema_version="mesh.publication_alias_note.v1", - ) - alias = define_database_entry_type( - "mesh.publication_alias_note.ui", - [("body", 0)], - cls=UiNote, - schema_id="mesh.publication_alias_note.v1", - schema_name="mesh.publication_alias_note", - schema_version="mesh.publication_alias_note.v1", - ) - node = create_node(runtime_id="python-publication-node") - node.register_document(document) - requests: list[dict[str, Any]] = [] - - class SnapshotClient: - def fetch_snapshot_response( - self, - *, - schema_ids: list[str] | None = None, - record_keys: list[str] | None = None, - shard_id: str | None = None, - shard_epoch: int | None = None, - ) -> dict[str, Any]: - requests.append({ - "schema_ids": schema_ids, - "record_keys": record_keys, - "shard_id": shard_id, - "shard_epoch": shard_epoch, - }) - return { - "schemaVersion": "cultnet.snapshot_response_raw.v0", - "messageId": "publication-sync-document", - "documents": [ - { - "schemaId": document.catalog_entry().schema_id, - "recordKey": "note:published-peer", - "storedAt": "2026-06-27T00:00:00Z", - "payloadEncoding": "messagepack", - "payload": document.encode_payload(CanonicalNote("peer-publication-alias")), - } - ], - } - - source = CultMesh.publication_source_from_peer_snapshot( - SnapshotClient(), - shard_id="notes", - shard_epoch=3, - ) - - synced = node.sync_document_from_publication(source, alias, "note:published-peer") - facade_synced = CultMesh.sync_document_from_publication(node, source, alias, "note:published-peer") - - self.assertIsInstance(synced, UiNote) - self.assertEqual(synced.body, "peer-publication-alias") - self.assertIsInstance(facade_synced, UiNote) - self.assertEqual(facade_synced.body, "peer-publication-alias") - self.assertEqual(node.get_required(document, "note:published-peer").body, "peer-publication-alias") - self.assertEqual(requests, [ - { - "schema_ids": ["mesh.publication_alias_note.v1"], - "record_keys": ["note:published-peer"], - "shard_id": "notes", - "shard_epoch": 3, - }, - { - "schema_ids": ["mesh.publication_alias_note.v1"], - "record_keys": ["note:published-peer"], - "shard_id": "notes", - "shard_epoch": 3, - }, - ]) - - def test_cultmesh_node_sync_document_from_single_file_publication_uses_node_aliases(self) -> None: - @dataclass - class CanonicalNote: - body: str - - @dataclass - class UiNote: - body: str - - document = define_database_entry_type( - "mesh.file_publication_alias_note", - [("body", 0)], - cls=CanonicalNote, - schema_id="mesh.file_publication_alias_note.v1", - schema_name="mesh.file_publication_alias_note", - schema_version="mesh.file_publication_alias_note.v1", - ) - alias = define_database_entry_type( - "mesh.file_publication_alias_note.ui", - [("body", 0)], - cls=UiNote, - schema_id="mesh.file_publication_alias_note.v1", - schema_name="mesh.file_publication_alias_note", - schema_version="mesh.file_publication_alias_note.v1", - ) - with tempfile.TemporaryDirectory() as tmp: - path = Path(tmp) / "publication.ccmp" - writer = CultCache() - writer.register_document_type(document) - writer.add_generic_store(SingleFileMessagePackBackingStore(path)) - writer.put(document, "note:published-file", CanonicalNote("file-publication-alias")) - - node = create_node(runtime_id="python-file-publication-node") - node.register_document(document) - source = CultMesh.publication_source_from_single_file(path) - - synced = CultMesh.sync_document_from_publication(node, source, alias, "note:published-file") - - self.assertIsInstance(synced, UiNote) - self.assertEqual(synced.body, "file-publication-alias") - self.assertEqual(node.get_required(document, "note:published-file").body, "file-publication-alias") - self.assertIsInstance(node.get_required(alias, "note:published-file"), UiNote) - - def test_cultmesh_node_sync_documents_from_publication_uses_binding_sources_and_aliases(self) -> None: - @dataclass - class CanonicalNote: - body: str - - @dataclass - class UiNote: - body: str - - document = define_database_entry_type( - "mesh.catalog_publication_alias_note", - [("body", 0)], - cls=CanonicalNote, - schema_id="mesh.catalog_publication_alias_note.v1", - schema_name="mesh.catalog_publication_alias_note", - schema_version="mesh.catalog_publication_alias_note.v1", - ) - alias = define_database_entry_type( - "mesh.catalog_publication_alias_note.ui", - [("body", 0)], - cls=UiNote, - schema_id="mesh.catalog_publication_alias_note.v1", - schema_name="mesh.catalog_publication_alias_note", - schema_version="mesh.catalog_publication_alias_note.v1", - ) - with tempfile.TemporaryDirectory() as tmp: - first_path = Path(tmp) / "first.ccmp" - second_path = Path(tmp) / "second.ccmp" - first = CultCache() - first.register_document_type(document) - first.add_generic_store(SingleFileMessagePackBackingStore(first_path)) - first.put(document, "note:first", CanonicalNote("first catalog publication")) - second = CultCache() - second.register_document_type(document) - second.add_generic_store(SingleFileMessagePackBackingStore(second_path)) - second.put(document, "note:second", CanonicalNote("second catalog publication")) - - node = create_node(runtime_id="python-catalog-publication-node") - node.register_document(document) - source = CultMesh.publication_source_from_single_file(first_path) - bindings = [ - CultMesh.publication_document(alias, "note:first"), - CultMesh.publication_document( - alias, - "note:second", - source=CultMesh.publication_source_from_single_file(second_path), - ), - ] - - synced = node.sync_documents_from_publication(source, bindings) - facade_synced = CultMesh.sync_documents_from_publication(node, source, bindings) - - self.assertEqual([value.body for value in synced], [ - "first catalog publication", - "second catalog publication", - ]) - self.assertTrue(all(isinstance(value, UiNote) for value in synced)) - self.assertEqual([value.body for value in facade_synced], [ - "first catalog publication", - "second catalog publication", - ]) - self.assertEqual(node.get_required(document, "note:first").body, "first catalog publication") - self.assertEqual(node.get_required(document, "note:second").body, "second catalog publication") - self.assertIsInstance(node.get_required(alias, "note:second"), UiNote) - - def test_cultmesh_node_emits_raw_put_and_delete_messages_for_local_writes(self) -> None: - document = define_database_entry_type( - "mesh.emit_item", - [ - ("name", 0), - ("category", 1), - ("value", 2, 0), - ], - cls=Item, - ) - node = create_node(runtime_id="python-node") - node.register_document(document) - - put = node.put_raw_message( - document, - "item:emit", - Item("wand", "gear", 4), - message_id="put-emit", - shard_id="mesh", - shard_epoch=1, - ).to_wire() - self.assertEqual(node.get_required(document, "item:emit").name, "wand") - self.assertEqual(put["schemaVersion"], "cultnet.document_put_raw.v0") - self.assertEqual(put["messageId"], "put-emit") - self.assertEqual(put["document"]["schemaId"], document.catalog_entry().schema_id) - self.assertEqual(put["document"]["recordKey"], "item:emit") - self.assertEqual(put["document"]["sourceRuntimeId"], "python-node") - self.assertEqual(put["shardId"], "mesh") - self.assertEqual(put["shardEpoch"], 1) - - delete = node.delete_raw_message( - document, - "item:emit", - message_id="delete-emit", - shard_id="mesh", - shard_epoch=1, - ).to_wire() - self.assertIsNone(node.get(document, "item:emit")) - self.assertEqual(delete["schemaVersion"], "cultnet.document_delete.v0") - self.assertEqual(delete["messageId"], "delete-emit") - self.assertEqual(delete["schemaId"], document.catalog_entry().schema_id) - self.assertEqual(delete["recordKey"], "item:emit") - self.assertEqual(delete["shardId"], "mesh") - self.assertEqual(delete["shardEpoch"], 1) - - def test_cultmesh_node_resolves_same_schema_aliases_for_local_state(self) -> None: - @dataclass - class CanonicalNote: - body: str - - @dataclass - class UiNote: - body: str - - document = define_database_entry_type( - "mesh.alias_note", - [("body", 0)], - cls=CanonicalNote, - schema_id="mesh.alias_note.v1", - schema_name="mesh.alias_note", - schema_version="mesh.alias_note.v1", - ) - alias = define_database_entry_type( - "mesh.alias_note.ui", - [("body", 0)], - cls=UiNote, - schema_id="mesh.alias_note.v1", - schema_name="mesh.alias_note", - schema_version="mesh.alias_note.v1", - ) - node = create_node(runtime_id="python-node") - node.register_document(document) - seen: list[CultMeshDatabaseChange] = [] - all_seen: list[CultMeshDatabaseChange] = [] - node.database.watch(all_seen.append, document=alias) - node.database.watch_record(alias, "note:alias", seen.append) - - node.put(alias, "note:alias", UiNote("from-alias-put")) - self.assertEqual(node.get_required(document, "note:alias").body, "from-alias-put") - alias_value = node.get_required(alias, "note:alias") - self.assertIsInstance(alias_value, UiNote) - self.assertEqual(alias_value.body, "from-alias-put") - alias_collection = node.get_all(alias) - self.assertEqual(len(alias_collection), 1) - self.assertIsInstance(alias_collection[0], UiNote) - self.assertEqual(alias_collection[0].body, "from-alias-put") - self.assertEqual(seen[0].document.type, alias.type) - self.assertEqual(seen[0].schema_id, "mesh.alias_note.v1") - self.assertIsInstance(seen[0].value, UiNote) - self.assertEqual(seen[0].value.body, "from-alias-put") - self.assertEqual(all_seen[0].document.type, alias.type) - self.assertIsInstance(all_seen[0].value, UiNote) - self.assertEqual(all_seen[0].value.body, "from-alias-put") - - node.put(document, "note:alias", CanonicalNote("from-canonical-put")) - self.assertEqual(seen[1].document.type, alias.type) - self.assertIsInstance(seen[1].value, UiNote) - self.assertIsInstance(seen[1].previous_value, UiNote) - self.assertEqual(seen[1].value.body, "from-canonical-put") - self.assertEqual(seen[1].previous_value.body, "from-alias-put") - - put = node.put_raw_message( - alias, - "note:alias", - UiNote("from-alias-raw-put"), - message_id="alias-put", - ).to_wire() - self.assertEqual(put["document"]["schemaId"], "mesh.alias_note.v1") - self.assertEqual(put["document"]["recordKey"], "note:alias") - self.assertEqual(node.get_required(alias, "note:alias").body, "from-alias-raw-put") - - delete = node.delete_raw_message(alias, "note:alias", message_id="alias-delete").to_wire() - self.assertEqual(delete["schemaId"], "mesh.alias_note.v1") - self.assertIsNone(node.get(alias, "note:alias")) - self.assertEqual(node.get_all(alias), []) - - def test_cultmesh_reactive_document_coalesces_direct_alias_member_writes(self) -> None: - @dataclass - class CanonicalNote: - body: str - revision: int - - @dataclass - class UiNote: - body: str - revision: int - - document = define_database_entry_type( - "mesh.reactive_alias_note", - [("body", 0), ("revision", 1)], - cls=CanonicalNote, - schema_id="mesh.reactive_alias_note.v1", - schema_name="mesh.reactive_alias_note", - schema_version="mesh.reactive_alias_note.v1", - ) - alias = define_database_entry_type( - "mesh.reactive_alias_note.ui", - [("body", 0), ("revision", 1)], - cls=UiNote, - schema_id="mesh.reactive_alias_note.v1", - schema_name="mesh.reactive_alias_note", - schema_version="mesh.reactive_alias_note.v1", - ) - node = create_node(runtime_id="python-reactive") - node.register_document(document) - node.put(document, "note:reactive", CanonicalNote("initial", 1)) - seen: list[CultMeshDatabaseChange] = [] - node.database.watch_record(alias, "note:reactive", seen.append) - - reactive = node.reactive_document( - alias, - "note:reactive", - CultMeshReactiveDocumentOptions(flush_delay_seconds=0.02), - ) - try: - self.assertIsInstance(reactive.current, UiNote) - reactive.current.body = "first-local-edit" - reactive.current.body = "second-local-edit" - reactive.current.revision = 2 - - self._wait_until(lambda: node.get_required(alias, "note:reactive").body == "second-local-edit") - - self.assertFalse(reactive.is_dirty) - self.assertEqual(node.get_required(document, "note:reactive").revision, 2) - self.assertEqual(seen[-1].value.body, "second-local-edit") - finally: - reactive.dispose() - - def test_cultmesh_reactive_document_tracks_canonical_reconciliation_delta(self) -> None: - @dataclass - class Note: - body: str - revision: int - - document = define_database_entry_type( - "mesh.reactive_reconcile_note", - [("body", 0), ("revision", 1)], - cls=Note, - schema_id="mesh.reactive_reconcile_note.v1", - schema_name="mesh.reactive_reconcile_note", - schema_version="mesh.reactive_reconcile_note.v1", - ) - node = create_node(runtime_id="python-reactive-reconcile") - node.register_document(document) - node.put(document, "note:reconcile", Note("initial", 1)) - - reactive = CultMesh.reactive_document( - node, - document, - "note:reconcile", - CultMeshReactiveDocumentOptions(flush_delay_seconds=60), - ) - try: - reactive.update(lambda note: setattr(note, "body", "local-prediction")) - reactive.current.revision = 2 - - node.put(document, "note:reconcile", Note("authoritative", 7)) - - self.assertEqual(reactive.current.body, "local-prediction") - self.assertIsNotNone(reactive.reconciliation) - self.assertEqual(reactive.reconciliation.canonical.body, "authoritative") - self.assertEqual(reactive.reconciliation.predicted.body, "local-prediction") - self.assertEqual(reactive.reconciliation.delta["body"], "local-prediction") - self.assertEqual(reactive.reconciliation.delta["revision"], -5) - - reactive.flush() - self.assertEqual(node.get_required(document, "note:reconcile").body, "local-prediction") - self.assertIsNone(reactive.reconciliation) - finally: - reactive.dispose() - - def test_cultnet_document_delete_helper_matches_schema_v0_shape(self) -> None: - message = document_delete( - message_id="delete-1", - schema_id="schema-a", - record_key="record-a", - shard_id="interop", - shard_epoch=1, - ).to_wire() - self.assertEqual(message["schemaVersion"], "cultnet.document_delete.v0") - self.assertEqual(message["messageId"], "delete-1") - self.assertEqual(message["schemaId"], "schema-a") - self.assertEqual(message["recordKey"], "record-a") - self.assertEqual(message["shardId"], "interop") - self.assertEqual(message["shardEpoch"], 1) - - def test_cultnet_auth_session_helpers_match_schema_v0_shape(self) -> None: - self.assertEqual( - login(nonce="nonce", auth="encrypted-auth", password="encrypted-password").to_wire(), - { - "schemaVersion": "cultnet.login.v0", - "nonce": "nonce", - "auth": "encrypted-auth", - "password": "encrypted-password", - }, - ) - self.assertEqual( - register( - nonce="nonce", - email="encrypted-email", - password="encrypted-password", - name="encrypted-name", - ).to_wire(), - { - "schemaVersion": "cultnet.register.v0", - "nonce": "nonce", - "email": "encrypted-email", - "password": "encrypted-password", - "name": "encrypted-name", - }, - ) - self.assertEqual( - verify_session(nonce="nonce", session="encrypted-session").to_wire(), - { - "schemaVersion": "cultnet.verify.v0", - "nonce": "nonce", - "session": "encrypted-session", - }, - ) - self.assertEqual( - login_success(nonce="nonce", session="encrypted-session").to_wire(), - { - "schemaVersion": "cultnet.login_success.v0", - "nonce": "nonce", - "session": "encrypted-session", - }, - ) - - def test_cultnet_wire_catalog_describes_python_handled_messages(self) -> None: - descriptors = wire_message_schema_descriptors(include_schema_json=True) - by_version = {descriptor["schemaVersion"]: descriptor for descriptor in descriptors} - for schema_version in [ - "cultnet.error.v0", - "cultnet.login.v0", - "cultnet.register.v0", - "cultnet.verify.v0", - "cultnet.login_success.v0", - "cultnet.transport_profile.v0", - "cultnet.document_delete.v0", - "cultnet.database_change_raw.v0", - "cultnet.shard_log_response.v0", - "cultnet.simulation_consensus_candidate.v0", - "cultmesh.peer_exchange_response.v0", - ]: - self.assertIn(schema_version, by_version) - expected_kind = "shared_contract" if schema_version == "cultnet.transport_profile.v0" else "wire_message" - self.assertEqual(by_version[schema_version]["kind"], expected_kind) - self.assertIn("cultnet.schema.v0", by_version[schema_version]["wireContracts"]) - self.assertIn(schema_version, by_version[schema_version]["schemaJson"]) - self.assertEqual(len(by_version[schema_version]["contentHash"]), 64) - - delete_schema = json.loads(by_version["cultnet.document_delete.v0"]["schemaJson"]) - self.assertEqual( - delete_schema["required"], - ["schemaVersion", "messageId", "schemaId", "recordKey"], - ) - self.assertEqual(delete_schema["properties"]["recordKey"]["type"], "string") - - error_schema = json.loads(by_version["cultnet.error.v0"]["schemaJson"]) - self.assertEqual(error_schema["required"], ["schemaVersion", "error"]) - self.assertEqual(error_schema["properties"]["error"]["type"], "string") - self.assertIn("details", error_schema["properties"]) - - login_schema = json.loads(by_version["cultnet.login.v0"]["schemaJson"]) - self.assertEqual(login_schema["required"], ["schemaVersion", "nonce", "auth", "password"]) - self.assertEqual(login_schema["properties"]["auth"]["type"], "string") - - register_schema = json.loads(by_version["cultnet.register.v0"]["schemaJson"]) - self.assertEqual(register_schema["required"], ["schemaVersion", "nonce", "email", "password", "name"]) - self.assertEqual(register_schema["properties"]["email"]["type"], "string") - - verify_schema = json.loads(by_version["cultnet.verify.v0"]["schemaJson"]) - self.assertEqual(verify_schema["required"], ["schemaVersion", "nonce", "session"]) - self.assertEqual(verify_schema["properties"]["session"]["type"], "string") - - success_schema = json.loads(by_version["cultnet.login_success.v0"]["schemaJson"]) - self.assertEqual(success_schema["required"], ["schemaVersion", "nonce", "session"]) - - transport_profile_schema = json.loads(by_version["cultnet.transport_profile.v0"]["schemaJson"]) - self.assertEqual(transport_profile_schema["required"], ["schemaVersion", "runtimeId", "transports"]) - transport = transport_profile_schema["properties"]["transports"]["items"] - self.assertEqual(transport["properties"]["protocol"]["enum"], ["tcp_framed", "litenetlib", "websocket", "rudp"]) - self.assertEqual( - transport["properties"]["reconnectPolicy"]["properties"]["schemaVersion"]["const"], - "cultnet.reconnect_policy.v0", - ) - channel = transport["properties"]["channels"]["items"] - self.assertEqual(channel["properties"]["ordering"]["enum"], ["ordered", "unordered", "sequenced"]) - - change_schema = json.loads(by_version["cultnet.database_change_raw.v0"]["schemaJson"]) - self.assertEqual(change_schema["properties"]["changeKind"]["enum"], ["added", "updated", "removed"]) - self.assertIn("document", change_schema["properties"]) - self.assertIn("schemaId", change_schema["properties"]) - self.assertIn("recordKey", change_schema["properties"]) - - log_schema = json.loads(by_version["cultnet.shard_log_response.v0"]["schemaJson"]) - self.assertIn("entries", log_schema["required"]) - self.assertIn("resyncRequired", log_schema["required"]) - self.assertIn("compactedThrough", log_schema["properties"]) - - hello_schema = json.loads(by_version["cultnet.hello.v0"]["schemaJson"]) - mutation_contract = hello_schema["properties"]["supportedMutationContracts"]["items"] - self.assertEqual(mutation_contract["type"], "object") - self.assertIn("operations", mutation_contract["properties"]) - self.assertEqual( - hello_schema["properties"]["transportProfiles"]["items"]["$ref"], - "https://github.com/GameCult/cultnet-ts/contracts/cultnet.transport-profile.schema.json", - ) - - consensus_schema = json.loads(by_version["cultnet.simulation_consensus_candidate.v0"]["schemaJson"]) - for required_field in ["witnessCount", "supportWeight", "totalWeight", "hasQuorum", "confidence"]: - self.assertIn(required_field, consensus_schema["required"]) - - def test_cultnet_schema_catalog_applies_filters_and_responses(self) -> None: - catalog = wire_message_schema_catalog(include_schema_json=True) - descriptor = catalog.get("https://github.com/GameCult/cultnet-ts/contracts/cultnet.document-put-raw.schema.json") - self.assertIsNotNone(descriptor) - assert descriptor is not None - self.assertEqual(descriptor.kind, "wire_message") - self.assertEqual(descriptor.schema_version, "cultnet.document_put_raw.v0") - self.assertIsNotNone(descriptor.schema_json) - - filtered = catalog.list(kinds=["wire_message"], schema_ids=[descriptor.schema_id]) - self.assertEqual(filtered, [descriptor]) - - response = catalog.create_response( - message_id="catalog-response", - include_schema_json=False, - schema_ids=[descriptor.schema_id], - kinds=["wire_message"], - ) - self.assertEqual(response["schemaVersion"], "cultnet.schema_catalog_response.v0") - self.assertNotIn("schemaJson", response["schemas"][0]) - - remote = CultNetSchemaCatalog() - applied = remote.apply_response({ - "schemaVersion": "cultnet.schema_catalog_response.v0", - "messageId": "remote", - "schemas": [ - CultNetSchemaDescriptor( - schema_id="schema:custom", - kind="shared_contract", - schema_version="custom.v0", - wire_contracts=("cultnet.schema.v0",), - content_hash="hash", - schema_json="{}", - ).to_wire() - ], - }) - self.assertEqual(applied[0].schema_id, "schema:custom") - self.assertEqual(remote.get("schema:custom"), applied[0]) - - def test_cultnet_shard_helpers_match_schema_v0_shape(self) -> None: - catalog = shard_catalog_request( - message_id="catalog-1", - schema_ids=["schema-a"], - record_keys=["record-a"], - ).to_wire() - self.assertEqual(catalog["schemaVersion"], "cultnet.shard_catalog_request.v0") - self.assertEqual(catalog["messageId"], "catalog-1") - self.assertEqual(catalog["schemaIds"], ["schema-a"]) - self.assertEqual(catalog["recordKeys"], ["record-a"]) - - log = shard_log_request( - message_id="log-1", - shard_id="interop", - shard_epoch=7, - after_sequence=3, - limit=2, - ).to_wire() - self.assertEqual(log["schemaVersion"], "cultnet.shard_log_request.v0") - self.assertEqual(log["messageId"], "log-1") - self.assertEqual(log["shardId"], "interop") - self.assertEqual(log["shardEpoch"], 7) - self.assertEqual(log["afterSequence"], 3) - self.assertEqual(log["limit"], 2) - - def test_cultnet_shard_catalog_applies_filters_and_responses(self) -> None: - catalog = CultNetShardCatalog() - primary = catalog.upsert(CultNetShardDescriptor( - shard_id="notes", - owner_runtime_id="python-runtime", - epoch=2, - is_primary=True, - schema_ids=("schema-note.v1",), - key_prefix="note:", - primary_endpoints=("cultnet://127.0.0.1:3075",), - read_replica_endpoints=("cultnet://127.0.0.1:3075",), - region="local", - )) - catalog.upsert({ - "shardId": "facts", - "ownerRuntimeId": "python-runtime", - "epoch": 1, - "schemaIds": ["schema-fact"], - "keyPrefix": "fact:", - "primaryEndpoints": ["cultnet://127.0.0.1:3076"], - }) - - self.assertTrue(primary.serves(schema_id="schema-note.v1", record_key="note:1")) - self.assertTrue(primary.serves(schema_id="schema-note", record_key="note:1")) - self.assertFalse(primary.serves(schema_id="schema-note", record_key="fact:1")) - self.assertEqual(catalog.list(schema_ids=["schema-note.v1"]), [primary]) - self.assertEqual(catalog.list(schema_ids=["schema-note"]), [primary]) - self.assertEqual(catalog.list(record_keys=["note:1"]), [primary]) - - response = catalog.create_response( - message_id="shards", - schema_ids=["schema-note"], - record_keys=["note:1"], - ) - self.assertEqual(response["schemaVersion"], "cultnet.shard_catalog_response.v0") - self.assertEqual(response["shards"][0]["shardId"], "notes") - self.assertEqual(response["shards"][0]["ownerRuntimeId"], "python-runtime") - - remote = CultNetShardCatalog() - applied = remote.apply_response(response) - self.assertEqual(applied[0].shard_id, "notes") - self.assertEqual(remote.get("notes"), applied[0]) - - def test_cultnet_shard_log_response_tracks_cursor_and_resync_state(self) -> None: - response = CultNetShardLogResponse.from_wire({ - "schemaVersion": "cultnet.shard_log_response.v0", - "messageId": "log", - "shardId": "notes", - "shardEpoch": 2, - "entries": [ - CultNetShardLogEntry( - sequence=2, - change_kind="updated", - put={"schemaVersion": "cultnet.document_put_raw.v0", "messageId": "put", "document": {"schemaId": "schema-note", "recordKey": "note:1", "payload": b"p"}}, - committed_at="2026-06-14T00:00:00Z", - ).to_wire(), - {"sequence": 3, "changeKind": "removed", "delete": {"schemaId": "schema-note", "recordKey": "note:1"}}, - ], - "resyncRequired": False, - }) - self.assertEqual(response.last_sequence, 3) - self.assertFalse(response.resync_required) - self.assertEqual(response.entries[0].put["document"]["recordKey"], "note:1") - self.assertIsInstance(response.entries[0].raw_document, CultNetRawDocumentRecord) - self.assertEqual(response.entries[0].raw_document.record_key, "note:1") - self.assertEqual(response.entries[1].delete_schema_id, "schema-note") - self.assertEqual(response.entries[1].delete_record_key, "note:1") - self.assertEqual(response.to_wire()["entries"][1]["delete"]["recordKey"], "note:1") - self.assertIs(response.require_usable(), response) - - resync = CultNetShardLogResponse.from_wire({ - "schemaVersion": "cultnet.shard_log_response.v0", - "messageId": "log-resync", - "shardId": "notes", - "shardEpoch": 2, - "entries": [], - "resyncRequired": True, - "reason": "compacted", - "compactedThrough": 12, - }) - self.assertEqual(resync.last_sequence, 12) - with self.assertRaisesRegex(ValueError, "compacted"): - resync.require_usable() - - def test_cultnet_simulation_observation_helper_matches_schema_v0_shape(self) -> None: - claim_hash = compute_simulation_claim_hash("frame:42", "subject:player-1", "hit") - self.assertEqual(len(claim_hash), 64) - message = simulation_observation( - message_id="obs-1", - witness_runtime_id="python-test", - shard_id="interop", - shard_epoch=1, - frame=42, - subject_id="player-1", - claim_kind="hit", - claim_hash=claim_hash, - claim_summary="player-1 hit target-a", - observed_at="2026-06-13T00:00:02Z", - ).to_wire() - self.assertEqual(message["schemaVersion"], "cultnet.simulation_observation.v0") - self.assertEqual(message["messageId"], "obs-1") - self.assertEqual(message["observation"]["witnessRuntimeId"], "python-test") - self.assertEqual(message["observation"]["claimHash"], claim_hash) - self.assertEqual(message["observation"]["weight"], 1.0) - observation = CultNetSimulationObservation.from_wire(message) - self.assertEqual(observation.witness_runtime_id, "python-test") - self.assertEqual(observation.claim_summary, "player-1 hit target-a") - self.assertEqual(observation.to_message_wire(message_id="obs-2")["observation"], message["observation"]) - - def test_cultnet_witness_artifact_bundle_uses_csharp_slot_order(self) -> None: - import msgpack # type: ignore - - bundle = witness_artifact_bundle( - bundle_id="bundle-1", - witness_kind="interop-proof", - captured_at="2026-06-13T00:00:03Z", - subject={"documentType": "cultnet.interop-note", "subjectId": "note:python"}, - contracts=[{"role": "payload", "schemaId": "schema-a"}], - artifacts=[{"role": "log", "uri": "cultcache://bundle-1/log", "mediaType": "text/plain"}], - timing_witnesses=[{"stage": "roundtrip", "startedAt": "2026-06-13T00:00:03Z", "completedAt": "2026-06-13T00:00:04Z", "latencyMs": 1.0}], - provenance={"pipelineId": "interop", "runId": "run-1", "runtimeId": "python-test"}, - ) - payload = encode_witness_artifact_bundle_payload(bundle) - decoded = decode_witness_artifact_bundle_payload(payload) - typed = CultNetWitnessArtifactBundle.from_wire(bundle) - typed_payload = typed.to_payload() - typed_decoded = CultNetWitnessArtifactBundle.from_payload(typed_payload) - slots = msgpack.unpackb(payload, raw=False) - - self.assertEqual(payload, typed_payload) - self.assertEqual(slots[0], "bundle-1") - self.assertEqual(slots[1], "interop-proof") - self.assertEqual(slots[2], "2026-06-13T00:00:03Z") - self.assertEqual(slots[3]["subjectId"], "note:python") - self.assertEqual(slots[4][0]["schemaId"], "schema-a") - self.assertEqual(slots[5][0]["uri"], "cultcache://bundle-1/log") - self.assertEqual(slots[6][0]["latencyMs"], 1.0) - self.assertEqual(slots[7]["runtimeId"], "python-test") - self.assertEqual(decoded["bundleId"], "bundle-1") - self.assertEqual(decoded["witnessKind"], "interop-proof") - self.assertEqual(decoded["subject"]["subjectId"], "note:python") - self.assertEqual(decoded["contracts"][0]["schemaId"], "schema-a") - self.assertEqual(decoded["artifacts"][0]["mediaType"], "text/plain") - self.assertEqual(decoded["provenance"]["runtimeId"], "python-test") - self.assertEqual(typed.bundle_id, "bundle-1") - self.assertEqual(typed_decoded.to_wire(), bundle) - - def test_cultmesh_node_uses_cultcache_store(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - document = define_database_entry_type("mesh.note", [("body", 0)]) - store_path = Path(tmp) / "mesh.cc" - node = create_node(store_path, runtime_id="mesh-test") - self.assertIsInstance(node.database, CultMeshDatabase) - node.database.register_document(document) - node.database.put(document, "note:1", {"body": "hello"}) - self.assertEqual(node.get_required(document, "note:1")["body"], "hello") - - reopened = create_node(store_path, runtime_id="mesh-test") - reopened.database.register_document(document) - reopened.database.pull() - self.assertEqual(reopened.database.get(document, "note:1")["body"], "hello") - - def test_cultmesh_node_options_attach_default_durable_shard_log_store(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - document = define_database_entry_type( - "mesh.durable_option_note", - [("body", 0)], - schema_id="mesh.durable_option_note.v1", - ) - cache_path = Path(tmp) / "world.cc" - node = CultMesh.create_node( - cache_path, - runtime_id="durable-option-primary", - options=CultMeshNodeOptions(enable_durable_shard_logs=True), - ) - node.database.register_document(document) - node.database.put_raw_message(document, "note:1", {"body": "stored"}, shard_id="notes", shard_epoch=5) - - log_path = Path(tmp) / "world.cultmesh" / "shard-logs" - self.assertTrue(log_path.exists()) - - reopened = CultMesh.start_node( - cache_path, - runtime_id="durable-option-primary", - enable_durable_shard_logs=True, - ) - reopened.database.register_document(document) - response = reopened.database.build_shard_log_response(shard_id="notes", shard_epoch=5) - - self.assertEqual(reopened.database.shard_ids(), ["notes"]) - self.assertEqual([entry.sequence for entry in response.entries], [1]) - self.assertEqual(response.entries[0].raw_document.record_key, "note:1") - - def test_cultmesh_database_watchers_observe_local_changes_and_unsubscribe(self) -> None: - document = define_database_entry_type("mesh.watch", [("body", 0)]) - node = CultMesh.create_node(runtime_id="mesh-watch") - node.database.register_document(document) - changes: list[CultMeshDatabaseChange] = [] - all_changes: list[CultMeshDatabaseChange] = [] - - unsubscribe_record = node.database.watch_record(document, "note:1", changes.append) - unsubscribe_all = node.database.watch(all_changes.append, document=document) - - node.database.put(document, "note:1", {"body": "hello"}) - node.database.put(document, "note:2", {"body": "skip"}) - node.database.put(document, "note:1", {"body": "updated"}) - node.database.delete(document, "note:1") - unsubscribe_record() - node.database.put(document, "note:1", {"body": "after-unsubscribe"}) - unsubscribe_all() - - self.assertEqual([change.change_kind for change in changes], ["added", "updated", "removed"]) - self.assertEqual([change.record_key for change in changes], ["note:1", "note:1", "note:1"]) - self.assertEqual(changes[1].previous_value, {"body": "hello"}) - self.assertIsNone(changes[2].value) - self.assertEqual([change.record_key for change in all_changes], ["note:1", "note:2", "note:1", "note:1", "note:1"]) - - def test_cultmesh_database_register_document_is_idempotent(self) -> None: - document = define_database_entry_type("mesh.idempotent", [("body", 0)]) - conflicting = define_database_entry_type("mesh.idempotent", [("title", 0)]) - node = CultMesh.create_node(runtime_id="mesh-idempotent") - - node.database.register_document(document) - node.database.register_document(document) - node.database.put(document, "note:1", {"body": "still registered once"}) - - self.assertEqual(node.database.get_required(document, "note:1")["body"], "still registered once") - self.assertEqual([registered.type for registered in node.documents], ["mesh.idempotent"]) - with self.assertRaises(ValueError): - node.database.register_document(conflicting) - - def test_cultmesh_database_global_document_facade_uses_singleton_key_and_watchers(self) -> None: - document = define_database_entry_type( - "mesh.global_settings", - [("theme", 0)], - global_document=True, - ) - node = CultMesh.create_node(runtime_id="mesh-global") - node.database.register_document(document) - changes: list[CultMeshDatabaseChange] = [] - - node.database.watch_global(document, changes.append) - node.database.put_global(document, {"theme": "ash"}) - node.put_global(document, {"theme": "ember"}) - node.delete_global(document) - - self.assertIsNone(node.database.get_global(document)) - self.assertEqual([change.record_key for change in changes], [node.cache.GLOBAL_KEY] * 3) - self.assertEqual([change.change_kind for change in changes], ["added", "updated", "removed"]) - self.assertEqual(changes[1].previous_value, {"theme": "ash"}) - self.assertEqual(changes[2].previous_value, {"theme": "ember"}) - with self.assertRaises(Exception): - node.database.put(document, "settings:wrong", {"theme": "bad"}) - - def test_cultmesh_database_creates_raw_snapshot_response_from_envelopes(self) -> None: - document = define_database_entry_type( - "mesh.snapshot_note", - [("body", 0)], - schema_id="mesh.snapshot_note.v1", - ) - source = CultMesh.create_node(runtime_id="mesh-snapshot-source") - source.database.register_document(document) - source.database.put(document, "note:1", {"body": "include"}) - source.database.put(document, "note:2", {"body": "skip"}) - - typed_response = source.database.build_snapshot_response( - message_id="snapshot-1", - schema_ids=["mesh.snapshot_note.v1"], - record_keys=["note:1"], - shard_id="notes", - shard_epoch=4, - shard_log_sequence=7, - ) - response = source.database.create_snapshot_response( - message_id="snapshot-1", - schema_ids=["mesh.snapshot_note.v1"], - record_keys=["note:1"], - shard_id="notes", - shard_epoch=4, - shard_log_sequence=7, - ) - - self.assertIsInstance(typed_response, CultNetRawSnapshotResponse) - self.assertEqual(typed_response.message_id, "snapshot-1") - self.assertEqual(typed_response.documents[0].record_key, "note:1") - self.assertEqual(typed_response.to_wire(), response) - self.assertEqual(source.build_snapshot_response(schema_ids=["mesh.snapshot_note.v1"]).documents[0].record_key, "note:1") - self.assertEqual(response["schemaVersion"], "cultnet.snapshot_response_raw.v0") - self.assertEqual(response["messageId"], "snapshot-1") - self.assertEqual(response["shardId"], "notes") - self.assertEqual(response["shardEpoch"], 4) - self.assertEqual(response["shardLogSequence"], 7) - self.assertEqual([record["recordKey"] for record in response["documents"]], ["note:1"]) - self.assertEqual(response["documents"][0]["schemaId"], "mesh.snapshot_note.v1") - self.assertEqual(response["documents"][0]["payloadEncoding"], "messagepack") - - target = CultMesh.create_node(runtime_id="mesh-snapshot-target") - target.database.register_document(document) - applied = target.database.apply_snapshot_response(response) - self.assertEqual([(record.schema_id, record.record_key) for record in applied], [("mesh.snapshot_note.v1", "note:1")]) - self.assertEqual(target.database.get_required(document, "note:1"), {"body": "include"}) - self.assertIsNone(target.database.get(document, "note:2")) - - typed_target = CultMesh.create_node(runtime_id="mesh-snapshot-typed-target") - typed_target.database.register_document(document) - typed_applied = typed_target.database.apply_snapshot_response(typed_response) - self.assertEqual([(record.schema_id, record.record_key) for record in typed_applied], [("mesh.snapshot_note.v1", "note:1")]) - self.assertEqual(typed_target.database.get_required(document, "note:1"), {"body": "include"}) - - def test_cultmesh_database_filters_snapshot_by_logged_shard_membership(self) -> None: - document = define_database_entry_type( - "mesh.sharded_snapshot_note", - [("body", 0)], - schema_id="mesh.sharded_snapshot_note.v1", - ) - source = CultMesh.create_node(runtime_id="mesh-sharded-snapshot-source") - source.database.register_document(document) - source.database.put_raw_message(document, "note:notes", {"body": "include"}, shard_id="notes", shard_epoch=2) - source.database.put_raw_message(document, "note:other", {"body": "skip"}, shard_id="other", shard_epoch=1) - - response = source.database.build_snapshot_response( - schema_ids=["mesh.sharded_snapshot_note.v1"], - shard_id="notes", - shard_epoch=2, - ) - - self.assertEqual(response.shard_id, "notes") - self.assertEqual(response.shard_epoch, 2) - self.assertEqual([record.record_key for record in response.documents], ["note:notes"]) - - def test_python_interop_peer_filters_shard_snapshot_by_logged_membership(self) -> None: - runtime_id = f"python-interop-test-{uuid4().hex}" - state = build_state( - runtime_id=runtime_id, - runtime_kind="python", - display_name="Python Interop Test", - agent_id="python-interop-test-agent", - schema_path=str(Path("packages/cultnet-ts/integration/contracts/cultnet.interop-note.schema.json")), - ) - binding = state.bindings["note"] - logged_value = { - "schemaVersion": "cultnet.interop_note.v0", - "documentId": "note:logged", - "authorRuntimeId": runtime_id, - "title": "Logged", - "body": "This record belongs to the shard log.", - "tags": ["interop", "logged"], - } - unlogged_value = { - **logged_value, - "documentId": "note:unlogged", - "title": "Unlogged", - "body": "This record is in cache but not in the shard log.", - } - state.cache.put(binding.document, "note:logged", logged_value) - state.cache.put(binding.document, "note:unlogged", unlogged_value) - logged_record = { - "schemaId": state.note_schema_id, - "recordKey": "note:logged", - "storedAt": "2026-06-14T00:00:00Z", - "payloadEncoding": "messagepack", - "payload": binding.document.encode_payload(logged_value), - } - append_shard_log_put( - state, - {"messageId": "logged-put"}, - logged_record, - ) - - response = raw_snapshot_response( - state, - { - "schemaVersion": "cultnet.snapshot_request.v0", - "messageId": "interop-shard-snapshot", - "schemaIds": [state.note_schema_id], - "shardId": state.shard_id, - }, - ) - - self.assertEqual(response["shardId"], state.shard_id) - self.assertEqual(response["shardLogSequence"], 1) - self.assertEqual([record["recordKey"] for record in response["documents"]], ["note:logged"]) - - def test_cultmesh_database_creates_shard_log_response_from_raw_mutations(self) -> None: - document = define_database_entry_type( - "mesh.log_note", - [("body", 0)], - schema_id="mesh.log_note.v1", - ) - source = CultMesh.create_node(runtime_id="mesh-log-source") - source.database.register_document(document) - source.database.put_raw_message(document, "note:1", {"body": "first"}, shard_id="notes", shard_epoch=3) - source.database.put_raw_message(document, "note:1", {"body": "second"}, shard_id="notes", shard_epoch=3) - source.database.put_raw_message(document, "note:2", {"body": "other"}, shard_id="other", shard_epoch=1) - source.database.delete_raw_message(document, "note:1", shard_id="notes", shard_epoch=3) - - self.assertEqual(source.database.shard_ids(), ["notes", "other"]) - self.assertEqual(source.database.shard_schema_ids("notes"), ["mesh.log_note.v1"]) - - typed_response = source.database.build_shard_log_response( - message_id="log-1", - shard_id="notes", - shard_epoch=3, - after_sequence=1, - limit=2, - ) - response = source.database.create_shard_log_response( - message_id="log-1", - shard_id="notes", - shard_epoch=3, - after_sequence=1, - limit=2, - ) - - self.assertIsInstance(typed_response, CultNetShardLogResponse) - self.assertEqual(typed_response.message_id, "log-1") - self.assertEqual([entry.sequence for entry in typed_response.entries], [2, 3]) - self.assertEqual(typed_response.to_wire(), response) - self.assertEqual([entry.sequence for entry in source.build_shard_log_response(shard_id="notes").entries], [1, 2, 3]) - self.assertEqual(response["schemaVersion"], "cultnet.shard_log_response.v0") - self.assertEqual(response["messageId"], "log-1") - self.assertEqual(response["shardId"], "notes") - self.assertEqual(response["shardEpoch"], 3) - self.assertFalse(response["resyncRequired"]) - self.assertEqual([entry["sequence"] for entry in response["entries"]], [2, 3]) - self.assertEqual([entry["changeKind"] for entry in response["entries"]], ["updated", "removed"]) - self.assertEqual(response["entries"][0]["put"]["document"]["recordKey"], "note:1") - self.assertEqual(response["entries"][1]["delete"]["recordKey"], "note:1") - empty = source.database.create_shard_log_response(shard_id="missing") - self.assertEqual(empty["shardEpoch"], 0) - self.assertEqual(empty["entries"], []) - self.assertTrue(empty["resyncRequired"]) - self.assertEqual(empty["reason"], "unknown_shard") - stale = source.database.create_shard_log_response(shard_id="notes", shard_epoch=2) - self.assertEqual(stale["shardEpoch"], 3) - self.assertEqual(stale["entries"], []) - self.assertTrue(stale["resyncRequired"]) - self.assertEqual(stale["reason"], "stale_epoch") - with self.assertRaises(ValueError): - source.database.create_shard_log_response(shard_id="notes", after_sequence=-1) - with self.assertRaises(ValueError): - source.database.create_shard_log_response(shard_id="notes", limit=-1) - - target = CultMesh.create_node(runtime_id="mesh-log-target") - target.database.register_document(document) - target.database.put(document, "note:1", {"body": "first"}) - applied = target.database.apply_shard_log_response(response) - self.assertEqual([record.change_kind for record in applied], ["updated", "removed"]) - self.assertIsNone(target.database.get(document, "note:1")) - self.assertIsNone(target.database.get(document, "note:2")) - - typed_target = CultMesh.create_node(runtime_id="mesh-log-typed-target") - typed_target.database.register_document(document) - typed_target.database.put(document, "note:1", {"body": "first"}) - typed_applied = typed_target.database.apply_shard_log_response(typed_response) - self.assertEqual([record.change_kind for record in typed_applied], ["updated", "removed"]) - self.assertIsNone(typed_target.database.get(document, "note:1")) - - def test_cultnet_file_shard_mutation_log_store_persists_authoritative_entries(self) -> None: - document = define_database_entry_type( - "mesh.durable_log_note", - [("body", 0)], - schema_id="mesh.durable_log_note.v1", - ) - with tempfile.TemporaryDirectory() as tmp: - store: CultNetShardMutationLogStore = CultNetFileShardMutationLogStore(Path(tmp) / "logs") - source = CultMesh.create_node(runtime_id="durable-primary") - source.database.register_document(document) - source.database.use_shard_mutation_log_store(store) - source.database.put_raw_message(document, "note:1", {"body": "first"}, shard_id="durable", shard_epoch=4) - source.database.put_raw_message(document, "note:2", {"body": "second"}, shard_id="durable", shard_epoch=4) - - reopened = CultMesh.create_node(runtime_id="durable-primary") - reopened.database.register_document(document) - reopened.database.use_shard_mutation_log_store(CultNetFileShardMutationLogStore(Path(tmp) / "logs")) - response = reopened.database.build_shard_log_response(shard_id="durable", shard_epoch=4) - self.assertEqual(reopened.database.shard_ids(), ["durable"]) - self.assertEqual(reopened.database.shard_schema_ids("durable"), ["mesh.durable_log_note.v1"]) - self.assertEqual([entry.sequence for entry in response.entries], [1, 2]) - self.assertEqual(response.entries[0].raw_document.record_key, "note:1") - - store.compact_through("durable", 1) - compacted = reopened.database.build_shard_log_response(shard_id="durable", shard_epoch=4, after_sequence=0) - retained = reopened.database.build_shard_log_response(shard_id="durable", shard_epoch=4, after_sequence=1) - self.assertTrue(compacted.resync_required) - self.assertEqual(compacted.reason, "compacted_history") - self.assertEqual(compacted.compacted_through, 1) - self.assertEqual([entry.sequence for entry in retained.entries], [2]) - - def test_cultnet_shard_replicator_pulls_logs_and_resyncs_from_snapshot(self) -> None: - document = define_database_entry_type( - "mesh.replica_note", - [("body", 0)], - schema_id="mesh.replica_note.v1", - ) - source = CultMesh.create_node(runtime_id="primary") - source.database.register_document(document) - source.database.put_raw_message(document, "note:1", {"body": "one"}, shard_id="notes", shard_epoch=3) - source.database.put_raw_message(document, "note:2", {"body": "two"}, shard_id="notes", shard_epoch=3) - with tempfile.TemporaryDirectory() as tmp: - cursor_path = Path(tmp) / "replica-cursors.msgpack" - server = CultMesh.serve_node(source) - try: - shard = CultNetShardDescriptor( - shard_id="notes", - owner_runtime_id="primary", - epoch=3, - is_primary=False, - schema_ids=("mesh.replica_note.v1",), - primary_endpoints=(f"cultnet://127.0.0.1:{server.port}",), - ) - target = CultMesh.create_node(runtime_id="replica") - target.database.register_document(document) - cursor_store = CultNetFileShardReplicaCursorStore(cursor_path) - replicator = CultNetShardReplicator( - target.database, - CultNetShardReplicatorOptions( - fetcher=CultNetSchemaShardLogFetcher(timeout_seconds=2.0), - snapshot_fetcher=CultNetSchemaShardSnapshotFetcher(timeout_seconds=2.0), - cursor_store=cursor_store, - batch_size=16, - ), - ) - - first_sequence = replicator.pull_once(shard) - restarted_replicator = CultNetShardReplicator( - target.database, - CultNetShardReplicatorOptions( - fetcher=CultNetSchemaShardLogFetcher(timeout_seconds=2.0), - snapshot_fetcher=CultNetSchemaShardSnapshotFetcher(timeout_seconds=2.0), - cursor_store=CultNetFileShardReplicaCursorStore(cursor_path), - batch_size=16, - ), - ) - second_sequence = restarted_replicator.pull_once(shard) - - self.assertEqual(first_sequence, 2) - self.assertEqual(second_sequence, 2) - self.assertEqual(target.get_required(document, "note:1")["body"], "one") - self.assertEqual(target.get_required(document, "note:2")["body"], "two") - cursor = CultNetFileShardReplicaCursorStore(cursor_path).read("notes") - self.assertIsNotNone(cursor) - assert cursor is not None - self.assertEqual(cursor.last_applied_sequence, 2) - self.assertEqual(cursor.shard_epoch, 3) - finally: - server.stop() - - resync_target = CultMesh.create_node(runtime_id="resync-replica") - resync_target.database.register_document(document) - snapshot = source.database.build_snapshot_response(shard_id="notes", shard_epoch=3, shard_log_sequence=2) - - class CompactingFetcher: - def fetch(self, shard: CultNetShardDescriptor, *, after_sequence: int, limit: int | None = None) -> CultNetShardLogResponse: - return CultNetShardLogResponse( - message_id="compact", - shard_id=shard.shard_id, - shard_epoch=shard.epoch, - entries=(), - resync_required=True, - reason="compacted_history", - compacted_through=after_sequence, - ) - - class SnapshotFetcher: - def fetch(self, shard: CultNetShardDescriptor) -> CultNetRawSnapshotResponse: - return snapshot - - resync_replicator = CultNetShardReplicator( - resync_target.database, - CultNetShardReplicatorOptions( - fetcher=CompactingFetcher(), - snapshot_fetcher=SnapshotFetcher(), - cursor_store=CultNetInMemoryShardReplicaCursorStore(), - ), - ) - resync_sequence = resync_replicator.pull_once(CultNetShardDescriptor( - shard_id="notes", - owner_runtime_id="primary", - epoch=3, - is_primary=False, - schema_ids=("mesh.replica_note.v1",), - primary_endpoints=("cultnet://primary:3075",), - )) - - self.assertEqual(resync_sequence, 2) - self.assertEqual(resync_target.get_required(document, "note:1")["body"], "one") - - def test_cultnet_shard_replicator_background_loop_polls_non_primary_shards(self) -> None: - document = define_database_entry_type( - "mesh.background_replica_note", - [("body", 0)], - schema_id="mesh.background_replica_note.v1", - ) - source = CultMesh.create_node(runtime_id="background-primary") - source.database.register_document(document) - target = CultMesh.create_node(runtime_id="background-replica") - target.database.register_document(document) - cursor_store = CultNetInMemoryShardReplicaCursorStore() - errors: list[Exception] = [] - - class FlakySourceFetcher: - def __init__(self) -> None: - self.calls = 0 - - def fetch(self, shard: CultNetShardDescriptor, *, after_sequence: int, limit: int | None = None) -> CultNetShardLogResponse: - self.calls += 1 - if self.calls == 1: - raise RuntimeError("transient fetch failure") - return source.database.build_shard_log_response( - shard_id=shard.shard_id, - shard_epoch=shard.epoch, - after_sequence=after_sequence, - limit=limit, - ) - - fetcher = FlakySourceFetcher() - shard = CultNetShardDescriptor( - shard_id="background", - owner_runtime_id="background-primary", - epoch=7, - is_primary=False, - schema_ids=("mesh.background_replica_note.v1",), - primary_endpoints=("cultnet://127.0.0.1:3075",), - ) - replicator = CultNetShardReplicator( - target.database, - CultNetShardReplicatorOptions( - fetcher=fetcher, - cursor_store=cursor_store, - batch_size=1, - poll_interval_seconds=0.01, - on_error=errors.append, - ), - ) - try: - source.database.put_raw_message(document, "note:1", {"body": "one"}, shard_id="background", shard_epoch=7) - replicator.start([shard]) - self._wait_until(lambda: target.database.get(document, "note:1") is not None) - source.database.put_raw_message(document, "note:2", {"body": "two"}, shard_id="background", shard_epoch=7) - self._wait_until(lambda: target.database.get(document, "note:2") is not None) - finally: - replicator.stop() - - cursor = cursor_store.read("background") - self.assertEqual(target.get_required(document, "note:1")["body"], "one") - self.assertEqual(target.get_required(document, "note:2")["body"], "two") - self.assertIsNotNone(cursor) - assert cursor is not None - self.assertEqual(cursor.last_applied_sequence, 2) - self.assertEqual(cursor.shard_epoch, 7) - self.assertEqual(len(errors), 1) - self.assertEqual(str(errors[0]), "transient fetch failure") - - def test_cultnet_schema_write_forwarder_sends_raw_writes_to_primary(self) -> None: - document = define_database_entry_type( - "mesh.forward_note", - [("body", 0)], - schema_id="mesh.forward_note.v1", - ) - primary = CultMesh.create_node(runtime_id="forward-primary") - primary.database.register_document(document) - server = CultMesh.serve_node(primary) - try: - shard = CultNetShardDescriptor( - shard_id="forward", - owner_runtime_id="forward-primary", - epoch=9, - is_primary=False, - schema_ids=("mesh.forward_note.v1",), - primary_endpoints=(f"cultnet://127.0.0.1:{server.port}",), - ) - forwarder: CultNetShardWriteForwarder = CultNetSchemaWriteForwarder(timeout_seconds=2.0) - put = document_put_raw( - message_id="forward-put", - key="note:forward", - schema_id=document.catalog_entry().schema_id, - stored_at="2026-06-14T00:00:00Z", - payload=document.encode_payload({"body": "forwarded"}), - ).to_wire() - delete = document_delete( - message_id="forward-delete", - schema_id=document.catalog_entry().schema_id, - record_key="note:forward", - ).to_wire() - - forwarder.forward_put(shard, put) - self._eventually(lambda: primary.database.get(document, "note:forward") is not None) - self.assertEqual(primary.database.get_required(document, "note:forward")["body"], "forwarded") - forwarder.forward_delete(shard, delete) - self._eventually(lambda: primary.database.get(document, "note:forward") is None) - - log = primary.database.build_shard_log_response(shard_id="forward", shard_epoch=9) - self.assertIsNone(primary.database.get(document, "note:forward")) - self.assertEqual([entry.change_kind for entry in log.entries], ["added", "removed"]) - self.assertEqual(log.entries[0].put["shardId"], "forward") - self.assertEqual(log.entries[1].delete["shardEpoch"], 9) - finally: - server.stop() - - def _eventually(self, predicate: Callable[[], bool], *, timeout_seconds: float = 2.0) -> None: - deadline = time.monotonic() + timeout_seconds - while time.monotonic() < deadline: - if predicate(): - return - time.sleep(0.01) - self.assertTrue(predicate()) - - def test_cultmesh_database_watchers_observe_name_and_index_changes(self) -> None: - document = define_database_entry_type( - "mesh.named_watch", - [("name", 0), ("kind", 1)], - name="name", - indexes={"kind": "kind"}, - ) - node = CultMesh.create_node(runtime_id="mesh-named-watch") - node.database.register_document(document) - name_changes: list[CultMeshDatabaseChange] = [] - index_changes: list[CultMeshDatabaseChange] = [] - - node.database.watch_by_name(document, "Potion", name_changes.append) - node.database.watch_by_index(document, "kind", "consumable", index_changes.append) - - node.database.put(document, "item:1", {"name": "Potion", "kind": "consumable"}) - node.database.put(document, "item:2", {"name": "Sword", "kind": "weapon"}) - node.database.put(document, "item:1", {"name": "Elixir", "kind": "rare"}) - node.database.delete(document, "item:1") - - self.assertEqual([change.change_kind for change in name_changes], ["added", "updated"]) - self.assertEqual([change.record_key for change in name_changes], ["item:1", "item:1"]) - self.assertEqual(name_changes[1].previous_value, {"name": "Potion", "kind": "consumable"}) - self.assertEqual([change.change_kind for change in index_changes], ["added", "updated"]) - self.assertEqual([change.record_key for change in index_changes], ["item:1", "item:1"]) - self.assertEqual(index_changes[1].value, {"name": "Elixir", "kind": "rare"}) - - def test_cultmesh_database_collection_reads_resolve_same_schema_aliases(self) -> None: - @dataclass - class CanonicalItem: - name: str - kind: str - - @dataclass - class UiItem: - name: str - kind: str - - document = define_database_entry_type( - "mesh.alias_collection_item", - [("name", 0), ("kind", 1)], - cls=CanonicalItem, - name="name", - indexes={"kind": "kind"}, - schema_id="mesh.alias_collection_item.v1", - schema_name="mesh.alias_collection_item", - schema_version="mesh.alias_collection_item.v1", - ) - alias = define_database_entry_type( - "mesh.alias_collection_item.ui", - [("name", 0), ("kind", 1)], - cls=UiItem, - name="name", - indexes={"kind": "kind"}, - schema_id="mesh.alias_collection_item.v1", - schema_name="mesh.alias_collection_item", - schema_version="mesh.alias_collection_item.v1", - ) - node = CultMesh.create_node(runtime_id="mesh-alias-collection") - node.database.register_document(document) - - node.database.put(alias, "item:potion", UiItem("Potion", "consumable")) - node.database.put(document, "item:sword", CanonicalItem("Sword", "weapon")) - - all_items = node.database.get_all(alias) - potion = node.database.get_by_name(alias, "Potion") - sword = node.get_by_index(alias, "kind", "weapon") - - self.assertEqual([item.name for item in all_items], ["Potion", "Sword"]) - self.assertTrue(all(isinstance(item, UiItem) for item in all_items)) - self.assertIsInstance(potion, UiItem) - self.assertEqual(potion.kind, "consumable") - self.assertIsInstance(sword, UiItem) - self.assertEqual(sword.name, "Sword") - self.assertIsNone(node.database.get_by_name(alias, "Missing")) - - def test_cultmesh_database_alias_collection_watches_emit_alias_values(self) -> None: - @dataclass - class CanonicalItem: - name: str - kind: str - - @dataclass - class UiItem: - name: str - kind: str - - document = define_database_entry_type( - "mesh.alias_collection_watch_item", - [("name", 0), ("kind", 1)], - cls=CanonicalItem, - name="name", - indexes={"kind": "kind"}, - schema_id="mesh.alias_collection_watch_item.v1", - schema_name="mesh.alias_collection_watch_item", - schema_version="mesh.alias_collection_watch_item.v1", - ) - alias = define_database_entry_type( - "mesh.alias_collection_watch_item.ui", - [("name", 0), ("kind", 1)], - cls=UiItem, - name="name", - indexes={"kind": "kind"}, - schema_id="mesh.alias_collection_watch_item.v1", - schema_name="mesh.alias_collection_watch_item", - schema_version="mesh.alias_collection_watch_item.v1", - ) - node = CultMesh.create_node(runtime_id="mesh-alias-collection-watch") - node.database.register_document(document) - name_changes: list[CultMeshDatabaseChange] = [] - index_changes: list[CultMeshDatabaseChange] = [] - - node.database.watch_by_name(alias, "Potion", name_changes.append) - node.database.watch_by_index(alias, "kind", "consumable", index_changes.append) - - node.database.put(document, "item:potion", CanonicalItem("Potion", "consumable")) - node.database.put(document, "item:potion", CanonicalItem("Elixir", "rare")) - - self.assertEqual([change.change_kind for change in name_changes], ["added", "updated"]) - self.assertTrue(all(change.document.type == alias.type for change in name_changes)) - self.assertIsInstance(name_changes[0].value, UiItem) - self.assertIsInstance(name_changes[1].value, UiItem) - self.assertIsInstance(name_changes[1].previous_value, UiItem) - self.assertEqual(name_changes[1].previous_value.name, "Potion") - self.assertEqual(name_changes[1].value.name, "Elixir") - self.assertEqual([change.change_kind for change in index_changes], ["added", "updated"]) - self.assertTrue(all(change.document.type == alias.type for change in index_changes)) - self.assertIsInstance(index_changes[0].value, UiItem) - self.assertIsInstance(index_changes[1].previous_value, UiItem) - - def test_cultmesh_database_watch_by_name_and_index_validate_lookup_shape(self) -> None: - document = define_database_entry_type("mesh.unnamed_watch", [("body", 0)]) - node = CultMesh.create_node(runtime_id="mesh-watch-validation") - node.database.register_document(document) - - with self.assertRaises(ValueError): - node.database.watch_by_name(document, "missing", lambda _: None) - with self.assertRaises(ValueError): - node.database.watch_by_index(document, "missing", "value", lambda _: None) - with self.assertRaises(ValueError): - node.database.watch_by_index(document, "", "value", lambda _: None) - with self.assertRaises(ValueError): - node.database.watch_by_index(document, "missing", "", lambda _: None) - - def test_cultmesh_facade_matches_peer_runtime_entrypoints(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - store_path = Path(tmp) / "mesh.cc" - node = CultMesh.start_node(store_path, runtime_id="mesh-facade") - - self.assertEqual(node.runtime_id, "mesh-facade") - self.assertIsInstance(CultMesh.create_node(), type(node)) - self.assertIsInstance(CultMesh.create_verse_catalog(), CultMeshVerseCatalog) - self.assertIsInstance(CultMesh.create_peer_catalog(), CultMeshPeerCatalog) - self.assertIsInstance(CultMesh.create_authority_lease_catalog(), CultMeshAuthorityLeaseCatalog) - self.assertIsInstance(CultMesh.create_stream_catalog(), CultMeshStreamCatalog) - self.assertIsInstance(CultMesh.create_schema_catalog(), CultNetSchemaCatalog) - self.assertIsInstance(CultMesh.create_builtin_schema_catalog(), CultNetSchemaCatalog) - self.assertIsInstance(CultMesh.create_shard_catalog(), CultNetShardCatalog) - builtins = CultMesh.create_builtin_schema_catalog( - include_schema_json=False, - kinds=["shared_contract"], - ) - self.assertEqual( - builtins.list()[0].schema_id, - "https://github.com/GameCult/cultnet-ts/contracts/cultnet.transport-profile.schema.json", - ) - self.assertIsNone(builtins.list()[0].schema_json) - peer_exchange_schema_id = ( - "https://github.com/GameCult/cultnet-ts/contracts/cultmesh.peer-exchange-request.schema.json" - ) - filtered_builtin = CultMesh.create_builtin_schema_catalog( - include_schema_json=True, - schema_ids=[peer_exchange_schema_id], - ) - self.assertEqual( - filtered_builtin.list()[0].schema_version, - "cultmesh.peer_exchange_request.v0", - ) - self.assertIn( - "cultmesh.peer_exchange_request.v0", - filtered_builtin.list()[0].schema_json or "", - ) - shard_catalog = CultMesh.create_shard_catalog() - shard_catalog.upsert( - CultNetShardDescriptor( - shard_id="python-notes-a", - owner_runtime_id="mesh-facade", - epoch=7, - is_primary=True, - schema_ids=("python.note.v1",), - key_prefix="note:", - ) - ) - self.assertEqual( - [ - shard.shard_id - for shard in shard_catalog.list(schema_ids=["python.note.v1"], record_keys=["note:1"]) - ], - ["python-notes-a"], - ) - verse_client = CultMesh.create_verse_discovery_client("127.0.0.1", 4010, timeout_seconds=1.5) - peer_client = CultMesh.create_peer_exchange_client("127.0.0.1", 4011, timeout_seconds=1.5) - raw_client = CultMesh.create_client("127.0.0.1", 4012, timeout_seconds=1.5) - connected_client = CultMesh.connect_client("127.0.0.1", 4013, timeout_seconds=1.5) - self.assertIsInstance(verse_client, CultMeshVerseDiscoveryClient) - self.assertIsInstance(peer_client, CultMeshPeerExchangeClient) - self.assertIsInstance(verse_client, CultMeshDiscoveryClient) - self.assertIsInstance(peer_client, CultMeshDiscoveryClient) - self.assertIsInstance(raw_client, CultNetRawClient) - self.assertIsInstance(connected_client, CultNetRawClient) - self.assertEqual((verse_client.host, verse_client.port, verse_client.timeout_seconds), ("127.0.0.1", 4010, 1.5)) - self.assertEqual((peer_client.host, peer_client.port, peer_client.timeout_seconds), ("127.0.0.1", 4011, 1.5)) - self.assertEqual((raw_client.host, raw_client.port, raw_client.timeout_seconds), ("127.0.0.1", 4012, 1.5)) - self.assertEqual((connected_client.host, connected_client.port, connected_client.timeout_seconds), ("127.0.0.1", 4013, 1.5)) - - def test_cultmesh_simulation_fact_uses_csharp_slot_contract(self) -> None: - import msgpack # type: ignore - - candidate = { - "shardId": "arena", - "shardEpoch": 4, - "frame": 100, - "subjectId": "bob", - "claimKind": "hit", - "claimHash": compute_simulation_claim_hash("hit", "alice", "bob", "frame:100"), - "claimSummary": "alice shot bob first", - "witnessCount": 2, - "supportWeight": 2.0, - "totalWeight": 2.0, - "confidence": 1.0, - "hasQuorum": True, - } - - fact = CultMeshSimulationFact.from_candidate(candidate, committed_at="2026-06-13T00:00:00Z") - payload = simulation_fact_document.encode_payload(fact) - slots = msgpack.unpackb(payload, raw=False) - - self.assertEqual(CultMeshSimulationFact.create_record_key(candidate), f"simulation:{fact.fact_id}") - self.assertEqual(slots[0], fact.fact_id) - self.assertEqual(slots[1], "arena") - self.assertEqual(slots[2], 4) - self.assertEqual(slots[3], 100) - self.assertEqual(slots[4], "bob") - self.assertEqual(slots[5], "hit") - self.assertEqual(slots[6], candidate["claimHash"]) - self.assertEqual(slots[7], "alice shot bob first") - self.assertEqual(slots[8], 2) - self.assertEqual(slots[9], 2.0) - self.assertEqual(slots[10], 2.0) - self.assertEqual(slots[11], 1.0) - self.assertEqual(slots[12], "2026-06-13T00:00:00Z") - - def test_cultmesh_simulation_fact_committer_rejects_without_quorum_and_stores_fact(self) -> None: - candidate = { - "shardId": "arena", - "shardEpoch": 4, - "frame": 100, - "subjectId": "bob", - "claimKind": "hit", - "claimHash": compute_simulation_claim_hash("hit", "alice", "bob", "frame:100"), - "claimSummary": "alice shot bob first", - "witnessCount": 2, - "supportWeight": 2.0, - "totalWeight": 2.0, - "confidence": 1.0, - "hasQuorum": True, - } - - with tempfile.TemporaryDirectory() as tmp: - node = CultMesh.create_node(Path(tmp) / "facts.cc", runtime_id="mesh-facts") - committer = CultMesh.create_simulation_fact_committer(node) - - rejected = dict(candidate) - rejected["hasQuorum"] = False - with self.assertRaisesRegex(ValueError, "before quorum"): - committer.commit(rejected) - - committed = committer.commit(candidate, committed_at="2026-06-13T00:00:00Z") - stored = node.get_required(simulation_fact_document, committed.key) - - self.assertEqual(stored.claim_hash, candidate["claimHash"]) - self.assertEqual(stored.committed_at, "2026-06-13T00:00:00Z") - self.assertEqual(committed.fact.fact_id, stored.fact_id) - - def test_cultnet_simulation_consensus_dedupes_witnesses_and_requires_quorum(self) -> None: - from cultnet_py import CultNetSimulationConsensus - - consensus = CultNetSimulationConsensus( - CultNetSimulationConsensusOptions(minimum_witnesses=2, quorum_ratio=1.0) - ) - claim_hash = compute_simulation_claim_hash("hit", "alice", "bob", "frame:100") - observations = [ - { - "witnessRuntimeId": "watcher-1", - "shardId": "arena", - "shardEpoch": 4, - "frame": 100, - "subjectId": "bob", - "claimKind": "hit", - "claimHash": claim_hash, - "claimSummary": "alice shot bob first", - "weight": 1.0, - }, - { - "witnessRuntimeId": "watcher-1", - "shardId": "arena", - "shardEpoch": 4, - "frame": 100, - "subjectId": "bob", - "claimKind": "hit", - "claimHash": "stale-duplicate", - "weight": 0.5, - }, - { - "witnessRuntimeId": "watcher-2", - "shardId": "arena", - "shardEpoch": 4, - "frame": 100, - "subjectId": "bob", - "claimKind": "hit", - "claimHash": claim_hash, - "weight": 1.0, - }, - ] - - candidates = consensus.build_candidates(observations) - typed_candidates = consensus.build_candidate_objects([ - CultNetSimulationObservation.from_wire(observation) - for observation in observations - ]) - - self.assertEqual(len(candidates), 1) - self.assertEqual(candidates[0]["claimHash"], claim_hash) - self.assertEqual(candidates[0]["witnessCount"], 2) - self.assertEqual(candidates[0]["supportWeight"], 2.0) - self.assertTrue(candidates[0]["hasQuorum"]) - self.assertEqual(len(typed_candidates), 1) - self.assertIsInstance(typed_candidates[0], CultNetSimulationConsensusCandidate) - self.assertEqual(typed_candidates[0].claim_hash, claim_hash) - self.assertEqual(typed_candidates[0].to_wire(), candidates[0]) - self.assertEqual(CultNetSimulationConsensusCandidate.from_wire(candidates[0]), typed_candidates[0]) - - def test_cultmesh_game_session_submits_observations_and_commits_quorum_once(self) -> None: - claim_hash = compute_simulation_claim_hash("hit", "alice", "bob", "frame:100") - first = simulation_observation( - message_id="obs-1", - witness_runtime_id="watcher-1", - shard_id="arena", - shard_epoch=4, - frame=100, - subject_id="bob", - claim_kind="hit", - claim_hash=claim_hash, - claim_summary="alice shot bob first", - ).to_wire() - second = simulation_observation( - message_id="obs-2", - witness_runtime_id="watcher-2", - shard_id="arena", - shard_epoch=4, - frame=100, - subject_id="bob", - claim_kind="hit", - claim_hash=claim_hash, - claim_summary="alice shot bob first", - ).to_wire() - - with tempfile.TemporaryDirectory() as tmp: - node = CultMesh.create_node(Path(tmp) / "session.cc", runtime_id="mesh-session") - session = CultMesh.create_game_session( - node, - CultMeshGameSessionOptions( - consensus_options=CultNetSimulationConsensusOptions(minimum_witnesses=2, quorum_ratio=1.0) - ), - ) - seen_candidates: list[CultNetSimulationConsensusCandidate] = [] - seen_facts: list[CultMeshDatabaseChange] = [] - unsubscribe_candidates = session.watch_candidates(seen_candidates.append) - unsubscribe_facts = session.watch_simulation_facts(seen_facts.append) - - self.assertEqual(session.submit_and_commit(CultNetSimulationObservation.from_wire(first)), []) - typed_candidates = session.submit_observation_candidates(CultNetSimulationObservation.from_wire(second)) - commits = session.commit_quorum_candidate_objects(typed_candidates) - replay = session.commit_quorum_candidates(session.submit_observation(second)) - unsubscribe_candidates() - unsubscribe_facts() - - self.assertEqual(len(typed_candidates), 1) - self.assertTrue(typed_candidates[0].has_quorum) - self.assertEqual(len(commits), 1) - self.assertEqual(replay, []) - self.assertEqual([candidate.claim_hash for candidate in seen_candidates], [claim_hash, claim_hash, claim_hash]) - self.assertEqual(len(seen_facts), 1) - self.assertEqual(seen_facts[0].change_kind, "added") - self.assertEqual(seen_facts[0].value.claim_hash, claim_hash) - stored = node.get_required(simulation_fact_document, commits[0].key) - self.assertEqual(stored.claim_hash, claim_hash) - self.assertEqual(stored.witness_count, 2) - - def test_cultmesh_game_session_serves_catalogs_and_observations_over_wire(self) -> None: - verse_catalog = CultMeshVerseCatalog() - peer_catalog = CultMeshPeerCatalog() - verse_catalog.upsert(CultMeshVerseDescriptor( - verse_id="session-verse", - display_name="Session Verse", - authority_model="local", - compatibility=CultMeshVerseCompatibility("cultmesh.v0", "session-rules"), - )) - peer_catalog.upsert(CultMeshPeerCard( - peer_id="session-peer", - verse_id="session-verse", - endpoints=("cultnet://127.0.0.1:0",), - roles=("simulation-observer", "read-replica"), - shard_ids=("arena",), - )) - session = CultMesh.create_game_session( - CultMesh.create_node(runtime_id="session-peer"), - CultMeshGameSessionOptions( - verse_catalog=verse_catalog, - peer_catalog=peer_catalog, - consensus_options=CultNetSimulationConsensusOptions(minimum_witnesses=2, quorum_ratio=1.0), - ), - ) - server = session.serve(display_name="Session Peer") - try: - raw_client = CultMesh.create_client("127.0.0.1", server.port, timeout_seconds=2.0) - hello_response = raw_client.request( - hello(runtime_id="session-prober"), - expected_schema_version="cultnet.hello.v0", - ) - discovery_client = CultMesh.create_verse_discovery_client("127.0.0.1", server.port, timeout_seconds=2.0) - verses = discovery_client.fetch_verses(transport_version="cultmesh.v0") - peers = discovery_client.fetch_peers(verse_id="session-verse", roles=["simulation-observer"]) - claim_hash = compute_simulation_claim_hash("hit", "alice", "bob", "frame:100") - first = raw_client.request( - simulation_observation( - message_id="session-wire-1", - witness_runtime_id="watcher-1", - shard_id="arena", - shard_epoch=4, - frame=100, - subject_id="bob", - claim_kind="hit", - claim_hash=claim_hash, - claim_summary="alice shot bob first", - ), - expected_schema_version="cultnet.simulation_consensus_candidate.v0", - ) - second = raw_client.request( - simulation_observation( - message_id="session-wire-2", - witness_runtime_id="watcher-2", - shard_id="arena", - shard_epoch=4, - frame=100, - subject_id="bob", - claim_kind="hit", - claim_hash=claim_hash, - claim_summary="alice shot bob first", - ), - expected_schema_version="cultnet.simulation_consensus_candidate.v0", - ) - finally: - server.stop() - - self.assertIn("cultnet.simulation_observation.v0", hello_response["supportedMessageVersions"]) - self.assertEqual(verses[0].verse_id, "session-verse") - self.assertEqual(peers[0].peer_id, "session-peer") - self.assertIn("simulation-observer", peers[0].roles) - self.assertFalse(first["hasQuorum"]) - self.assertEqual(first["witnessCount"], 1) - self.assertTrue(second["hasQuorum"]) - self.assertEqual(second["witnessCount"], 2) - - def test_cultmesh_game_session_can_disable_served_simulation_observations(self) -> None: - verse_catalog = CultMeshVerseCatalog() - verse_catalog.upsert(CultMeshVerseDescriptor( - verse_id="session-quiet", - display_name="Quiet Session", - authority_model="local", - compatibility=CultMeshVerseCompatibility("cultmesh.v0", "quiet-rules"), - )) - session = CultMesh.create_game_session( - CultMesh.create_node(runtime_id="quiet-session-peer"), - CultMeshGameSessionOptions( - verse_catalog=verse_catalog, - serve_simulation_observations=False, - ), - ) - server = session.serve(display_name="Quiet Session Peer") - try: - raw_client = CultMesh.create_client("127.0.0.1", server.port, timeout_seconds=2.0) - hello_response = raw_client.request( - hello(runtime_id="quiet-session-prober"), - expected_schema_version="cultnet.hello.v0", - ) - discovery_client = CultMesh.create_verse_discovery_client("127.0.0.1", server.port, timeout_seconds=2.0) - verses = discovery_client.fetch_verses(transport_version="cultmesh.v0") - with self.assertRaisesRegex(CultNetPeerError, "Simulation observations are not enabled") as raised: - raw_client.request( - simulation_observation( - message_id="quiet-session-sim", - witness_runtime_id="watcher-1", - shard_id="arena", - shard_epoch=4, - frame=100, - subject_id="bob", - claim_kind="hit", - claim_hash=compute_simulation_claim_hash("hit", "alice", "bob", "frame:100"), - ), - expected_schema_version="cultnet.simulation_consensus_candidate.v0", - ) - finally: - server.stop() - - self.assertNotIn("cultnet.simulation_observation.v0", hello_response["supportedMessageVersions"]) - self.assertEqual(verses[0].verse_id, "session-quiet") - self.assertEqual(raised.exception.response["code"], "simulation_observations_disabled") - - def test_cultmesh_game_session_prediction_requires_scope_and_reconciles_shard_log(self) -> None: - note_doc = define_database_entry_type( - "mesh.input", - [("body", 0)], - schema_id="mesh.input.v1", - ) - with tempfile.TemporaryDirectory() as tmp: - node = CultMesh.create_node(Path(tmp) / "prediction.cc", runtime_id="client-a") - node.register_document(note_doc) - session = CultMesh.create_game_session( - node, - CultMeshGameSessionOptions( - client_authority_scopes=( - CultNetClientAuthorityScope( - "client-a", - schema_ids=("mesh.input.v1",), - key_prefix="input:client-a", - ), - ) - ), - ) - - with self.assertRaisesRegex(ValueError, "does not have client prediction authority"): - session.predict(note_doc, "input:other", {"body": "nope"}) - - prediction = session.predict(note_doc, "input:client-a:move", {"body": "predicted"}) - put = document_put_raw( - message_id="authoritative", - key=prediction.key, - schema_id=prediction.schema_id, - stored_at="2026-06-13T00:00:00Z", - payload=note_doc.encode_payload({"body": "authoritative"}), - shard_id="inputs", - shard_epoch=1, - ) - authoritative_response = { - "schemaVersion": "cultnet.shard_log_response.v0", - "messageId": "inputs-log", - "shardId": "inputs", - "shardEpoch": 1, - "entries": [ - { - "sequence": 1, - "committedAt": "2026-06-13T00:00:00Z", - "changeKind": "updated", - "put": put.to_wire(), - } - ], - "resyncRequired": False, - } - changes = session.apply_shard_log_response(CultNetShardLogResponse.from_wire(authoritative_response)) - - self.assertEqual(changes[0].change_kind, "reconciled") - self.assertEqual(node.get_required(note_doc, prediction.key)["body"], "authoritative") - self.assertEqual(session.pending_predictions(), ()) - self.assertEqual(session.resimulation_inputs(), ()) - - def test_cultmesh_game_session_partial_reconciliation_keeps_unmatched_predictions(self) -> None: - note_doc = define_database_entry_type( - "mesh.partial_input", - [("body", 0)], - schema_id="mesh.partial_input.v1", - ) - node = CultMesh.create_node(runtime_id="client-partial") - node.register_document(note_doc) - session = CultMesh.create_game_session( - node, - CultMeshGameSessionOptions( - client_authority_scopes=( - CultNetClientAuthorityScope( - "client-partial", - schema_ids=("mesh.partial_input.v1",), - key_prefix="input:client-partial", - ), - ) - ), - ) - pending = session.predict(note_doc, "input:client-partial:pending", {"body": "pending-local"}) - accepted = session.predict(note_doc, "input:client-partial:accepted", {"body": "accepted-local"}) - put = document_put_raw( - message_id="authoritative-partial", - key=accepted.key, - schema_id=accepted.schema_id, - stored_at="2026-06-13T00:00:00Z", - payload=note_doc.encode_payload({"body": "accepted-authority"}), - shard_id="inputs", - shard_epoch=1, - ) - - changes = session.apply_shard_log_response({ - "schemaVersion": "cultnet.shard_log_response.v0", - "messageId": "inputs-partial-log", - "shardId": "inputs", - "shardEpoch": 1, - "entries": [ - { - "sequence": 1, - "committedAt": "2026-06-13T00:00:00Z", - "changeKind": "updated", - "put": put.to_wire(), - } - ], - "resyncRequired": False, - }) - - self.assertEqual([(change.change_kind, change.record_key) for change in changes], [ - ("reconciled", accepted.key), - ]) - self.assertEqual(node.get_required(note_doc, accepted.key)["body"], "accepted-authority") - self.assertEqual(node.get_required(note_doc, pending.key)["body"], "pending-local") - self.assertEqual([prediction.key for prediction in session.pending_predictions()], [pending.key]) - self.assertEqual([prediction.key for prediction in session.resimulation_inputs()], [pending.key]) - - def test_cultmesh_game_session_rolls_back_pending_predictions(self) -> None: - note_doc = define_database_entry_type( - "mesh.rollback_input", - [("body", 0)], - schema_id="mesh.rollback_input.v1", - ) - node = CultMesh.create_node(runtime_id="client-rollback") - node.register_document(note_doc) - node.put(note_doc, "input:client-rollback:existing", {"body": "previous"}) - session = CultMesh.create_game_session( - node, - CultMeshGameSessionOptions( - client_authority_scopes=( - CultNetClientAuthorityScope( - "client-rollback", - schema_ids=("mesh.rollback_input.v1",), - key_prefix="input:client-rollback", - ), - ) - ), - ) - - existing = session.predict(note_doc, "input:client-rollback:existing", {"body": "predicted-existing"}) - created = session.predict(note_doc, "input:client-rollback:new", {"body": "predicted-new"}) - self.assertEqual( - [prediction.key for prediction in session.pending_predictions()], - ["input:client-rollback:existing", "input:client-rollback:new"], - ) - self.assertEqual([prediction.key for prediction in session.resimulation_inputs()], [ - "input:client-rollback:existing", - "input:client-rollback:new", - ]) - - restored = session.rollback_prediction(existing) - self.assertIsNotNone(restored) - assert restored is not None - self.assertEqual(restored.change_kind, "rolled_back") - self.assertEqual(restored.value, {"body": "previous"}) - self.assertEqual(node.get_required(note_doc, "input:client-rollback:existing")["body"], "previous") - - rolled_back = session.rollback_predictions() - self.assertEqual([(change.change_kind, change.record_key, change.value) for change in rolled_back], [ - ("rolled_back", created.key, None), - ]) - self.assertIsNone(node.get(note_doc, created.key)) - self.assertEqual(session.pending_predictions(), ()) - - def test_cultmesh_game_session_authoritative_delete_rolls_back_prediction(self) -> None: - note_doc = define_database_entry_type( - "mesh.deleted_input", - [("body", 0)], - schema_id="mesh.deleted_input.v1", - ) - node = CultMesh.create_node(runtime_id="client-delete") - node.register_document(note_doc) - session = CultMesh.create_game_session( - node, - CultMeshGameSessionOptions( - client_authority_scopes=( - CultNetClientAuthorityScope( - "client-delete", - schema_ids=("mesh.deleted_input.v1",), - key_prefix="input:client-delete", - ), - ) - ), - ) - - prediction = session.predict(note_doc, "input:client-delete:move", {"body": "predicted"}) - delete = document_delete( - message_id="authoritative-delete", - schema_id=prediction.schema_id, - record_key=prediction.key, - shard_id="inputs", - shard_epoch=1, - ) - changes = session.apply_shard_log_response({ - "schemaVersion": "cultnet.shard_log_response.v0", - "messageId": "inputs-delete-log", - "shardId": "inputs", - "shardEpoch": 1, - "entries": [ - { - "sequence": 1, - "committedAt": "2026-06-13T00:00:00Z", - "changeKind": "removed", - "delete": delete.to_wire(), - } - ], - "resyncRequired": False, - }) - - self.assertEqual(changes[0].change_kind, "rolled_back") - self.assertIsNone(node.get(note_doc, prediction.key)) - self.assertEqual(session.pending_predictions(), ()) - - def test_cultmesh_database_watchers_observe_authoritative_shard_log_reconciliation(self) -> None: - note_doc = define_database_entry_type( - "mesh.watched_input", - [("body", 0)], - schema_id="mesh.watched_input.v1", - ) - node = CultMesh.create_node(runtime_id="client-watch") - node.database.register_document(note_doc) - seen: list[CultMeshDatabaseChange] = [] - node.database.watch_record(note_doc, "input:client-watch:move", seen.append) - session = CultMesh.create_game_session( - node, - CultMeshGameSessionOptions( - client_authority_scopes=( - CultNetClientAuthorityScope( - "client-watch", - schema_ids=("mesh.watched_input.v1",), - key_prefix="input:client-watch", - ), - ) - ), - ) - - prediction = session.predict(note_doc, "input:client-watch:move", {"body": "predicted"}) - put = document_put_raw( - message_id="authoritative-watch", - key=prediction.key, - schema_id=prediction.schema_id, - stored_at="2026-06-13T00:00:00Z", - payload=note_doc.encode_payload({"body": "authoritative"}), - shard_id="inputs", - shard_epoch=1, - ) - session_changes = session.apply_shard_log_response({ - "schemaVersion": "cultnet.shard_log_response.v0", - "messageId": "inputs-watch-log", - "shardId": "inputs", - "shardEpoch": 1, - "entries": [ - { - "sequence": 1, - "committedAt": "2026-06-13T00:00:00Z", - "changeKind": "updated", - "put": put.to_wire(), - } - ], - "resyncRequired": False, - }) - - self.assertEqual([change.change_kind for change in seen], ["added", "updated"]) - self.assertEqual(seen[1].value, {"body": "authoritative"}) - self.assertEqual(seen[1].previous_value, {"body": "predicted"}) - self.assertEqual(session_changes[0].change_kind, "reconciled") - - def test_cultmesh_verse_catalog_response_matches_schema_v0_wire_shape(self) -> None: - import msgpack # type: ignore - - catalog = CultMeshVerseCatalog() - catalog.upsert( - CultMeshVerseDescriptor( - verse_id="aetheria-main", - display_name="Aetheria", - authority_model="OperatorCluster", - compatibility=CultMeshVerseCompatibility( - transport_version="cultmesh.v0", - rules_hash="rules", - compatible_verse_ids=("aetheria-modded",), - required_plugin_ids=("core",), - optional_plugin_ids=("skylands",), - ), - discovery_endpoints=("cultmesh://aetheria.example.test:3075",), - authority_runtime_ids=("runtime-a",), - description="main branch", - ) - ) - - response = catalog.create_response(verse_catalog_request("verses-1")) - decoded = msgpack.unpackb(msgpack.packb(response, use_bin_type=True), raw=False) - self.assertEqual(decoded["schemaVersion"], "cultmesh.verse_catalog_response.v0") - self.assertEqual(decoded["messageId"], "verses-1") - self.assertEqual(decoded["verses"][0]["verseId"], "aetheria-main") - self.assertEqual(decoded["verses"][0]["compatibility"]["requiredPluginIds"], ["core"]) - - def test_cultmesh_verse_catalog_watches_updates_and_finds_transfer_targets(self) -> None: - source = CultMeshVerseDescriptor( - verse_id="aetheria-main", - display_name="Aetheria", - authority_model="federated", - compatibility=CultMeshVerseCompatibility( - transport_version="cultmesh.v0", - rules_hash="rules-main", - ), - ) - compatible = CultMeshVerseDescriptor( - verse_id="aetheria-modded", - display_name="Aetheria Modded", - authority_model="federated", - compatibility=CultMeshVerseCompatibility( - transport_version="cultmesh.v0", - rules_hash="rules-modded", - compatible_verse_ids=("aetheria-main",), - ), - ) - incompatible = CultMeshVerseDescriptor( - verse_id="old-world", - display_name="Old World", - authority_model="solo", - compatibility=CultMeshVerseCompatibility( - transport_version="cultmesh.v0", - rules_hash="rules-old", - ), - ) - catalog = CultMeshVerseCatalog() - seen: list[str] = [] - unsubscribe = catalog.watch(lambda verse: seen.append(verse.verse_id)) - - catalog.upsert(source) - catalog.apply_response({ - "schemaVersion": "cultmesh.verse_catalog_response.v0", - "messageId": "verses-watch", - "verses": [compatible.to_wire(), incompatible.to_wire()], - }) - unsubscribe() - catalog.upsert(CultMeshVerseDescriptor( - verse_id="after-unsubscribe", - display_name="After", - authority_model="none", - compatibility=CultMeshVerseCompatibility("cultmesh.v0", "rules-after"), - )) - - self.assertEqual(seen, ["aetheria-main", "aetheria-modded", "old-world"]) - self.assertEqual([verse.verse_id for verse in catalog.verses], ["aetheria-main", "aetheria-modded", "after-unsubscribe", "old-world"]) - self.assertEqual(catalog.get("aetheria-modded"), compatible) - self.assertEqual([verse.verse_id for verse in catalog.find_transfer_targets(source)], ["aetheria-modded"]) - - def test_cultmesh_peer_exchange_response_matches_schema_v0_wire_shape(self) -> None: - import msgpack # type: ignore - - catalog = CultMeshPeerCatalog() - catalog.upsert( - CultMeshPeerCard( - peer_id="peer-a", - verse_id="aetheria-main", - endpoints=("cultnet://peer-a.example.test:3075",), - roles=("discovery", "read-replica"), - shard_ids=("players",), - region="eu-west", - authority_lease_id="lease-1", - expires_at="2026-05-20T12:00:00.0000000Z", - signature="sig", - ) - ) - - response = catalog.create_response( - peer_exchange_request("pex-1", verse_id="aetheria-main", roles=["read-replica"]) - ) - decoded = msgpack.unpackb(msgpack.packb(response, use_bin_type=True), raw=False) - self.assertEqual(decoded["schemaVersion"], "cultmesh.peer_exchange_response.v0") - self.assertEqual(decoded["messageId"], "pex-1") - self.assertEqual(decoded["peers"][0]["peerId"], "peer-a") - self.assertIn("read-replica", decoded["peers"][0]["roles"]) - self.assertEqual(decoded["peers"][0]["authorityLeaseId"], "lease-1") - - def test_cultmesh_peer_catalog_watches_updates_and_gets_peers(self) -> None: - first = CultMeshPeerCard( - peer_id="peer-a", - verse_id="aetheria-main", - endpoints=("cultnet://peer-a.example.test:3075",), - roles=("discovery", "read-replica"), - ) - second = CultMeshPeerCard( - peer_id="peer-b", - verse_id="aetheria-main", - endpoints=("cultnet://peer-b.example.test:3075",), - roles=("shard-primary",), - ) - catalog = CultMeshPeerCatalog() - seen: list[str] = [] - unsubscribe = catalog.watch(lambda peer: seen.append(peer.peer_id)) - - catalog.upsert(first) - catalog.apply_response({ - "schemaVersion": "cultmesh.peer_exchange_response.v0", - "messageId": "peers-watch", - "peers": [second.to_wire()], - }) - unsubscribe() - catalog.upsert(CultMeshPeerCard( - peer_id="peer-c", - verse_id="aetheria-main", - endpoints=("cultnet://peer-c.example.test:3075",), - roles=("read-replica",), - )) - - self.assertEqual(seen, ["peer-a", "peer-b"]) - self.assertTrue(first.has_role("read-replica")) - self.assertEqual([peer.peer_id for peer in catalog.peers], ["peer-a", "peer-b", "peer-c"]) - self.assertEqual(catalog.get("peer-b"), second) - self.assertEqual([peer.peer_id for peer in catalog.find("aetheria-main", role="read-replica")], ["peer-a", "peer-c"]) - - def test_cultmesh_discovery_client_fetches_typed_catalogs_over_cultnet_frames(self) -> None: - import msgpack # type: ignore - from cultnet_py import read_frame, write_frame - - verses = CultMeshVerseCatalog() - verses.upsert( - CultMeshVerseDescriptor( - verse_id="aetheria-main", - display_name="Aetheria", - authority_model="federated", - compatibility=CultMeshVerseCompatibility( - transport_version="cultmesh.v0", - rules_hash="rules-1", - required_plugin_ids=("core",), - ), - discovery_endpoints=("cultmesh://aetheria.example.test:3075",), - authority_runtime_ids=("runtime-a",), - ) - ) - peers = CultMeshPeerCatalog() - peers.upsert( - CultMeshPeerCard( - peer_id="peer-a", - verse_id="aetheria-main", - endpoints=("cultnet://peer-a.example.test:3075",), - roles=("read-replica",), - shard_ids=("players",), - ) - ) - - ready = threading.Event() - server_error: list[BaseException] = [] - - def serve_requests() -> None: - try: - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as server: - server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - server.bind(("127.0.0.1", 0)) - port_holder.append(server.getsockname()[1]) - server.listen(8) - ready.set() - for _ in range(8): - connection, _ = server.accept() - with connection: - stream = connection.makefile("rwb") - request = msgpack.unpackb(read_frame(stream), raw=False) - if request["schemaVersion"] == "cultmesh.verse_catalog_request.v0": - response = verses.create_response(request) - elif request["schemaVersion"] == "cultmesh.peer_exchange_request.v0": - response = peers.create_response(request) - else: - raise AssertionError(f"unexpected request {request['schemaVersion']}") - write_frame(stream, msgpack.packb(response, use_bin_type=True)) - stream.flush() - except BaseException as error: - server_error.append(error) - ready.set() - - port_holder: list[int] = [] - thread = threading.Thread(target=serve_requests, daemon=True) - thread.start() - self.assertTrue(ready.wait(2.0)) - self.assertFalse(server_error) - - client = CultMeshDiscoveryClient("127.0.0.1", port_holder[0], timeout_seconds=2.0) - fetched_verses = client.fetch_verses(transport_version="cultmesh.v0") - fetched_peers = client.fetch_peers(verse_id="aetheria-main", roles=["read-replica"]) - local_verses = CultMeshVerseCatalog() - local_peers = CultMeshPeerCatalog() - synced_verses = client.sync_verse_catalog(local_verses, transport_version="cultmesh.v0") - synced_peers = client.sync_peer_catalog(local_peers, verse_id="aetheria-main", roles=["read-replica"]) - discovered_verses = CultMeshVerseCatalog() - discovered_peers = CultMeshPeerCatalog() - endpoint = f"cultnet://127.0.0.1:{port_holder[0]}" - verse_discovery = CultMeshVerseDiscoveryClient.from_endpoint(endpoint, timeout_seconds=2.0) - peer_discovery = CultMeshPeerExchangeClient.from_endpoint(endpoint, timeout_seconds=2.0) - discovered_verse_count = verse_discovery.discover( - discovered_verses, - endpoints=[endpoint, endpoint, ""], - transport_version="cultmesh.v0", - ) - discovered_peer_count = peer_discovery.discover( - discovered_peers, - verse_id="aetheria-main", - endpoints=[endpoint, endpoint, ""], - roles=["read-replica"], - ) - fanout_verses = CultMeshVerseCatalog() - fanout_verses.upsert( - CultMeshVerseDescriptor( - verse_id="aetheria-main", - display_name="Known Aetheria", - authority_model="federated", - compatibility=CultMeshVerseCompatibility("cultmesh.v0", "rules-1"), - discovery_endpoints=(endpoint, endpoint, ""), - ) - ) - fanout_peers = CultMeshPeerCatalog() - fanout_peers.upsert( - CultMeshPeerCard( - peer_id="known-peer", - verse_id="aetheria-main", - endpoints=(endpoint, endpoint, ""), - roles=("read-replica",), - ) - ) - fanout_verse_count = verse_discovery.fanout(fanout_verses, transport_version="cultmesh.v0") - fanout_peer_count = peer_discovery.fanout( - fanout_peers, - verse_id="aetheria-main", - roles=["read-replica"], - ) - - thread.join(2.0) - self.assertFalse(server_error) - self.assertEqual(fetched_verses[0].verse_id, "aetheria-main") - self.assertEqual(fetched_verses[0].compatibility.required_plugin_ids, ("core",)) - self.assertEqual(fetched_peers[0].peer_id, "peer-a") - self.assertEqual(fetched_peers[0].shard_ids, ("players",)) - self.assertEqual(synced_verses[0].verse_id, "aetheria-main") - self.assertEqual(local_peers.find("aetheria-main", role="read-replica")[0].peer_id, "peer-a") - self.assertEqual(synced_peers[0].roles, ("read-replica",)) - self.assertEqual(discovered_verse_count, 1) - self.assertEqual(discovered_peer_count, 1) - self.assertEqual(discovered_verses.get("aetheria-main").display_name, "Aetheria") - self.assertEqual(discovered_peers.find("aetheria-main", role="read-replica")[0].peer_id, "peer-a") - self.assertEqual(fanout_verse_count, 1) - self.assertEqual(fanout_peer_count, 1) - self.assertEqual(fanout_verses.get("aetheria-main").display_name, "Aetheria") - self.assertIn( - "peer-a", - {peer.peer_id for peer in fanout_peers.find("aetheria-main", role="read-replica")}, - ) - - def test_cultmesh_discovery_client_fans_out_simulation_observations_to_peers(self) -> None: - first_hub = CultNetSimulationObservationHub( - CultNetSimulationConsensusOptions(minimum_witnesses=1, quorum_ratio=1.0) - ) - second_hub = CultNetSimulationObservationHub( - CultNetSimulationConsensusOptions(minimum_witnesses=1, quorum_ratio=1.0) - ) - first_server = CultMesh.serve_node( - CultMesh.create_node(runtime_id="sim-peer-a"), - observation_hub=first_hub, - ) - second_server = CultMesh.serve_node( - CultMesh.create_node(runtime_id="sim-peer-b"), - observation_hub=second_hub, - ) - try: - peers = CultMeshPeerCatalog() - peers.upsert(CultMeshPeerCard( - peer_id="sim-peer-a", - verse_id="arena", - endpoints=(f"cultnet://127.0.0.1:{first_server.port}",), - roles=("simulation-witness",), - )) - peers.upsert(CultMeshPeerCard( - peer_id="sim-peer-b", - verse_id="arena", - endpoints=(f"cultnet://127.0.0.1:{second_server.port}",), - roles=("simulation-witness",), - )) - claim_hash = compute_simulation_claim_hash("hit", "alice", "bob", "frame:200") - message = simulation_observation( - message_id="fanout-observation", - witness_runtime_id="python-fanout", - shard_id="arena", - shard_epoch=1, - frame=200, - subject_id="bob", - claim_kind="hit", - claim_hash=claim_hash, - claim_summary="alice hit bob", - ).to_wire() - - client = CultMeshDiscoveryClient("127.0.0.1", first_server.port, timeout_seconds=2.0) - candidates = client.fanout_simulation_observation( - peers, - message, - verse_id="arena", - roles=["simulation-witness"], - ) - finally: - first_server.stop() - second_server.stop() - - self.assertEqual([candidate["schemaVersion"] for candidate in candidates], [ - "cultnet.simulation_consensus_candidate.v0", - "cultnet.simulation_consensus_candidate.v0", - ]) - self.assertEqual({candidate["claimHash"] for candidate in candidates}, {claim_hash}) - self.assertEqual({candidate["messageId"] for candidate in candidates}, {"fanout-observation"}) - self.assertEqual({candidate["hasQuorum"] for candidate in candidates}, {True}) - - def test_cultmesh_simulation_observation_fanout_flushes_queued_messages(self) -> None: - hub = CultNetSimulationObservationHub( - CultNetSimulationConsensusOptions(minimum_witnesses=1, quorum_ratio=1.0) - ) - server = CultMesh.serve_node(CultMesh.create_node(runtime_id="sim-flush-peer"), observation_hub=hub) - try: - peers = CultMeshPeerCatalog() - peers.upsert(CultMeshPeerCard( - peer_id="sim-flush-peer", - verse_id="arena", - endpoints=(f"cultnet://127.0.0.1:{server.port}",), - roles=("simulation-witness",), - )) - claim_hash = compute_simulation_claim_hash("hit", "alice", "bob", "frame:210") - seen_candidates: list[dict[str, object]] = [] - fanout = CultMeshSimulationObservationFanout( - CultMeshDiscoveryClient("127.0.0.1", server.port, timeout_seconds=2.0), - peers, - verse_id="arena", - roles=["simulation-witness"], - on_candidate=seen_candidates.append, - ) - fanout.enqueue(simulation_observation( - message_id="queued-observation", - witness_runtime_id="python-fanout", - shard_id="arena", - shard_epoch=1, - frame=210, - subject_id="bob", - claim_kind="hit", - claim_hash=claim_hash, - claim_summary="alice hit bob", - ).to_wire()) - - candidates = fanout.flush() - finally: - server.stop() - - self.assertEqual(fanout.pending_count(), 0) - self.assertEqual([candidate["claimHash"] for candidate in candidates], [claim_hash]) - self.assertEqual(seen_candidates[0]["messageId"], "queued-observation") - - def test_cultmesh_simulation_observation_fanout_keeps_failed_messages_pending(self) -> None: - closed_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - closed_socket.bind(("127.0.0.1", 0)) - closed_port = closed_socket.getsockname()[1] - closed_socket.close() - peers = CultMeshPeerCatalog() - peers.upsert(CultMeshPeerCard( - peer_id="closed-simulation-peer", - verse_id="arena", - endpoints=(f"cultnet://127.0.0.1:{closed_port}",), - roles=("simulation-witness",), - )) - errors: list[tuple[str, str]] = [] - fanout = CultMeshSimulationObservationFanout( - CultMeshDiscoveryClient("127.0.0.1", closed_port, timeout_seconds=0.2), - peers, - verse_id="arena", - roles=["simulation-witness"], - on_error=lambda endpoint, error: errors.append((endpoint, str(error))), - ) - fanout.enqueue(simulation_observation( - message_id="retry-observation", - witness_runtime_id="python-fanout", - shard_id="arena", - shard_epoch=1, - frame=212, - subject_id="bob", - claim_kind="miss", - claim_hash=compute_simulation_claim_hash("miss", "alice", "bob", "frame:212"), - claim_summary="alice missed bob", - ).to_wire()) - - candidates = fanout.flush() - - self.assertEqual(candidates, []) - self.assertEqual(fanout.pending_count(), 1) - self.assertEqual(len(errors), 1) - self.assertIn(f"127.0.0.1:{closed_port}", errors[0][0]) - - def test_cultmesh_simulation_observation_fanout_runs_background_loop(self) -> None: - hub = CultNetSimulationObservationHub( - CultNetSimulationConsensusOptions(minimum_witnesses=1, quorum_ratio=1.0) - ) - server = CultMesh.serve_node(CultMesh.create_node(runtime_id="sim-loop-peer"), observation_hub=hub) - try: - peers = CultMeshPeerCatalog() - peers.upsert(CultMeshPeerCard( - peer_id="sim-loop-peer", - verse_id="arena", - endpoints=(f"cultnet://127.0.0.1:{server.port}",), - roles=("simulation-witness",), - )) - claim_hash = compute_simulation_claim_hash("block", "dana", "eve", "frame:211") - seen_candidates: list[dict[str, object]] = [] - fanout = CultMeshSimulationObservationFanout( - CultMeshDiscoveryClient("127.0.0.1", server.port, timeout_seconds=2.0), - peers, - verse_id="arena", - roles=["simulation-witness"], - interval_seconds=0.05, - on_candidate=seen_candidates.append, - ) - fanout.start() - fanout.enqueue(simulation_observation( - message_id="loop-observation", - witness_runtime_id="python-fanout", - shard_id="arena", - shard_epoch=1, - frame=211, - subject_id="eve", - claim_kind="block", - claim_hash=claim_hash, - claim_summary="dana blocked eve", - ).to_wire()) - deadline = time.monotonic() + 2.0 - while not seen_candidates and time.monotonic() < deadline: - time.sleep(0.01) - fanout.stop() - finally: - server.stop() - - self.assertEqual(fanout.pending_count(), 0) - self.assertTrue(seen_candidates) - self.assertEqual(seen_candidates[0]["claimHash"], claim_hash) - self.assertEqual(seen_candidates[0]["messageId"], "loop-observation") - - def test_cultmesh_discovery_client_fans_out_snapshots_to_peers(self) -> None: - document = define_database_entry_type( - "mesh.snapshot_fanout_note", - [("body", 0)], - schema_id="mesh.snapshot_fanout_note.v1", - ) - first = CultMesh.create_node(runtime_id="snapshot-peer-a") - first.database.register_document(document) - first.database.put_raw_message(document, "note:a", {"body": "from-a"}, shard_id="notes", shard_epoch=1) - second = CultMesh.create_node(runtime_id="snapshot-peer-b") - second.database.register_document(document) - second.database.put_raw_message(document, "note:b", {"body": "from-b"}, shard_id="notes", shard_epoch=1) - target = CultMesh.create_node(runtime_id="snapshot-target") - target.database.register_document(document) - first_server = CultMesh.serve_node(first) - second_server = CultMesh.serve_node(second) - try: - peers = CultMeshPeerCatalog() - peers.upsert(CultMeshPeerCard( - peer_id="snapshot-peer-a", - verse_id="mesh", - endpoints=(f"cultnet://127.0.0.1:{first_server.port}", f"cultnet://127.0.0.1:{first_server.port}"), - roles=("read-replica",), - )) - peers.upsert(CultMeshPeerCard( - peer_id="snapshot-peer-b", - verse_id="mesh", - endpoints=(f"cultnet://127.0.0.1:{second_server.port}",), - roles=("read-replica",), - )) - - client = CultMeshDiscoveryClient("127.0.0.1", first_server.port, timeout_seconds=2.0) - responses = client.fanout_snapshot_responses( - peers, - verse_id="mesh", - roles=["read-replica"], - schema_ids=["mesh.snapshot_fanout_note.v1"], - ) - applied = client.sync_snapshots( - target.database, - peers, - verse_id="mesh", - roles=["read-replica"], - schema_ids=["mesh.snapshot_fanout_note.v1"], - ) - finally: - first_server.stop() - second_server.stop() - - self.assertEqual(len(responses), 2) - self.assertTrue(all(isinstance(response, CultNetRawSnapshotResponse) for response in responses)) - self.assertEqual( - sorted(record.record_key for response in responses for record in response.documents), - ["note:a", "note:b"], - ) - self.assertEqual(sorted(record.record_key for record in applied), ["note:a", "note:b"]) - self.assertEqual(target.database.get_required(document, "note:a")["body"], "from-a") - self.assertEqual(target.database.get_required(document, "note:b")["body"], "from-b") - - def test_cultmesh_discovery_client_sync_documents_returns_requested_aliases(self) -> None: - @dataclass - class CanonicalNote: - body: str - - @dataclass - class UiNote: - body: str - - document = define_database_entry_type( - "mesh.snapshot_fanout_alias_note", - [("body", 0)], - cls=CanonicalNote, - schema_id="mesh.snapshot_fanout_alias_note.v1", - schema_name="mesh.snapshot_fanout_alias_note", - schema_version="mesh.snapshot_fanout_alias_note.v1", - ) - alias = define_database_entry_type( - "mesh.snapshot_fanout_alias_note.ui", - [("body", 0)], - cls=UiNote, - schema_id="mesh.snapshot_fanout_alias_note.v1", - schema_name="mesh.snapshot_fanout_alias_note", - schema_version="mesh.snapshot_fanout_alias_note.v1", - ) - source = CultMesh.create_node(runtime_id="snapshot-alias-source") - source.database.register_document(document) - source.database.put_raw_message(document, "note:alias", CanonicalNote("from-source")) - target = CultMesh.create_node(runtime_id="snapshot-alias-target") - target.database.register_document(document) - server = CultMesh.serve_node(source) - try: - peers = CultMeshPeerCatalog() - peers.upsert(CultMeshPeerCard( - peer_id="snapshot-alias-source", - verse_id="mesh", - endpoints=(f"cultnet://127.0.0.1:{server.port}",), - roles=("read-replica",), - )) - - values = CultMeshDiscoveryClient("127.0.0.1", server.port, timeout_seconds=2.0).sync_documents( - target.database, - peers, - verse_id="mesh", - roles=["read-replica"], - documents=[(alias, "note:alias")], - ) - finally: - server.stop() - - self.assertEqual(len(values), 1) - self.assertIsInstance(values[0], UiNote) - self.assertEqual(values[0].body, "from-source") - self.assertEqual(target.database.get_required(document, "note:alias").body, "from-source") - self.assertIsInstance(target.database.get_required(alias, "note:alias"), UiNote) - - def test_cultmesh_document_subscription_syncs_alias_snapshot_and_changes(self) -> None: - @dataclass - class CanonicalNote: - body: str - - @dataclass - class UiNote: - body: str - - document = define_database_entry_type( - "mesh.subscription_alias_note", - [("body", 0)], - cls=CanonicalNote, - schema_id="mesh.subscription_alias_note.v1", - schema_name="mesh.subscription_alias_note", - schema_version="mesh.subscription_alias_note.v1", - ) - alias = define_database_entry_type( - "mesh.subscription_alias_note.ui", - [("body", 0)], - cls=UiNote, - schema_id="mesh.subscription_alias_note.v1", - schema_name="mesh.subscription_alias_note", - schema_version="mesh.subscription_alias_note.v1", - ) - source = CultMesh.create_node(runtime_id="subscription-alias-source") - source.database.register_document(document) - source.database.put_raw_message(document, "note:live", CanonicalNote("initial")) - target = CultMesh.create_node(runtime_id="subscription-alias-target") - target.database.register_document(document) - server = CultMesh.serve_node(source) - try: - raw_client = CultMesh.create_client("127.0.0.1", server.port, timeout_seconds=2.0) - with CultMesh.subscribe_document(target, raw_client, alias, "note:live") as subscription: - initial = subscription.sync_initial() - subscription.send(document_put_raw( - message_id="subscription-alias-put", - key="note:live", - schema_id="mesh.subscription_alias_note.v1", - stored_at="2026-06-14T00:00:00Z", - payload=document.encode_payload(CanonicalNote("updated")), - )) - updated = subscription.read_next_change() - subscription.send(document_delete( - message_id="subscription-alias-delete", - schema_id="mesh.subscription_alias_note.v1", - record_key="note:live", - )) - removed = subscription.read_next_change() - finally: - server.stop() - - self.assertIsInstance(initial, UiNote) - self.assertEqual(initial.body, "initial") - self.assertEqual(updated.change_kind, "updated") - self.assertIsInstance(updated.value, UiNote) - self.assertEqual(updated.value.body, "updated") - self.assertIsInstance(updated.previous_value, UiNote) - self.assertEqual(updated.previous_value.body, "initial") - self.assertEqual(removed.change_kind, "removed") - self.assertIsNone(removed.value) - self.assertIsInstance(removed.previous_value, UiNote) - self.assertEqual(removed.previous_value.body, "updated") - self.assertIsNone(target.database.get(alias, "note:live")) - - def test_cultmesh_snapshot_fanout_syncs_once_and_reports_applied_records(self) -> None: - document = define_database_entry_type( - "mesh.snapshot_loop_note", - [("body", 0)], - schema_id="mesh.snapshot_loop_note.v1", - ) - source = CultMesh.create_node(runtime_id="snapshot-loop-source") - source.database.register_document(document) - source.database.put_raw_message(document, "note:sync-once", {"body": "from-source"}) - target = CultMesh.create_node(runtime_id="snapshot-loop-target") - target.database.register_document(document) - server = CultMesh.serve_node(source) - try: - peers = CultMeshPeerCatalog() - peers.upsert(CultMeshPeerCard( - peer_id="snapshot-loop-source", - verse_id="mesh", - endpoints=(f"cultnet://127.0.0.1:{server.port}",), - roles=("read-replica",), - )) - seen: list[str] = [] - fanout = CultMeshSnapshotFanout( - CultMeshDiscoveryClient("127.0.0.1", server.port, timeout_seconds=2.0), - target.database, - peers, - verse_id="mesh", - roles=["read-replica"], - schema_ids=["mesh.snapshot_loop_note.v1"], - on_applied=lambda record: seen.append(record.record_key), - ) - - applied = fanout.sync_once() - finally: - server.stop() - - self.assertEqual([record.record_key for record in applied], ["note:sync-once"]) - self.assertEqual(seen, ["note:sync-once"]) - self.assertEqual(target.database.get_required(document, "note:sync-once")["body"], "from-source") - - def test_cultmesh_snapshot_fanout_syncs_typed_documents_and_reports_alias_values(self) -> None: - @dataclass - class CanonicalNote: - body: str - - @dataclass - class UiNote: - body: str - - document = define_database_entry_type( - "mesh.snapshot_loop_alias_note", - [("body", 0)], - cls=CanonicalNote, - schema_id="mesh.snapshot_loop_alias_note.v1", - schema_name="mesh.snapshot_loop_alias_note", - schema_version="mesh.snapshot_loop_alias_note.v1", - ) - alias = define_database_entry_type( - "mesh.snapshot_loop_alias_note.ui", - [("body", 0)], - cls=UiNote, - schema_id="mesh.snapshot_loop_alias_note.v1", - schema_name="mesh.snapshot_loop_alias_note", - schema_version="mesh.snapshot_loop_alias_note.v1", - ) - source = CultMesh.create_node(runtime_id="snapshot-loop-alias-source") - source.database.register_document(document) - source.database.put_raw_message(document, "note:typed-loop", CanonicalNote("from-source")) - target = CultMesh.create_node(runtime_id="snapshot-loop-alias-target") - target.database.register_document(document) - server = CultMesh.serve_node(source) - try: - peers = CultMeshPeerCatalog() - peers.upsert(CultMeshPeerCard( - peer_id="snapshot-loop-alias-source", - verse_id="mesh", - endpoints=(f"cultnet://127.0.0.1:{server.port}",), - roles=("read-replica",), - )) - seen: list[UiNote] = [] - fanout = CultMeshSnapshotFanout( - CultMeshDiscoveryClient("127.0.0.1", server.port, timeout_seconds=2.0), - target.database, - peers, - verse_id="mesh", - roles=["read-replica"], - documents=[(alias, "note:typed-loop")], - on_document=seen.append, - ) - - values = fanout.sync_once() - finally: - server.stop() - - self.assertEqual(len(values), 1) - self.assertIsInstance(values[0], UiNote) - self.assertEqual(values[0].body, "from-source") - self.assertEqual(seen, [values[0]]) - self.assertEqual(target.database.get_required(document, "note:typed-loop").body, "from-source") - self.assertIsInstance(target.database.get_required(alias, "note:typed-loop"), UiNote) - - def test_cultmesh_snapshot_fanout_runs_background_loop(self) -> None: - document = define_database_entry_type( - "mesh.snapshot_background_note", - [("body", 0)], - schema_id="mesh.snapshot_background_note.v1", - ) - source = CultMesh.create_node(runtime_id="snapshot-background-source") - source.database.register_document(document) - source.database.put_raw_message(document, "note:background", {"body": "from-background"}) - target = CultMesh.create_node(runtime_id="snapshot-background-target") - target.database.register_document(document) - server = CultMesh.serve_node(source) - try: - peers = CultMeshPeerCatalog() - peers.upsert(CultMeshPeerCard( - peer_id="snapshot-background-source", - verse_id="mesh", - endpoints=(f"cultnet://127.0.0.1:{server.port}",), - roles=("read-replica",), - )) - seen: list[str] = [] - fanout = CultMeshSnapshotFanout( - CultMeshDiscoveryClient("127.0.0.1", server.port, timeout_seconds=2.0), - target.database, - peers, - verse_id="mesh", - roles=["read-replica"], - schema_ids=["mesh.snapshot_background_note.v1"], - interval_seconds=0.05, - on_applied=lambda record: seen.append(record.record_key), - ) - fanout.start() - deadline = time.monotonic() + 2.0 - while not seen and time.monotonic() < deadline: - time.sleep(0.01) - fanout.stop() - finally: - server.stop() - - self.assertTrue(seen) - self.assertEqual(target.database.get_required(document, "note:background")["body"], "from-background") - - def test_cultmesh_peer_health_monitor_probes_reachable_and_failed_endpoints(self) -> None: - document = define_database_entry_type( - "mesh.health_note", - [("body", 0)], - schema_id="mesh.health_note.v1", - ) - node = CultMesh.create_node(runtime_id="health-peer") - node.database.register_document(document) - server = CultMesh.serve_node(node) - closed_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - closed_socket.bind(("127.0.0.1", 0)) - closed_port = closed_socket.getsockname()[1] - closed_socket.close() - try: - peers = CultMeshPeerCatalog() - peers.upsert(CultMeshPeerCard( - peer_id="health-peer", - verse_id="mesh", - endpoints=(f"cultnet://127.0.0.1:{server.port}",), - roles=("read-replica",), - )) - peers.upsert(CultMeshPeerCard( - peer_id="closed-peer", - verse_id="mesh", - endpoints=(f"cultnet://127.0.0.1:{closed_port}",), - roles=("read-replica",), - )) - seen: list[tuple[str, bool]] = [] - monitor = CultMeshPeerHealthMonitor( - runtime_id="health-prober", - timeout_seconds=0.2, - on_update=lambda health: seen.append((health.peer_id, health.is_reachable)), - ) - - results = monitor.probe_catalog(peers, verse_id="mesh", roles=["read-replica"]) - finally: - server.stop() - - by_peer = {result.peer_id: result for result in results} - self.assertTrue(by_peer["health-peer"].is_reachable) - self.assertEqual(by_peer["health-peer"].runtime_id, "health-peer") - self.assertEqual(by_peer["health-peer"].runtime_kind, "python") - self.assertEqual(by_peer["health-peer"].display_name, "health-peer") - self.assertEqual(by_peer["health-peer"].supported_document_types, ("mesh.health_note",)) - self.assertIn("cultnet.hello.v0", by_peer["health-peer"].supported_message_versions) - self.assertIn("cultnet.error.v0", by_peer["health-peer"].supported_message_versions) - self.assertEqual(by_peer["health-peer"].supported_mutation_contracts[0]["documentType"], "mesh.health_note") - self.assertIn("shardLog", by_peer["health-peer"].supported_mutation_contracts[0]["operations"]) - self.assertFalse(by_peer["closed-peer"].is_reachable) - self.assertIsNone(by_peer["closed-peer"].runtime_kind) - self.assertEqual(by_peer["closed-peer"].supported_document_types, ()) - self.assertEqual(by_peer["closed-peer"].supported_message_versions, ()) - self.assertEqual(by_peer["closed-peer"].supported_mutation_contracts, ()) - self.assertTrue(by_peer["closed-peer"].error) - self.assertEqual({peer_id for peer_id, _ in seen}, {"health-peer", "closed-peer"}) - self.assertEqual(len(monitor.latest()), 2) - - def test_cultmesh_peer_health_monitor_runs_background_probe_loop(self) -> None: - server = CultMesh.serve_node(CultMesh.create_node(runtime_id="health-loop-peer")) - try: - peers = CultMeshPeerCatalog() - peers.upsert(CultMeshPeerCard( - peer_id="health-loop-peer", - verse_id="mesh", - endpoints=(f"cultnet://127.0.0.1:{server.port}",), - roles=("read-replica",), - )) - seen: list[str] = [] - monitor = CultMeshPeerHealthMonitor( - runtime_id="health-loop-prober", - timeout_seconds=0.2, - interval_seconds=0.05, - on_update=lambda health: seen.append(health.peer_id), - ) - monitor.start(peers, verse_id="mesh", roles=["read-replica"]) - deadline = time.monotonic() + 2.0 - while time.monotonic() < deadline: - latest = monitor.latest() - if latest and latest[0].is_reachable: - break - time.sleep(0.01) - monitor.stop() - finally: - server.stop() - - self.assertTrue(seen) - latest = monitor.latest() - self.assertEqual(latest[0].peer_id, "health-loop-peer") - self.assertTrue(latest[0].is_reachable) - - def test_cultmesh_daemon_entrypoint_serves_framed_cultnet_hello(self) -> None: - with tempfile.TemporaryDirectory() as temp: - ready_path = Path(temp) / "ready.json" - package_src = Path(__file__).resolve().parents[1] / "src" - env = dict(os.environ) - existing_pythonpath = env.get("PYTHONPATH") - env["PYTHONPATH"] = ( - str(package_src) - if not existing_pythonpath - else f"{package_src}{os.pathsep}{existing_pythonpath}" - ) - process = subprocess.Popen( - [ - sys.executable, - "-m", - "cultmesh_py.daemon", - "--runtime-id", - "daemon-test-peer", - "--display-name", - "Daemon Test Peer", - "--port", - "0", - "--ready-file", - str(ready_path), - "--seed-interop-note", - "--seed-shard-id", - "daemon-interop", - "--verse-id", - "daemon-verse", - "--verse-display-name", - "Daemon Verse", - "--role", - "shard-primary", - ], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, - env=env, - ) - try: - deadline = time.monotonic() + 5.0 - while not ready_path.exists() and time.monotonic() < deadline: - if process.poll() is not None: - break - time.sleep(0.05) - if not ready_path.exists(): - _, stderr = process.communicate(timeout=2.0) - self.fail(f"cultmesh daemon did not publish readiness: {stderr}") - ready = json.loads(ready_path.read_text(encoding="utf-8")) - self.assertEqual(ready["schemaVersion"], READY_SCHEMA_VERSION) - self.assertEqual(ready["runtimeId"], "daemon-test-peer") - self.assertEqual(ready["runtimeKind"], "python") - self.assertEqual(ready["displayName"], "Daemon Test Peer") - self.assertEqual(ready["endpoint"], f"cultnet://127.0.0.1:{ready['port']}") - self.assertEqual(ready["shardIds"], ["daemon-interop"]) - self.assertEqual(ready["supportedDocumentTypes"], ["cultcache.interop-note"]) - self.assertIn("cultnet.hello.v0", ready["supportedMessageVersions"]) - self.assertIn("cultnet.document_put_raw.v0", ready["supportedMessageVersions"]) - self.assertNotIn("cultnet.simulation_observation.v0", ready["supportedMessageVersions"]) - self.assertEqual(ready["supportedMutationContracts"][0]["documentType"], "cultcache.interop-note") - self.assertIn("shardLog", ready["supportedMutationContracts"][0]["operations"]) - self.assertEqual(ready["snapshotLimits"], {"maxSnapshotBytes": None, "maxSnapshotDocuments": None}) - self.assertEqual(ready["verses"][0]["verseId"], "daemon-verse") - self.assertEqual(ready["verses"][0]["displayName"], "Daemon Verse") - self.assertEqual(ready["verses"][0]["discoveryEndpoints"], [ready["endpoint"]]) - self.assertEqual(ready["verses"][0]["authorityRuntimeIds"], ["daemon-test-peer"]) - self.assertEqual(ready["peers"][0]["peerId"], "daemon-test-peer") - self.assertEqual(ready["peers"][0]["endpoints"], [ready["endpoint"]]) - self.assertEqual(ready["peers"][0]["roles"], ["shard-primary"]) - self.assertEqual(ready["peers"][0]["shardIds"], ["daemon-interop"]) - - client = CultMesh.create_client("127.0.0.1", int(ready["port"]), timeout_seconds=2.0) - hello_response = client.request( - hello(runtime_id="daemon-test-prober"), - expected_schema_version="cultnet.hello.v0", - ) - - self.assertEqual(hello_response["runtimeId"], "daemon-test-peer") - self.assertEqual(hello_response["displayName"], "Daemon Test Peer") - self.assertIn("cultnet.hello.v0", hello_response["supportedMessageVersions"]) - self.assertIn("cultnet.schema_catalog_request.v0", hello_response["supportedMessageVersions"]) - self.assertNotIn("cultnet.simulation_observation.v0", hello_response["supportedMessageVersions"]) - self.assertEqual(hello_response["supportedDocumentTypes"], ["cultcache.interop-note"]) - self.assertEqual(hello_response["transportProfiles"][0]["transports"][0]["protocol"], "tcp_framed") - self.assertEqual(hello_response["transportProfiles"][0]["transports"][0]["port"], int(ready["port"])) - self.assertEqual(ready["supportedMessageVersions"], hello_response["supportedMessageVersions"]) - self.assertEqual(ready["supportedMutationContracts"], hello_response["supportedMutationContracts"]) - self.assertEqual(ready["transportProfiles"], hello_response["transportProfiles"]) - - schema_response = client.fetch_schema_catalog(schema_ids=["cultcache.interop-note"], include_schema_json=True) - self.assertEqual(schema_response["schemas"][0]["schemaVersion"], INTEROP_SCHEMA_VERSION) - self.assertEqual(schema_response["schemas"][0]["documentType"], "cultcache.interop-note") - - snapshot_response = client.fetch_snapshot_response(schema_ids=["cultcache.interop-note"]) - self.assertEqual(snapshot_response.documents[0].record_key, "note:daemon-test-peer") - note = interop_note_document.decode_payload(snapshot_response.documents[0].payload) - self.assertEqual(note["authorRuntimeId"], "daemon-test-peer") - self.assertIn("daemon", note["tags"]) - - shard_catalog = client.fetch_shard_catalog(schema_ids=["cultcache.interop-note"]) - self.assertEqual(shard_catalog["shards"][0]["shardId"], "daemon-interop") - self.assertEqual(shard_catalog["shards"][0]["schemaIds"], ["cultcache.interop-note"]) - - shard_log = client.fetch_shard_log_response(shard_id="daemon-interop", shard_epoch=1) - self.assertFalse(shard_log.resync_required) - self.assertEqual(shard_log.entries[0].sequence, 1) - self.assertEqual(shard_log.entries[0].change_kind, "added") - self.assertIsNotNone(shard_log.entries[0].raw_document) - assert shard_log.entries[0].raw_document is not None - self.assertEqual(shard_log.entries[0].raw_document.record_key, "note:daemon-test-peer") - - discovery_client = CultMesh.create_verse_discovery_client("127.0.0.1", int(ready["port"]), timeout_seconds=2.0) - verses = discovery_client.fetch_verses(transport_version="cultmesh.v0") - self.assertEqual(verses[0].verse_id, "daemon-verse") - self.assertEqual(verses[0].display_name, "Daemon Verse") - self.assertEqual(verses[0].discovery_endpoints, (ready["endpoint"],)) - self.assertEqual(verses[0].authority_runtime_ids, ("daemon-test-peer",)) - self.assertEqual([verse.to_wire() for verse in verses], ready["verses"]) - - peers = discovery_client.fetch_peers(verse_id="daemon-verse", roles=["shard-primary"]) - self.assertEqual(peers[0].peer_id, "daemon-test-peer") - self.assertEqual(peers[0].endpoints, (ready["endpoint"],)) - self.assertEqual(peers[0].roles, ("shard-primary",)) - self.assertEqual(peers[0].shard_ids, ("daemon-interop",)) - self.assertEqual([peer.to_wire() for peer in peers], ready["peers"]) - - monitor = CultMeshPeerHealthMonitor(runtime_id="daemon-health-prober", timeout_seconds=2.0) - health = monitor.probe_peer(peers[0])[0] - self.assertTrue(health.is_reachable) - self.assertEqual(health.runtime_id, "daemon-test-peer") - self.assertEqual(health.supported_document_types, ("cultcache.interop-note",)) - self.assertEqual(health.supported_mutation_contracts[0]["documentType"], "cultcache.interop-note") - finally: - process.terminate() - try: - process.communicate(timeout=5.0) - except subprocess.TimeoutExpired: - process.kill() - process.communicate(timeout=5.0) - - def test_cultmesh_daemon_restarts_with_persisted_cache_and_shard_log(self) -> None: - with tempfile.TemporaryDirectory() as temp: - root = Path(temp) - cache_path = root / "mesh.cc" - shard_log_path = root / "shard-log" - env = dict(os.environ) - package_src = Path(__file__).resolve().parents[1] / "src" - existing_pythonpath = env.get("PYTHONPATH") - env["PYTHONPATH"] = ( - str(package_src) - if not existing_pythonpath - else f"{package_src}{os.pathsep}{existing_pythonpath}" - ) - - first_ready = root / "first-ready.json" - first = subprocess.Popen( - [ - sys.executable, - "-m", - "cultmesh_py.daemon", - "--runtime-id", - "durable-daemon-peer", - "--port", - "0", - "--cache-file", - str(cache_path), - "--enable-durable-shard-logs", - "--shard-log-file", - str(shard_log_path), - "--seed-interop-note", - "--seed-shard-id", - "durable-interop", - "--ready-file", - str(first_ready), - ], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, - env=env, - ) - try: - self._wait_for_ready_file(first, first_ready) - finally: - self._terminate_process(first) - - second_ready = root / "second-ready.json" - second = subprocess.Popen( - [ - sys.executable, - "-m", - "cultmesh_py.daemon", - "--runtime-id", - "durable-daemon-peer", - "--port", - "0", - "--cache-file", - str(cache_path), - "--enable-durable-shard-logs", - "--shard-log-file", - str(shard_log_path), - "--register-interop-note", - "--verse-id", - "durable-verse", - "--role", - "shard-primary", - "--ready-file", - str(second_ready), - ], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, - env=env, - ) - try: - ready = self._wait_for_ready_file(second, second_ready) - client = CultMesh.create_client("127.0.0.1", int(ready["port"]), timeout_seconds=2.0) - - snapshot_response = client.fetch_snapshot_response(schema_ids=["cultcache.interop-note"]) - self.assertEqual(snapshot_response.documents[0].record_key, "note:durable-daemon-peer") - note = interop_note_document.decode_payload(snapshot_response.documents[0].payload) - self.assertEqual(note["authorRuntimeId"], "durable-daemon-peer") - - shard_log = client.fetch_shard_log_response(shard_id="durable-interop", shard_epoch=1) - self.assertFalse(shard_log.resync_required) - self.assertEqual(shard_log.entries[0].sequence, 1) - self.assertIsNotNone(shard_log.entries[0].raw_document) - assert shard_log.entries[0].raw_document is not None - self.assertEqual(shard_log.entries[0].raw_document.record_key, "note:durable-daemon-peer") - - discovery_client = CultMesh.create_verse_discovery_client("127.0.0.1", int(ready["port"]), timeout_seconds=2.0) - peers = discovery_client.fetch_peers(verse_id="durable-verse", roles=["shard-primary"]) - self.assertEqual(peers[0].shard_ids, ("durable-interop",)) - finally: - self._terminate_process(second) - - def test_cultmesh_daemon_serves_live_database_subscription_mutations(self) -> None: - with tempfile.TemporaryDirectory() as temp: - ready_path = Path(temp) / "ready.json" - package_src = Path(__file__).resolve().parents[1] / "src" - env = dict(os.environ) - existing_pythonpath = env.get("PYTHONPATH") - env["PYTHONPATH"] = ( - str(package_src) - if not existing_pythonpath - else f"{package_src}{os.pathsep}{existing_pythonpath}" - ) - process = subprocess.Popen( - [ - sys.executable, - "-m", - "cultmesh_py.daemon", - "--runtime-id", - "live-daemon-peer", - "--port", - "0", - "--seed-interop-note", - "--seed-shard-id", - "live-interop", - "--ready-file", - str(ready_path), - ], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, - env=env, - ) - try: - ready = self._wait_for_ready_file(process, ready_path) - client = CultMesh.create_client("127.0.0.1", int(ready["port"]), timeout_seconds=2.0) - live_note = { - "schemaVersion": INTEROP_SCHEMA_VERSION, - "documentId": "note:daemon-live", - "authorRuntimeId": "daemon-live-client", - "title": "daemon live subscription note", - "body": "live subscription put from Python daemon smoke", - "tags": ["daemon", "subscription", "interop"], - } - - with client.subscribe_database( - subscription_id="daemon-live-sub", - schema_ids=["cultcache.interop-note"], - ) as subscription: - snapshot = subscription.read_next_snapshot_response() - subscription.send(document_put_raw( - message_id="daemon-live-put", - key="note:daemon-live", - schema_id="cultcache.interop-note", - stored_at="2026-06-14T00:00:00Z", - payload=interop_note_document.encode_payload(live_note), - source_runtime_id="daemon-live-client", - shard_id="live-interop", - shard_epoch=1, - )) - change = subscription.read_next_change() - - self.assertEqual(snapshot.documents[0].record_key, "note:live-daemon-peer") - self.assertEqual(change.change_kind, "added") - self.assertEqual(change.record_key, "note:daemon-live") - self.assertIsNotNone(change.raw_document) - assert change.raw_document is not None - self.assertEqual( - interop_note_document.decode_payload(change.raw_document.payload), - live_note, - ) - - shard_log = client.fetch_shard_log_response(shard_id="live-interop", shard_epoch=1) - self.assertFalse(shard_log.resync_required) - self.assertEqual([entry.sequence for entry in shard_log.entries], [1, 2]) - self.assertEqual(shard_log.entries[1].change_kind, "added") - self.assertIsNotNone(shard_log.entries[1].raw_document) - assert shard_log.entries[1].raw_document is not None - self.assertEqual(shard_log.entries[1].raw_document.record_key, "note:daemon-live") - finally: - self._terminate_process(process) - - def test_cultmesh_daemon_serves_compacted_shard_log_resync_boundary(self) -> None: - with tempfile.TemporaryDirectory() as temp: - root = Path(temp) - cache_path = root / "mesh.cc" - shard_log_path = root / "shard-log" - store = CultNetFileShardMutationLogStore(shard_log_path) - node = CultMesh.create_node( - cache_path, - runtime_id="compacted-daemon-peer", - enable_durable_shard_logs=True, - shard_log_path=shard_log_path, - ) - node.database.register_document(interop_note_document) - node.database.put_raw_message( - interop_note_document, - "note:compacted-one", - { - "schemaVersion": INTEROP_SCHEMA_VERSION, - "documentId": "note:compacted-one", - "authorRuntimeId": "compacted-daemon-peer", - "title": "compacted daemon note one", - "body": "first retained cache value", - "tags": ["daemon", "compaction"], - }, - message_id="compacted-put-1", - shard_id="compacted-interop", - shard_epoch=7, - ) - node.database.put_raw_message( - interop_note_document, - "note:compacted-two", - { - "schemaVersion": INTEROP_SCHEMA_VERSION, - "documentId": "note:compacted-two", - "authorRuntimeId": "compacted-daemon-peer", - "title": "compacted daemon note two", - "body": "second retained cache value", - "tags": ["daemon", "compaction"], - }, - message_id="compacted-put-2", - shard_id="compacted-interop", - shard_epoch=7, - ) - store.compact_through("compacted-interop", 1) - - ready_path = root / "ready.json" - package_src = Path(__file__).resolve().parents[1] / "src" - env = dict(os.environ) - existing_pythonpath = env.get("PYTHONPATH") - env["PYTHONPATH"] = ( - str(package_src) - if not existing_pythonpath - else f"{package_src}{os.pathsep}{existing_pythonpath}" - ) - process = subprocess.Popen( - [ - sys.executable, - "-m", - "cultmesh_py.daemon", - "--runtime-id", - "compacted-daemon-peer", - "--port", - "0", - "--cache-file", - str(cache_path), - "--enable-durable-shard-logs", - "--shard-log-file", - str(shard_log_path), - "--register-interop-note", - "--ready-file", - str(ready_path), - ], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, - env=env, - ) - try: - ready = self._wait_for_ready_file(process, ready_path) - client = CultMesh.create_client("127.0.0.1", int(ready["port"]), timeout_seconds=2.0) - - snapshot_response = client.fetch_snapshot_response(schema_ids=["cultcache.interop-note"]) - self.assertEqual( - {document.record_key for document in snapshot_response.documents}, - {"note:compacted-one", "note:compacted-two"}, - ) - - compacted = client.fetch_shard_log_response( - shard_id="compacted-interop", - shard_epoch=7, - after_sequence=0, - ) - retained = client.fetch_shard_log_response( - shard_id="compacted-interop", - shard_epoch=7, - after_sequence=1, - ) - - self.assertTrue(compacted.resync_required) - self.assertEqual(compacted.reason, "compacted_history") - self.assertEqual(compacted.compacted_through, 1) - self.assertEqual(compacted.entries, ()) - self.assertFalse(retained.resync_required) - self.assertEqual([entry.sequence for entry in retained.entries], [2]) - self.assertIsNotNone(retained.entries[0].raw_document) - assert retained.entries[0].raw_document is not None - self.assertEqual(retained.entries[0].raw_document.record_key, "note:compacted-two") - finally: - self._terminate_process(process) - - def test_cultmesh_daemon_accepts_schema_write_forwarder_mutations(self) -> None: - with tempfile.TemporaryDirectory() as temp: - ready_path = Path(temp) / "ready.json" - package_src = Path(__file__).resolve().parents[1] / "src" - env = dict(os.environ) - existing_pythonpath = env.get("PYTHONPATH") - env["PYTHONPATH"] = ( - str(package_src) - if not existing_pythonpath - else f"{package_src}{os.pathsep}{existing_pythonpath}" - ) - process = subprocess.Popen( - [ - sys.executable, - "-m", - "cultmesh_py.daemon", - "--runtime-id", - "forwarder-daemon-peer", - "--port", - "0", - "--register-interop-note", - "--ready-file", - str(ready_path), - ], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, - env=env, - ) - try: - ready = self._wait_for_ready_file(process, ready_path) - endpoint = f"cultnet://127.0.0.1:{ready['port']}" - shard = CultNetShardDescriptor( - shard_id="forwarded-interop", - owner_runtime_id="forwarder-daemon-peer", - epoch=3, - is_primary=False, - schema_ids=("cultcache.interop-note",), - primary_endpoints=(endpoint,), - ) - forwarder: CultNetShardWriteForwarder = CultNetSchemaWriteForwarder(timeout_seconds=2.0) - note = { - "schemaVersion": INTEROP_SCHEMA_VERSION, - "documentId": "note:forwarded-daemon", - "authorRuntimeId": "forwarding-client", - "title": "forwarded daemon note", - "body": "forwarded through the schema write-forwarder", - "tags": ["daemon", "forwarded"], - } - - forwarder.forward_put(shard, document_put_raw( - message_id="forwarded-daemon-put", - key="note:forwarded-daemon", - schema_id="cultcache.interop-note", - stored_at="2026-06-14T00:00:00Z", - payload=interop_note_document.encode_payload(note), - source_runtime_id="forwarding-client", - ).to_wire()) - - client = CultMesh.create_client("127.0.0.1", int(ready["port"]), timeout_seconds=2.0) - self._eventually(lambda: bool(client.fetch_snapshot_response( - schema_ids=["cultcache.interop-note"], - record_keys=["note:forwarded-daemon"], - ).documents)) - snapshot = client.fetch_snapshot_response( - schema_ids=["cultcache.interop-note"], - record_keys=["note:forwarded-daemon"], - ) - self.assertEqual( - interop_note_document.decode_payload(snapshot.documents[0].payload), - note, - ) - - forwarder.forward_delete(shard, document_delete( - message_id="forwarded-daemon-delete", - schema_id="cultcache.interop-note", - record_key="note:forwarded-daemon", - ).to_wire()) - self._eventually(lambda: not client.fetch_snapshot_response( - schema_ids=["cultcache.interop-note"], - record_keys=["note:forwarded-daemon"], - ).documents) - - shard_log = client.fetch_shard_log_response(shard_id="forwarded-interop", shard_epoch=3) - self.assertEqual([entry.change_kind for entry in shard_log.entries], ["added", "removed"]) - self.assertEqual(shard_log.entries[0].put["shardId"], "forwarded-interop") - self.assertEqual(shard_log.entries[0].put["shardEpoch"], 3) - self.assertEqual(shard_log.entries[1].delete["shardId"], "forwarded-interop") - self.assertEqual(shard_log.entries[1].delete["shardEpoch"], 3) - finally: - self._terminate_process(process) - - def test_cultmesh_daemon_enforces_snapshot_document_limit_as_peer_error(self) -> None: - with tempfile.TemporaryDirectory() as temp: - ready_path = Path(temp) / "ready.json" - package_src = Path(__file__).resolve().parents[1] / "src" - env = dict(os.environ) - existing_pythonpath = env.get("PYTHONPATH") - env["PYTHONPATH"] = ( - str(package_src) - if not existing_pythonpath - else f"{package_src}{os.pathsep}{existing_pythonpath}" - ) - process = subprocess.Popen( - [ - sys.executable, - "-m", - "cultmesh_py.daemon", - "--runtime-id", - "limited-daemon-peer", - "--port", - "0", - "--seed-interop-note", - "--max-snapshot-documents", - "0", - "--ready-file", - str(ready_path), - ], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, - env=env, - ) - try: - ready = self._wait_for_ready_file(process, ready_path) - self.assertEqual(ready["snapshotLimits"], {"maxSnapshotBytes": None, "maxSnapshotDocuments": 0}) - client = CultMesh.create_client("127.0.0.1", int(ready["port"]), timeout_seconds=2.0) - - with self.assertRaisesRegex(CultNetPeerError, "Snapshot document limit exceeded") as raised: - client.fetch_snapshot_response(schema_ids=["cultcache.interop-note"]) - - self.assertEqual(raised.exception.response["schemaVersion"], "cultnet.error.v0") - self.assertEqual(raised.exception.response["code"], "snapshot_document_limit_exceeded") - self.assertEqual(raised.exception.response["details"]["documentCount"], 1) - self.assertEqual(raised.exception.response["details"]["maxSnapshotDocuments"], 0) - finally: - self._terminate_process(process) - - def test_cultmesh_daemon_serves_opt_in_simulation_observations(self) -> None: - with tempfile.TemporaryDirectory() as temp: - ready_path = Path(temp) / "ready.json" - package_src = Path(__file__).resolve().parents[1] / "src" - env = dict(os.environ) - existing_pythonpath = env.get("PYTHONPATH") - env["PYTHONPATH"] = ( - str(package_src) - if not existing_pythonpath - else f"{package_src}{os.pathsep}{existing_pythonpath}" - ) - process = subprocess.Popen( - [ - sys.executable, - "-m", - "cultmesh_py.daemon", - "--runtime-id", - "simulation-daemon-peer", - "--port", - "0", - "--enable-simulation-observations", - "--simulation-minimum-witnesses", - "2", - "--simulation-quorum-ratio", - "1.0", - "--verse-id", - "simulation-verse", - "--ready-file", - str(ready_path), - ], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, - env=env, - ) - try: - ready = self._wait_for_ready_file(process, ready_path) - self.assertIn("cultnet.simulation_observation.v0", ready["supportedMessageVersions"]) - self.assertIn("cultnet.simulation_consensus_candidate.v0", ready["supportedMessageVersions"]) - client = CultMesh.create_client("127.0.0.1", int(ready["port"]), timeout_seconds=2.0) - hello_response = client.request( - hello(runtime_id="simulation-daemon-prober"), - expected_schema_version="cultnet.hello.v0", - ) - self.assertEqual(ready["supportedMessageVersions"], hello_response["supportedMessageVersions"]) - discovery_client = CultMesh.create_verse_discovery_client("127.0.0.1", int(ready["port"]), timeout_seconds=2.0) - peers = discovery_client.fetch_peers(verse_id="simulation-verse", roles=["simulation-observer"]) - self.assertEqual(peers[0].peer_id, "simulation-daemon-peer") - self.assertIn("simulation-observer", peers[0].roles) - self.assertIn("read-replica", peers[0].roles) - - claim_hash = compute_simulation_claim_hash("hit", "alice", "bob", "frame:100") - first = client.request( - simulation_observation( - message_id="daemon-sim-1", - witness_runtime_id="watcher-1", - shard_id="arena", - shard_epoch=4, - frame=100, - subject_id="bob", - claim_kind="hit", - claim_hash=claim_hash, - claim_summary="alice shot bob first", - ), - expected_schema_version="cultnet.simulation_consensus_candidate.v0", - ) - second = client.request( - simulation_observation( - message_id="daemon-sim-2", - witness_runtime_id="watcher-2", - shard_id="arena", - shard_epoch=4, - frame=100, - subject_id="bob", - claim_kind="hit", - claim_hash=claim_hash, - claim_summary="alice shot bob first", - ), - expected_schema_version="cultnet.simulation_consensus_candidate.v0", - ) - - self.assertEqual(first["messageId"], "daemon-sim-1") - self.assertFalse(first["hasQuorum"]) - self.assertEqual(first["witnessCount"], 1) - self.assertEqual(second["messageId"], "daemon-sim-2") - self.assertTrue(second["hasQuorum"]) - self.assertEqual(second["witnessCount"], 2) - self.assertEqual(second["claimHash"], claim_hash) - finally: - self._terminate_process(process) - - def _wait_for_ready_file(self, process: subprocess.Popen[str], ready_path: Path) -> dict[str, object]: - deadline = time.monotonic() + 5.0 - while not ready_path.exists() and time.monotonic() < deadline: - if process.poll() is not None: - break - time.sleep(0.05) - if not ready_path.exists(): - _, stderr = process.communicate(timeout=2.0) - self.fail(f"cultmesh daemon did not publish readiness: {stderr}") - return json.loads(ready_path.read_text(encoding="utf-8")) - - def _terminate_process(self, process: subprocess.Popen[str]) -> None: - process.terminate() - try: - process.communicate(timeout=5.0) - except subprocess.TimeoutExpired: - process.kill() - process.communicate(timeout=5.0) - - def test_cultmesh_local_server_serves_node_and_catalogs_over_clients(self) -> None: - document = define_database_entry_type( - "mesh.server_note", - [("body", 0)], - schema_id="mesh.server_note.v1", - ) - node = CultMesh.create_node(runtime_id="mesh-server") - node.database.register_document(document) - node.database.put_raw_message(document, "note:1", {"body": "served"}, shard_id="notes", shard_epoch=1) - observation_hub = CultNetSimulationObservationHub( - CultNetSimulationConsensusOptions(minimum_witnesses=2, quorum_ratio=1.0) - ) - seen_candidates: list[CultNetSimulationConsensusCandidate] = [] - unsubscribe_candidates = observation_hub.watch_candidates(seen_candidates.append) - claim_hash = compute_simulation_claim_hash("hit", "alice", "bob", "frame:100") - verses = CultMeshVerseCatalog() - verses.upsert( - CultMeshVerseDescriptor( - verse_id="server-verse", - display_name="Server Verse", - authority_model="local", - compatibility=CultMeshVerseCompatibility("cultmesh.v0", "rules"), - ) - ) - peers = CultMeshPeerCatalog() - peers.upsert( - CultMeshPeerCard( - peer_id="mesh-server", - verse_id="server-verse", - endpoints=("cultnet://127.0.0.1:0",), - roles=("read-replica",), - ) - ) - - server = CultMesh.serve_node( - node, - verse_catalog=verses, - peer_catalog=peers, - observation_hub=observation_hub, - display_name="Mesh Server", - ) - try: - raw_client = CultMesh.create_client("127.0.0.1", server.port, timeout_seconds=2.0) - hello_response = raw_client.request(hello(runtime_id="probe"), expected_schema_version="cultnet.hello.v0") - schema_response = raw_client.fetch_schema_catalog(schema_ids=["mesh.server_note.v1"], include_schema_json=True) - rudp_client = CultMesh.create_client( - endpoint=f"rudp://127.0.0.1:{server.port}", - timeout_seconds=2.0, - connection_id=server.rudp_connection_id, - runtime_id="rudp-probe", - ) - rudp_hello_response = rudp_client.request( - hello(runtime_id="rudp-probe"), - expected_schema_version="cultnet.hello.v0", - ) - rudp_schema_response = rudp_client.fetch_schema_catalog( - schema_ids=["mesh.server_note.v1"], - include_schema_json=True, - ) - wire_schema_response = raw_client.fetch_schema_catalog(kinds=["wire_message"], include_schema_json=True) - synced_schema_catalog = CultNetSchemaCatalog() - synced_wire_descriptors = raw_client.sync_schema_catalog( - synced_schema_catalog, - kinds=["wire_message"], - include_schema_json=True, - ) - snapshot_response = raw_client.fetch_snapshot(schema_ids=["mesh.server_note.v1"]) - typed_snapshot_response = raw_client.fetch_snapshot_response(schema_ids=["mesh.server_note.v1"]) - shard_catalog = raw_client.fetch_shard_catalog(schema_ids=["mesh.server_note.v1"]) - synced_shard_catalog = CultNetShardCatalog() - synced_shards = raw_client.sync_shard_catalog(synced_shard_catalog, schema_ids=["mesh.server_note.v1"]) - shard_log = raw_client.fetch_shard_log(shard_id="notes", shard_epoch=1) - typed_shard_log = raw_client.fetch_shard_log_response(shard_id="notes", shard_epoch=1) - self.assertEqual(hello_response["transportProfiles"][0]["transports"][0]["protocol"], "tcp_framed") - self.assertEqual(hello_response["transportProfiles"][0]["transports"][0]["port"], server.port) - stale_shard_log = raw_client.fetch_shard_log(shard_id="notes", shard_epoch=0) - discovery_client = CultMesh.create_verse_discovery_client("127.0.0.1", server.port, timeout_seconds=2.0) - fetched_verses = discovery_client.fetch_verses(transport_version="cultmesh.v0") - fetched_peers = discovery_client.fetch_peers(verse_id="server-verse", roles=["read-replica"]) - first_candidate = raw_client.request( - simulation_observation( - message_id="obs-1", - witness_runtime_id="watcher-1", - shard_id="arena", - shard_epoch=4, - frame=100, - subject_id="bob", - claim_kind="hit", - claim_hash=claim_hash, - claim_summary="alice shot bob first", - ), - expected_schema_version="cultnet.simulation_consensus_candidate.v0", - ) - quorum_candidate = raw_client.request( - simulation_observation( - message_id="obs-2", - witness_runtime_id="watcher-2", - shard_id="arena", - shard_epoch=4, - frame=100, - subject_id="bob", - claim_kind="hit", - claim_hash=claim_hash, - claim_summary="alice shot bob first", - ), - expected_schema_version="cultnet.simulation_consensus_candidate.v0", - ) - with raw_client.subscribe_database(subscription_id="sub-server", schema_ids=["mesh.server_note.v1"]) as subscription: - subscription_snapshot = subscription.read_next_snapshot_response() - subscription.send(document_put_raw( - message_id="server-put", - key="note:2", - schema_id="mesh.server_note.v1", - stored_at="2026-06-14T00:00:00Z", - payload=document.encode_payload({"body": "subscribed"}), - shard_id="primary", - shard_epoch=1, - )) - subscription_change = subscription.read_next_change() - subscription.send(document_delete( - message_id="server-delete", - schema_id="mesh.server_note.v1", - record_key="note:2", - shard_id="primary", - shard_epoch=1, - )) - subscription_delete = subscription.read_next_change() - finally: - unsubscribe_candidates() - server.stop() - - self.assertEqual(hello_response["runtimeId"], "mesh-server") - self.assertEqual(hello_response["displayName"], "Mesh Server") - self.assertIn("cultnet.database_subscribe.v0", hello_response["supportedMessageVersions"]) - self.assertIn("cultnet.error.v0", hello_response["supportedMessageVersions"]) - self.assertIn("cultnet.document_put_raw.v0", hello_response["supportedMessageVersions"]) - self.assertIn("cultnet.simulation_observation.v0", hello_response["supportedMessageVersions"]) - self.assertEqual(hello_response["supportedMutationContracts"][0]["documentType"], "mesh.server_note") - self.assertIn("documentDelete", hello_response["supportedMutationContracts"][0]["operations"]) - self.assertIn("shardLog", hello_response["supportedMutationContracts"][0]["operations"]) - advertised_transports = { - transport["protocol"]: transport - for profile in hello_response["transportProfiles"] - for transport in profile["transports"] - } - self.assertEqual(advertised_transports["tcp_framed"]["port"], server.port) - self.assertEqual(advertised_transports["rudp"]["port"], server.port) - self.assertEqual(rudp_hello_response["runtimeId"], "mesh-server") - self.assertEqual(rudp_schema_response["schemas"][0]["schemaId"], "mesh.server_note.v1") - self.assertEqual(schema_response["schemas"][0]["schemaId"], "mesh.server_note.v1") - self.assertIn("schemaJson", schema_response["schemas"][0]) - self.assertIn("cultnet.document_delete.v0", schema_response["schemas"][0]["wireContracts"]) - self.assertIn("cultnet.shard_log_response.v0", schema_response["schemas"][0]["wireContracts"]) - wire_descriptors = {schema["schemaVersion"]: schema for schema in wire_schema_response["schemas"]} - self.assertEqual(wire_descriptors["cultnet.error.v0"]["kind"], "wire_message") - self.assertEqual(wire_descriptors["cultnet.document_put_raw.v0"]["kind"], "wire_message") - self.assertIn("schemaJson", wire_descriptors["cultnet.document_put_raw.v0"]) - self.assertIn("cultmesh.peer_exchange_response.v0", wire_descriptors) - self.assertIn("cultnet.error.v0", {descriptor.schema_version for descriptor in synced_wire_descriptors}) - self.assertIn("cultnet.document_put_raw.v0", {descriptor.schema_version for descriptor in synced_wire_descriptors}) - self.assertIsNotNone(synced_schema_catalog.get("https://github.com/GameCult/cultnet-ts/contracts/cultnet.document-put-raw.schema.json")) - self.assertEqual(snapshot_response["documents"][0]["recordKey"], "note:1") - self.assertEqual(typed_snapshot_response.documents[0].record_key, "note:1") - self.assertEqual(typed_snapshot_response.documents[0].schema_id, "mesh.server_note.v1") - self.assertEqual(shard_catalog["shards"][0]["schemaIds"], ["mesh.server_note.v1"]) - self.assertEqual(shard_catalog["shards"][0]["epoch"], 1) - self.assertEqual(shard_catalog["shards"][0]["shardId"], "notes") - self.assertEqual(synced_shards[0].shard_id, "notes") - self.assertEqual(synced_shards[0].epoch, 1) - self.assertEqual(synced_shard_catalog.get("notes"), synced_shards[0]) - self.assertEqual(shard_log["entries"][0]["changeKind"], "added") - self.assertTrue(stale_shard_log["resyncRequired"]) - self.assertEqual(stale_shard_log["reason"], "stale_epoch") - self.assertEqual(stale_shard_log["shardEpoch"], 1) - self.assertEqual(stale_shard_log["entries"], []) - self.assertEqual(typed_shard_log.last_sequence, 1) - self.assertEqual(typed_shard_log.entries[0].put["document"]["recordKey"], "note:1") - self.assertIsInstance(typed_shard_log.entries[0].raw_document, CultNetRawDocumentRecord) - self.assertEqual(typed_shard_log.entries[0].raw_document.record_key, "note:1") - self.assertEqual(fetched_verses[0].verse_id, "server-verse") - self.assertEqual(fetched_peers[0].peer_id, "mesh-server") - self.assertEqual(first_candidate["messageId"], "obs-1") - self.assertFalse(first_candidate["hasQuorum"]) - self.assertEqual(quorum_candidate["messageId"], "obs-2") - self.assertEqual(quorum_candidate["claimHash"], claim_hash) - self.assertTrue(quorum_candidate["hasQuorum"]) - self.assertEqual(quorum_candidate["witnessCount"], 2) - self.assertEqual([candidate.claim_hash for candidate in seen_candidates], [claim_hash, claim_hash]) - self.assertIsInstance(subscription_snapshot, CultNetRawSnapshotResponse) - self.assertEqual(subscription_snapshot.to_wire()["schemaVersion"], "cultnet.snapshot_response_raw.v0") - self.assertEqual(subscription_change.change_kind, "added") - self.assertEqual(subscription_change.record_key, "note:2") - self.assertEqual(subscription_change.document["recordKey"], "note:2") - self.assertIsInstance(subscription_change.raw_document, CultNetRawDocumentRecord) - self.assertEqual(subscription_change.raw_document.schema_id, "mesh.server_note.v1") - self.assertEqual(subscription_delete.change_kind, "removed") - self.assertEqual(subscription_delete.record_key, "note:2") - self.assertIsNone(subscription_delete.raw_document) - self.assertIsNone(node.database.get(document, "note:2")) - - def test_cultmesh_local_server_resolves_snapshot_and_shard_catalog_schema_aliases(self) -> None: - document = define_database_entry_type( - "mesh.alias_server_note", - [("body", 0)], - schema_id="sha256:mesh-alias-server-note", - schema_name="mesh.alias_server_note", - schema_version="mesh.alias_server_note.v1", - ) - node = CultMesh.create_node(runtime_id="mesh-alias-server") - node.database.register_document(document) - node.database.put_raw_message( - document, - "note:alias", - {"body": "served through alias"}, - shard_id="notes", - shard_epoch=1, - ) - - server = CultMesh.serve_node(node) - try: - raw_client = CultMesh.create_client("127.0.0.1", server.port, timeout_seconds=2.0) - snapshot = raw_client.fetch_snapshot(schema_ids=["mesh.alias_server_note.v1"]) - typed_snapshot = raw_client.fetch_snapshot_response(schema_ids=["mesh.alias_server_note.v1"]) - shard_catalog = raw_client.fetch_shard_catalog(schema_ids=["mesh.alias_server_note.v1"]) - finally: - server.stop() - - local_schema_id = document.catalog_entry().schema_id - self.assertEqual(snapshot["documents"][0]["schemaId"], local_schema_id) - self.assertEqual(snapshot["documents"][0]["recordKey"], "note:alias") - self.assertEqual(typed_snapshot.documents[0].schema_id, local_schema_id) - self.assertEqual(shard_catalog["shards"][0]["schemaIds"], [local_schema_id]) - self.assertEqual(shard_catalog["shards"][0]["shardId"], "notes") - - def test_cultmesh_shard_snapshot_keeps_foreign_schema_alias_membership(self) -> None: - document = define_database_entry_type( - "mesh.alias_shard_note", - [ - ("schema_version", 0), - ("body", 1), - ], - schema_id="sha256:mesh-alias-shard-note", - schema_name="mesh.alias_shard_note", - schema_version="mesh.alias_shard_note.v1", - ) - node = CultMesh.create_node(runtime_id="mesh-alias-shard-server") - node.database.register_document(document) - - node.database.apply_raw_put_message(document_put_raw( - message_id="put-foreign-shard-alias", - key="note:foreign-shard", - schema_id="runtime.generated.mesh.alias_shard_note.ui.99", - stored_at="2026-06-27T00:00:00Z", - payload=document.encode_payload({ - "schema_version": "mesh.alias_shard_note.v1", - "body": "visible through shard snapshot", - }), - shard_id="notes", - shard_epoch=1, - ).to_wire()) - node.database.put_raw_message( - document, - "note:other-shard", - { - "schema_version": "mesh.alias_shard_note.v1", - "body": "hidden by shard filter", - }, - shard_id="other", - shard_epoch=1, - ) - - response = node.database.build_snapshot_response( - schema_ids=["mesh.alias_shard_note.v1"], - shard_id="notes", - shard_epoch=1, - ) - - local_schema_id = document.catalog_entry().schema_id - self.assertEqual(response.shard_id, "notes") - self.assertEqual(response.shard_epoch, 1) - self.assertEqual([(record.schema_id, record.record_key) for record in response.documents], [ - (local_schema_id, "note:foreign-shard"), - ]) - self.assertEqual(node.cache.get_required_envelope(document, "note:foreign-shard").schema_id, local_schema_id) - - node.database.apply_raw_delete_message(document_delete( - message_id="delete-foreign-shard-alias", - schema_id="mesh.alias_shard_note.v1", - record_key="note:foreign-shard", - shard_id="notes", - shard_epoch=1, - ).to_wire()) - deleted_response = node.database.build_snapshot_response( - schema_ids=["mesh.alias_shard_note.v1"], - shard_id="notes", - shard_epoch=1, - ) - self.assertEqual(deleted_response.documents, ()) - - def test_cultmesh_local_server_notifies_schema_alias_subscription_for_raw_put(self) -> None: - document = define_database_entry_type( - "mesh.alias_sub_note", - [ - ("schema_version", 0), - ("body", 1), - ], - schema_id="sha256:mesh-alias-sub-note", - schema_name="mesh.alias_sub_note", - schema_version="mesh.alias_sub_note.v1", - ) - node = CultMesh.create_node(runtime_id="mesh-alias-sub-server") - node.database.register_document(document) - - server = CultMesh.serve_node(node) - try: - raw_client = CultMesh.create_client("127.0.0.1", server.port, timeout_seconds=2.0) - with raw_client.subscribe_database( - subscription_id="sub-alias", - schema_ids=["mesh.alias_sub_note.v1"], - include_snapshot=False, - ) as subscription: - subscription.send(document_put_raw( - message_id="put-foreign-alias", - key="note:alias-sub", - schema_id="runtime.generated.mesh.alias_sub_note.ui.99", - stored_at="2026-06-27T00:00:00Z", - payload=document.encode_payload({ - "schema_version": "mesh.alias_sub_note.v1", - "body": "notified", - }), - )) - change = subscription.read_next_change() - finally: - server.stop() - - local_schema_id = document.catalog_entry().schema_id - self.assertEqual(change.subscription_id, "sub-alias") - self.assertEqual(change.change_kind, "added") - self.assertEqual(change.record_key, "note:alias-sub") - self.assertIsInstance(change.raw_document, CultNetRawDocumentRecord) - self.assertEqual(change.raw_document.schema_id, "runtime.generated.mesh.alias_sub_note.ui.99") - self.assertEqual(node.database.get_required(document, "note:alias-sub")["body"], "notified") - self.assertEqual(node.cache.get_required_envelope(document, "note:alias-sub").schema_id, local_schema_id) - - def test_cultmesh_local_server_notifies_schema_alias_subscription_for_delete(self) -> None: - document = define_database_entry_type( - "mesh.alias_delete_note", - [("body", 0)], - schema_id="sha256:mesh-alias-delete-note", - schema_name="mesh.alias_delete_note", - schema_version="mesh.alias_delete_note.v1", - ) - node = CultMesh.create_node(runtime_id="mesh-alias-delete-server") - node.database.register_document(document) - node.database.put_raw_message( - document, - "note:delete-alias", - {"body": "remove me"}, - shard_id="notes", - shard_epoch=1, - ) - - server = CultMesh.serve_node(node) - try: - raw_client = CultMesh.create_client("127.0.0.1", server.port, timeout_seconds=2.0) - with raw_client.subscribe_database( - subscription_id="sub-delete-alias", - schema_ids=[document.catalog_entry().schema_id], - include_snapshot=False, - ) as subscription: - subscription.send(document_delete( - message_id="delete-foreign-alias", - schema_id="mesh.alias_delete_note.v1", - record_key="note:delete-alias", - shard_id="notes", - shard_epoch=1, - )) - change = subscription.read_next_change() - finally: - server.stop() - - self.assertEqual(change.subscription_id, "sub-delete-alias") - self.assertEqual(change.change_kind, "removed") - self.assertEqual(change.schema_id, "mesh.alias_delete_note.v1") - self.assertEqual(change.record_key, "note:delete-alias") - self.assertIsNone(change.raw_document) - self.assertIsNone(node.database.get(document, "note:delete-alias")) - - def test_cultmesh_local_server_rejects_oversized_snapshot_responses(self) -> None: - document = define_database_entry_type( - "mesh.snapshot_limit_note", - [("body", 0)], - schema_id="mesh.snapshot_limit_note.v1", - ) - node = CultMesh.create_node(runtime_id="mesh-snapshot-limits") - node.database.register_document(document) - node.database.put_raw_message(document, "note:1", {"body": "first"}) - node.database.put_raw_message(document, "note:2", {"body": "second"}) - - server = CultMesh.serve_node(node, max_snapshot_documents=1) - try: - raw_client = CultMesh.create_client("127.0.0.1", server.port, timeout_seconds=2.0) - filtered_snapshot = raw_client.fetch_snapshot(record_keys=["note:1"]) - error = raw_client.request( - snapshot_request(schema_ids=["mesh.snapshot_limit_note.v1"]), - expected_schema_version="cultnet.error.v0", - ) - with self.assertRaisesRegex(CultNetPeerError, "Snapshot document limit exceeded"): - raw_client.fetch_snapshot(schema_ids=["mesh.snapshot_limit_note.v1"]) - finally: - server.stop() - - self.assertEqual(filtered_snapshot["schemaVersion"], "cultnet.snapshot_response_raw.v0") - self.assertEqual([document["recordKey"] for document in filtered_snapshot["documents"]], ["note:1"]) - self.assertEqual(error["schemaVersion"], "cultnet.error.v0") - self.assertIn("Snapshot document limit exceeded", error["error"]) - self.assertEqual(error["code"], "snapshot_document_limit_exceeded") - self.assertEqual(error["details"]["documentCount"], 2) - self.assertEqual(error["details"]["maxSnapshotDocuments"], 1) - - def test_cultmesh_local_server_returns_errors_for_bad_requests(self) -> None: - document = define_database_entry_type( - "mesh.error_note", - [("body", 0)], - schema_id="mesh.error_note.v1", - ) - node = CultMesh.create_node(runtime_id="mesh-error-server") - node.database.register_document(document) - - server = CultMesh.serve_node(node) - try: - raw_client = CultMesh.create_client("127.0.0.1", server.port, timeout_seconds=2.0) - unsupported = raw_client.request( - {"schemaVersion": "cultnet.nope.v0", "messageId": "bad-schema"}, - expected_schema_version="cultnet.error.v0", - ) - malformed_put = raw_client.request( - {"schemaVersion": "cultnet.document_put_raw.v0", "messageId": "bad-put"}, - expected_schema_version="cultnet.error.v0", - ) - malformed_delete = raw_client.request( - {"schemaVersion": "cultnet.document_delete.v0", "messageId": "bad-delete", "schemaId": "mesh.error_note.v1"}, - expected_schema_version="cultnet.error.v0", - ) - finally: - server.stop() - - self.assertEqual(unsupported["messageId"], "bad-schema") - self.assertEqual(unsupported["code"], "unsupported_schema_version") - self.assertEqual(unsupported["details"]["schemaVersion"], "cultnet.nope.v0") - self.assertIn("Unsupported CultNet message schema", unsupported["error"]) - self.assertEqual(malformed_put["messageId"], "bad-put") - self.assertEqual(malformed_put["code"], "malformed_document_put") - self.assertEqual(malformed_delete["messageId"], "bad-delete") - self.assertEqual(malformed_delete["code"], "malformed_document_delete") - - def test_cultmesh_authority_lease_requires_live_matching_lease(self) -> None: - peer = CultMeshPeerCard( - peer_id="voidbot-local", - verse_id="local", - endpoints=("cultmesh://localhost",), - roles=("shard-primary",), - authority_lease_id="lease:voidbot-local", - ) - leases = CultMeshAuthorityLeaseCatalog() - peers = CultMeshPeerCatalog() - peers.upsert(peer) - now = datetime.now(UTC) - self.assertFalse(leases.is_authorized(peer, "shard-primary", at=now)) - self.assertEqual(peers.find_authorized("local", "shard-primary", leases, at=now), []) - - leases.upsert( - CultMeshAuthorityLease( - lease_id="lease:voidbot-local", - verse_id="local", - peer_id="voidbot-local", - roles=("shard-primary", "shard-primary", ""), - valid_from=now - timedelta(seconds=1), - expires_at=now + timedelta(seconds=1), - shard_ids=("shard-a",), - issuer_runtime_id="odin", - signature="signed", - ) - ) - - lease = leases.get("lease:voidbot-local") - self.assertIsNotNone(lease) - assert lease is not None - self.assertEqual(leases.leases, (lease,)) - self.assertEqual(lease.roles, ("shard-primary",)) - self.assertEqual(lease.issuer_runtime_id, "odin") - self.assertEqual(lease.signature, "signed") - self.assertTrue(lease.is_valid_at(now)) - self.assertTrue(lease.covers(peer, "shard-primary", shard_id="shard-a", at=now)) - self.assertFalse(lease.covers(peer, "shard-primary", shard_id="shard-b", at=now)) - self.assertTrue(leases.is_authorized(peer, "shard-primary", shard_id="shard-a", at=now)) - self.assertEqual( - peers.find_authorized("local", "shard-primary", leases, shard_id="shard-a", at=now), - [peer], - ) - self.assertEqual( - peers.first_authorized("local", "shard-primary", leases, shard_id="shard-a", at=now), - peer, - ) - self.assertIsNone(peers.first_authorized("local", "read-replica", leases, at=now)) - self.assertFalse(leases.is_authorized(peer, "shard-primary", shard_id="shard-b", at=now)) - self.assertFalse(leases.is_authorized(peer, "read-replica", at=now)) - self.assertIsNone(leases.get("missing")) - - def test_cultmesh_authority_lease_catalog_can_require_verified_signatures(self) -> None: - peer = CultMeshPeerCard( - peer_id="voidbot-local", - verse_id="local", - endpoints=("cultmesh://localhost",), - roles=("shard-primary",), - shard_ids=("shard-a",), - authority_lease_id="lease:voidbot-local", - ) - now = datetime.now(UTC) - unsigned = CultMeshAuthorityLease( - lease_id="lease:voidbot-local", - verse_id="local", - peer_id="voidbot-local", - roles=("shard-primary",), - valid_from=now - timedelta(seconds=1), - expires_at=now + timedelta(seconds=30), - shard_ids=("shard-a",), - issuer_runtime_id="odin", - ) - verifier = CultMeshHmacAuthorityLeaseVerifier({"odin": b"lease-secret"}) - signed = verifier.issue(unsigned) - strict_leases = CultMesh.create_authority_lease_catalog( - signature_verifier=verifier.verify, - require_verified_signatures=True, - ) - strict_leases.upsert(unsigned) - self.assertFalse(strict_leases.is_verified(unsigned.lease_id)) - self.assertFalse(strict_leases.is_authorized(peer, "shard-primary", shard_id="shard-a", at=now)) - - strict_leases.upsert(signed) - self.assertTrue(strict_leases.is_verified(signed.lease_id)) - self.assertTrue(strict_leases.is_authorized(peer, "shard-primary", shard_id="shard-a", at=now)) - - wrong_key = CultMesh.create_authority_lease_catalog( - signature_verifier=CultMeshHmacAuthorityLeaseVerifier({"odin": b"wrong-secret"}).verify, - require_verified_signatures=True, - ) - wrong_key.upsert(signed) - self.assertFalse(wrong_key.is_verified(signed.lease_id)) - self.assertFalse(wrong_key.is_authorized(peer, "shard-primary", shard_id="shard-a", at=now)) - - tampered = CultMeshAuthorityLease( - lease_id=signed.lease_id, - verse_id=signed.verse_id, - peer_id=signed.peer_id, - roles=("shard-primary", "read-replica"), - valid_from=signed.valid_from, - expires_at=signed.expires_at, - shard_ids=signed.shard_ids, - issuer_runtime_id=signed.issuer_runtime_id, - signature=signed.signature, - ) - strict_leases.upsert(tampered) - self.assertFalse(strict_leases.is_verified(tampered.lease_id)) - self.assertFalse(strict_leases.is_authorized(peer, "shard-primary", shard_id="shard-a", at=now)) - - def test_cultmesh_stream_catalog_negotiates_transport_and_latest_frame(self) -> None: - streams = CultMeshStreamCatalog() - streams.declare( - CultMeshStreamDescriptor( - stream_id="mimir:kiyo-pro", - verse_id="studio", - owner_peer_id="starfire", - kind="video", - label="Kiyo Pro", - clock={"clockDomainId": "starfire-qpc", "confidence": 0.25}, - video={"width": 1920, "height": 1080, "pixelFormat": "YUY2", "framesPerSecond": 30}, - preferred_transports=("shared-d3d12-texture", "shared-memory", "cultcache-page"), - max_in_flight_frames=3, - ) - ) - - negotiation = streams.negotiate( - "mimir:kiyo-pro", - CultMeshStreamConsumerProfile( - peer_id="fensalir", - verse_id="studio", - supported_transports=("shared-d3d12-texture", "cultcache-page"), - accepted_kinds=("video",), - can_import_gpu_handles=True, - max_in_flight_frames=2, - ), - ) - - self.assertEqual(negotiation.transport, "shared-d3d12-texture") - self.assertEqual(negotiation.max_in_flight_frames, 2) - self.assertEqual(negotiation.copy_budget, "zero-copy-target") - - streams.publish_frame( - CultMeshStreamFrameHandle( - stream_id="mimir:kiyo-pro", - sequence=42, - timestamp_ns=1_000_000_000, - duration_ns=33_333_334, - transport="shared-d3d12-texture", - native_handle="0xfeed", - fence_handle="0xbeef", - fence_value=7, - unavoidable_copy_count=0, - ) - ) - self.assertEqual(streams.latest_frame("mimir:kiyo-pro").sequence, 42) - if __name__ == "__main__": unittest.main() diff --git a/packages/cultcache-rs/Cargo.toml b/packages/cultcache-rs/Cargo.toml index ef06727d..23d53801 100644 --- a/packages/cultcache-rs/Cargo.toml +++ b/packages/cultcache-rs/Cargo.toml @@ -1,7 +1,8 @@ [package] name = "cultcache-rs" -version = "0.1.0" +version = "0.2.0" edition = "2024" +license = "MIT" publish = false [workspace] @@ -17,14 +18,15 @@ chrono = { version = "0.4", features = ["serde"] } cultcache-rs-derive = { path = "cultcache-rs-derive" } fs2 = "0.4" rmp-serde = "1.3" +redb = "4.1.0" serde = { version = "1", features = ["derive"] } serde_bytes = "0.11" serde_json = "1" uuid = { version = "1", features = ["v4"] } -[target.'cfg(windows)'.dependencies] -windows-sys = { version = "0.59", features = ["Win32_Storage_FileSystem"] } - [dev-dependencies] pretty_assertions = "1.4" tempfile = "3" + +[target.'cfg(windows)'.dependencies] +windows-sys = { version = "0.59", features = ["Win32_Foundation", "Win32_Storage_FileSystem"] } diff --git a/packages/cultcache-rs/README.md b/packages/cultcache-rs/README.md index 74b20dd3..1ad01eb0 100644 --- a/packages/cultcache-rs/README.md +++ b/packages/cultcache-rs/README.md @@ -53,7 +53,7 @@ struct Settings { let mut cache = CultCache::new(); cache.register_entry_type::()?; -cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new("cache.msgpack")); +cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new("cache.cc")); cache.pull_all_backing_stores()?; cache.put("app", &Settings { @@ -136,7 +136,7 @@ pub struct PlayerData { let mut cache = CultCache::builder() .with_generated_entries(game_entries()) - .with_generic_store(SingleFileMessagePackBackingStore::new("cache.msgpack")) + .with_generic_store(SingleFileMessagePackBackingStore::new("cache.cc")) .build()?; cache.pull_all_backing_stores()?; @@ -173,39 +173,71 @@ without pretending Rust has runtime reflection hiding under the floorboards. - `get_envelope::` - `get_required_envelope::` - `get_all::` +- `get_all_with_keys::` +- `soa::` - `put::` - `put_envelope::` - `update::` - `delete::` - `snapshot` +- `snapshot_soa` +- `load_soa` - `SingleFileMessagePackBackingStore` +## Structure Of Arrays + +The C# CultCache hot path exposes CPU-local SoA scans as: + +```csharp +Span x = cache.Soa() + .Column(nameof(PlayerTransform.PositionX)) + .Span; +``` + +Rust cannot discover those columns through runtime reflection, so each domain +type declares its column extractors explicitly with `SoaDocument` or the +`cultcache_soa!` helper: + +```rust +cultcache_soa!(PlayerTransform { + "position_x" => |row: &PlayerTransform| row.position_x, + "health" => |row: &PlayerTransform| row.health, +}); + +let table = cache.soa::()?; +let health = table.column::("health")?; +``` + +This is the same public shape as the C# rite: callers ask the cache for a typed +table, then pull contiguous typed columns by name. The Rust version is explicit +about the extractor table because the closed-world schema boundary is a virtue, +not a missing reflection party trick. + ## Backing Store Routing Generic store: ```rust -cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new("cache.msgpack")); +cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new("cache.cc"))?; ``` Type-specific store: ```rust cache.add_backing_store( - SingleFileMessagePackBackingStore::new("players.msgpack"), + SingleFileMessagePackBackingStore::new("players.cc"), ["player"], -); +)?; ``` -When writing a `PlayerData`, the cache checks type-specific stores first. If none -match, it writes to the first generic store. Later matching stores are mirrors. +The contract, shared with every runtime, is one home store per document type +(`src/GameCult.Caching/Contracts/cultcache-store-composition.md` in CultLib): -This mirrors the C# behavior: - -- specific domain stores own their domain -- the first generic store is the primary generic write target -- later generic stores mirror writes -- this is not multi-master +- a type-specific store owns its types; otherwise the generic store is the home +- a second generic store is an error, and so is a type claimed by a second + store; both registrations return `Err` and attach nothing +- a write touches exactly one store; there are no mirrors +- a write whose type has no home store is an error ## Persistence Semantics @@ -250,38 +282,8 @@ marked `default` use `Default::default()` when older payloads are read. Deleted field slots should stay reserved until an explicit store migration rewrites the data. -The single-file MessagePack store rewrites one complete snapshot. Under the -exclusive sidecar lock it removes abandoned candidates without following them, -creates a unique same-directory candidate with create-new semantics, writes and -flushes it, and then replaces the committed pathname without deleting that -pathname first. A process interruption before replacement leaves the previous -snapshot authoritative. Once replacement succeeds, readers observe the complete -new snapshot rather than a partial file. - -Unix requests a parent-directory flush after replacement. Windows uses -`MoveFileExW` with replace-existing and write-through flags. If the final Unix -directory flush fails, the method returns a downcastable -`CommittedSnapshotDurabilityUncertain`: the complete new snapshot may already be -visible, so the caller must fail-stop or reopen and reconcile before issuing -higher-level work again. Treating that error as "not committed" and blindly -retrying can duplicate commands or receipts. - -Abandoned `..tmp` candidates are never read as committed state and are -removed by the next exclusive writer. Cleanup recognizes only the exact -canonical UUID form emitted for that target, including targets whose native -filename is not UTF-8; similar `.tmp` names remain untouched. Candidate creation -uses create-new semantics, so an unexpected file or symlink collision fails -instead of being opened or truncated. The canonical candidate filename namespace -is reserved to the store. - -These guarantees assume a local filesystem that honors the platform rename and -flush primitives, and writers that honor the sidecar lock. Network filesystems, -faulty storage hardware, and unrelated writers may provide weaker behavior. -Fault-injection tests prove the process-visible old-or-new outcomes around the -replacement boundary; they do not simulate sudden power loss or prove that a -device honored a successful flush. - -This is a sane starting point for small typed state surfaces, settings, Epiphany agent memory, +The single-file MessagePack store rewrites an atomic snapshot. That is a sane +starting point for small typed state surfaces, settings, Epiphany agent memory, heartbeat state, and other compact control-plane data. Large corpora should use a sharded store or a real database instead of asking one file to become a warehouse and then acting wounded when physics invoices us. @@ -290,9 +292,6 @@ The single-file store uses a sidecar lock file for shared reads and exclusive writes. That protects ordinary multi-process access to the same file. It is still not a multi-master replication protocol, and it is not a substitute for a coordinator when higher-level write ordering matters. -An explicit unlock error cannot replace the write outcome: dropping the lock-file -handle is the final OS lock-release path, so a committed-durability-uncertain -result remains inspectable rather than being masked by cleanup noise. ## Near-Term Ergonomic Improvements diff --git a/packages/cultcache-rs/cultcache-rs-derive/src/lib.rs b/packages/cultcache-rs/cultcache-rs-derive/src/lib.rs index f05a23f3..7d613194 100644 --- a/packages/cultcache-rs/cultcache-rs-derive/src/lib.rs +++ b/packages/cultcache-rs/cultcache-rs-derive/src/lib.rs @@ -75,6 +75,7 @@ pub fn derive_database_entry(input: TokenStream) -> TokenStream { }; let mut key: Option = None; let mut default = false; + let mut bytes = false; for attribute in field .attrs .iter() @@ -88,6 +89,9 @@ pub fn derive_database_entry(input: TokenStream) -> TokenStream { } else if meta.path.is_ident("default") { default = true; Ok(()) + } else if meta.path.is_ident("bytes") { + bytes = true; + Ok(()) } else { Err(meta.error("unsupported field cultcache attribute")) } @@ -117,6 +121,7 @@ pub fn derive_database_entry(input: TokenStream) -> TokenStream { ident: field_ident, ty: field.ty, default, + bytes, }); } slots.sort_by_key(|slot| slot.key); @@ -125,8 +130,14 @@ pub fn derive_database_entry(input: TokenStream) -> TokenStream { let serialize_elements = (0..=max_slot).map(|slot| { if let Some(field) = slots.iter().find(|field| field.key == slot) { let field_ident = &field.ident; - quote! { - tuple.serialize_element(&self.#field_ident)?; + if field.bytes { + quote! { + tuple.serialize_element(::serde_bytes::Bytes::new(&self.#field_ident))?; + } + } else { + quote! { + tuple.serialize_element(&self.#field_ident)?; + } } } else { quote! { @@ -138,8 +149,14 @@ pub fn derive_database_entry(input: TokenStream) -> TokenStream { let option_declarations = slots.iter().map(|field| { let variable = format_ident!("slot_{}", field.key); let ty = &field.ty; - quote! { - let mut #variable: ::std::option::Option<#ty> = ::std::option::Option::None; + if field.bytes { + quote! { + let mut #variable: ::std::option::Option<::serde_bytes::ByteBuf> = ::std::option::Option::None; + } + } else { + quote! { + let mut #variable: ::std::option::Option<#ty> = ::std::option::Option::None; + } } }); @@ -147,9 +164,17 @@ pub fn derive_database_entry(input: TokenStream) -> TokenStream { if let Some(field) = slots.iter().find(|field| field.key == slot) { let variable = format_ident!("slot_{}", field.key); let ty = &field.ty; - quote! { - #slot => { - #variable = seq.next_element::<#ty>()?; + if field.bytes { + quote! { + #slot => { + #variable = seq.next_element::<::serde_bytes::ByteBuf>()?; + } + } + } else { + quote! { + #slot => { + #variable = seq.next_element::<#ty>()?; + } } } } else { @@ -164,7 +189,16 @@ pub fn derive_database_entry(input: TokenStream) -> TokenStream { let build_fields = slots.iter().map(|field| { let field_ident = &field.ident; let variable = format_ident!("slot_{}", field.key); - if field.default { + if field.bytes && field.default { + quote! { + #field_ident: #variable.unwrap_or_default().into_vec() + } + } else if field.bytes { + let field_name = field_ident.to_string(); + quote! { + #field_ident: #variable.ok_or_else(|| ::serde::de::Error::missing_field(#field_name))?.into_vec() + } + } else if field.default { quote! { #field_ident: #variable.unwrap_or_default() } @@ -240,4 +274,5 @@ struct CultCacheSlot { ident: syn::Ident, ty: Type, default: bool, + bytes: bool, } diff --git a/packages/cultcache-rs/examples/cultcache_interop.rs b/packages/cultcache-rs/examples/cultcache_interop.rs index 4049acf8..e625db9e 100644 --- a/packages/cultcache-rs/examples/cultcache_interop.rs +++ b/packages/cultcache-rs/examples/cultcache_interop.rs @@ -74,7 +74,7 @@ fn read_note(file: &str) -> Result<()> { fn build_cache(file: &str) -> Result { let mut cache = CultCache::new(); cache.register_entry_type::()?; - cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(file)); + cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(file))?; Ok(cache) } diff --git a/packages/cultcache-rs/src/lib.rs b/packages/cultcache-rs/src/lib.rs index b85b763a..ed9cae32 100644 --- a/packages/cultcache-rs/src/lib.rs +++ b/packages/cultcache-rs/src/lib.rs @@ -1,18 +1,22 @@ use anyhow::Context; use anyhow::Result; use anyhow::anyhow; +use anyhow::bail; +use anyhow::ensure; use serde::Serialize; use serde::de::DeserializeOwned; +use std::any::Any; use std::collections::BTreeMap; use std::collections::BTreeSet; -use std::ffi::OsStr; -use std::fmt; use std::fs; use std::fs::File; use std::fs::OpenOptions; -use std::io::Write; +use std::marker::PhantomData; use std::path::Path; use std::path::PathBuf; +use std::sync::Arc; + +use redb::{Database, ReadableDatabase, ReadableTable, TableDefinition}; extern crate self as cultcache_rs; @@ -27,6 +31,83 @@ pub trait CultCacheRegistry { fn register_entries(&self, cache: &mut CultCache) -> Result<()>; } +pub trait SoaDocument: DatabaseEntry { + fn soa_columns(rows: &[Self]) -> BTreeMap<&'static str, CultSoaColumnValues>; +} + +pub struct CultSoaColumnValues { + type_name: &'static str, + values: Box, +} + +impl CultSoaColumnValues { + pub fn new(values: Vec) -> Self { + Self { + type_name: std::any::type_name::(), + values: Box::new(values), + } + } +} + +#[derive(Clone, Debug, PartialEq)] +pub struct CultSoaColumn { + name: String, + values: Vec, +} + +impl CultSoaColumn { + pub fn name(&self) -> &str { + &self.name + } + + pub fn values(&self) -> &[TValue] { + &self.values + } +} + +pub struct CultSoaTable { + keys: Vec, + documents: Vec, + columns: BTreeMap, + _document: PhantomData, +} + +impl CultSoaTable { + pub fn keys(&self) -> &[String] { + &self.keys + } + + pub fn documents(&self) -> &[TDocument] { + &self.documents + } + + pub fn len(&self) -> usize { + self.keys.len() + } + + pub fn is_empty(&self) -> bool { + self.keys.is_empty() + } + + pub fn column(&self, name: &str) -> Result> { + let Some(column) = self.columns.get(name) else { + return Err(anyhow!("SoA table has no column {name:?}")); + }; + let Some(values) = column.values.downcast_ref::>() else { + return Err(anyhow!( + "SoA table column {:?} stores {}, not {}", + name, + column.type_name, + std::any::type_name::() + )); + }; + Ok(CultSoaColumn { + name: name.to_string(), + values: values.clone(), + }) + } +} + #[macro_export] macro_rules! cultcache_registry { ($name:ident { $($entry:ty),* $(,)? }) => { @@ -44,6 +125,22 @@ macro_rules! cultcache_registry { }; } +#[macro_export] +macro_rules! cultcache_soa { + ($entry:ty { $($column:literal => $extractor:expr),* $(,)? }) => { + impl $crate::SoaDocument for $entry { + fn soa_columns(rows: &[Self]) -> ::std::collections::BTreeMap<&'static str, $crate::CultSoaColumnValues> { + let mut columns = ::std::collections::BTreeMap::new(); + $( + let values = rows.iter().map($extractor).collect::>(); + columns.insert($column, $crate::CultSoaColumnValues::new(values)); + )* + columns + } + } + }; +} + #[derive(Clone, Debug, PartialEq, serde::Serialize, serde::Deserialize)] #[serde(rename_all = "camelCase")] pub struct CultCacheEnvelope { @@ -57,6 +154,93 @@ pub struct CultCacheEnvelope { pub schema_id: Option, } +#[derive(Clone, Debug, Default, PartialEq, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct CultCacheSoaTable { + pub keys: Vec, + pub types: Vec, + pub payloads: Vec>, + pub stored_ats: Vec, + pub schema_ids: Vec>, +} + +impl CultCacheSoaTable { + pub fn from_envelopes(entries: impl IntoIterator) -> Self { + let mut table = Self::default(); + for entry in entries { + table.keys.push(entry.key); + table.types.push(entry.r#type); + table.payloads.push(entry.payload); + table.stored_ats.push(entry.stored_at); + table.schema_ids.push(entry.schema_id); + } + table + } + + pub fn len(&self) -> usize { + self.keys.len() + } + + pub fn is_empty(&self) -> bool { + self.keys.is_empty() + } + + pub fn validate(&self) -> Result<()> { + let len = self.keys.len(); + for (name, candidate) in [ + ("types", self.types.len()), + ("payloads", self.payloads.len()), + ("storedAts", self.stored_ats.len()), + ("schemaIds", self.schema_ids.len()), + ] { + if candidate != len { + return Err(anyhow!( + "CultCache SoA column {name} has length {candidate}, expected {len}" + )); + } + } + for (index, key) in self.keys.iter().enumerate() { + if key.trim().is_empty() { + return Err(anyhow!("CultCache SoA row {index} has an empty key")); + } + if self.types[index].trim().is_empty() { + return Err(anyhow!("CultCache SoA row {index} has an empty type")); + } + if self.stored_ats[index].trim().is_empty() { + return Err(anyhow!("CultCache SoA row {index} has an empty stored_at")); + } + } + Ok(()) + } + + pub fn into_envelopes(self) -> Result> { + self.validate()?; + let Self { + keys, + types, + payloads, + stored_ats, + schema_ids, + } = self; + Ok(keys + .into_iter() + .zip(types) + .zip(payloads) + .zip(stored_ats) + .zip(schema_ids) + .map( + |((((key, r#type), payload), stored_at), schema_id)| CultCacheEnvelope { + key, + r#type, + payload, + stored_at, + schema_id, + }, + ) + .collect()) + } +} + #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] struct PersistedStoreSnapshot( String, @@ -100,60 +284,56 @@ pub struct PushAllOptions { pub soft: bool, } -/// A complete new snapshot replaced the committed pathname, but its final -/// durability barrier failed. Callers must fail-stop or reopen and reconcile; -/// retrying as though the write never committed can duplicate higher-level work. -#[derive(Debug)] -pub struct CommittedSnapshotDurabilityUncertain { - path: PathBuf, - detail: String, +/// One per-entry expectation for [`SingleFileMessagePackBackingStore::compare_exchange`]. +/// `current` is the envelope the caller believes is stored, or `None` when it +/// believes the identity is absent. +#[derive(Clone, Debug, PartialEq)] +pub struct CultCacheExpectedEnvelope { + pub key: String, + pub r#type: String, + pub current: Option, } -impl CommittedSnapshotDurabilityUncertain { - fn new(path: PathBuf, error: impl fmt::Display) -> Self { - Self { - path, - detail: error.to_string(), - } - } - - pub fn path(&self) -> &Path { - &self.path - } - - pub fn detail(&self) -> &str { - &self.detail - } +/// Result of one nonblocking whole-snapshot compare-and-exchange. Contention is +/// reported rather than waited on, so the caller keeps retry and scheduling +/// authority instead of blocking inside the store. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum TryCompareExchangeSnapshotOutcome { + Exchanged, + Mismatch, + LockContended, } -impl fmt::Display for CommittedSnapshotDurabilityUncertain { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - write!( - formatter, - "snapshot {} was committed but its durability is uncertain: {}", - self.path.display(), - self.detail - ) - } -} - -impl std::error::Error for CommittedSnapshotDurabilityUncertain {} - pub trait CacheBackingStore: Send { fn pull_all(&self) -> Result>; fn push(&mut self, entry: &CultCacheEnvelope) -> Result<()>; fn delete(&mut self, entry: &CultCacheEnvelope) -> Result<()>; - fn push_all(&mut self, entries: &[CultCacheEnvelope], _options: PushAllOptions) -> Result<()> { - let existing = self.pull_all()?; - for entry in existing { - self.delete(&entry)?; - } - for entry in entries { - self.push(entry)?; - } - Ok(()) - } + /// Replaces everything the store holds with `entries` in one atomic step: + /// on `Err` the store still holds exactly what it held before. A cache + /// batch (`put_prepared_batch`) is all-or-nothing only through this. There + /// is no default, because a replace assembled from `delete` and `push` + /// calls leaves the store emptied or partial when one of them fails; a + /// store without an atomic replace does not compile: + /// + /// ```compile_fail,E0046 + /// use cultcache_rs::{CacheBackingStore, CultCacheEnvelope}; + /// + /// struct PerRecordStore; + /// + /// impl CacheBackingStore for PerRecordStore { + /// fn pull_all(&self) -> anyhow::Result> { + /// Ok(Vec::new()) + /// } + /// fn push(&mut self, _entry: &CultCacheEnvelope) -> anyhow::Result<()> { + /// Ok(()) + /// } + /// fn delete(&mut self, _entry: &CultCacheEnvelope) -> anyhow::Result<()> { + /// Ok(()) + /// } + /// } + /// ``` + fn push_all(&mut self, entries: &[CultCacheEnvelope], options: PushAllOptions) -> Result<()>; } #[derive(Clone, Debug, PartialEq, Eq)] @@ -170,6 +350,305 @@ impl SingleFileMessagePackBackingStore { &self.path } + /// Holds the pre-created sibling lock shared while a read-only consumer + /// evaluates and acts on one snapshot. The reader never creates or writes + /// either file. Writers using the sibling lock cannot replace the snapshot + /// until `action` returns. + /// + /// Releasing the lock is RAII cleanup. A release failure therefore cannot + /// rewrite the completed action's result; consumers that need to observe + /// that cleanup failure can use + /// [`Self::with_read_only_shared_snapshot_and_unlock_diagnostic`]. + pub fn with_read_only_shared_snapshot( + &self, + action: impl FnOnce(Vec) -> Result, + ) -> Result { + self.with_read_only_shared_snapshot_and_unlock_diagnostic(action, |_| {}) + } + + /// The diagnostic-bearing form of + /// [`Self::with_read_only_shared_snapshot`]. `on_unlock_failure` observes a + /// cleanup failure separately and never changes the action result. + pub fn with_read_only_shared_snapshot_and_unlock_diagnostic( + &self, + action: impl FnOnce(Vec) -> Result, + on_unlock_failure: impl FnOnce(anyhow::Error), + ) -> Result { + self.with_read_only_shared_snapshot_using( + action, + |lock| { + fs2::FileExt::unlock(lock) + .with_context(|| format!("failed to unlock {}", self.lock_path().display())) + }, + on_unlock_failure, + ) + } + + fn with_read_only_shared_snapshot_using( + &self, + action: impl FnOnce(Vec) -> Result, + unlock: U, + on_unlock_failure: D, + ) -> Result + where + U: FnOnce(&File) -> Result<()>, + D: FnOnce(anyhow::Error), + { + let lock_path = self.lock_path(); + let lock = OpenOptions::new() + .read(true) + .open(&lock_path) + .with_context(|| format!("failed to open pre-created {}", lock_path.display()))?; + fs2::FileExt::lock_shared(&lock) + .with_context(|| format!("failed to lock {}", lock_path.display()))?; + let guard = ReadOnlySharedLockGuard::new(lock, unlock, on_unlock_failure); + let result = self.read_all_unlocked().and_then(action); + drop(guard); + result + } + + /// Replaces one exact envelope under the store's cross-process exclusive + /// lock. A changed or missing expected envelope returns false without a write. + pub fn compare_and_swap_entry( + &self, + expected: &CultCacheEnvelope, + replacement: CultCacheEnvelope, + ) -> Result { + self.with_exclusive_lock(|| { + let mut entries = self.read_all_unlocked()?; + let Some(index) = entries + .iter() + .position(|candidate| entry_id(candidate) == entry_id(expected)) + else { + return Ok(false); + }; + if entries[index] != *expected { + return Ok(false); + } + entries[index] = replacement; + entries.sort_by_key(entry_id); + self.write_all_unlocked(&entries)?; + Ok(true) + }) + } + + /// Inserts one envelope only when its polymorphic type/key identity is + /// absent, under the same cross-process lock used by all single-file writes. + pub fn insert_entry_if_absent(&self, entry: CultCacheEnvelope) -> Result { + self.with_exclusive_lock(|| { + let mut entries = self.read_all_unlocked()?; + if entries + .iter() + .any(|candidate| entry_id(candidate) == entry_id(&entry)) + { + return Ok(false); + } + entries.push(entry); + entries.sort_by_key(entry_id); + self.write_all_unlocked(&entries)?; + Ok(true) + }) + } + + /// Atomically replaces an exact set of existing envelopes and inserts any + /// additional companion envelopes. Every precondition is checked beneath + /// the same cross-process lock as the single snapshot write. + pub fn compare_and_swap_batch( + &self, + expected: &[CultCacheEnvelope], + replacements: Vec, + ) -> Result { + if replacements.is_empty() { + return Err(anyhow!( + "conditional batch requires a non-empty replacement set" + )); + } + let expected_ids = unique_batch_ids(expected, "expected")?; + let replacement_ids = unique_batch_ids(&replacements, "replacement")?; + if !expected_ids.is_subset(&replacement_ids) { + return Err(anyhow!( + "conditional batch must replace every expected identity" + )); + } + + self.with_exclusive_lock(|| { + let mut entries = self.read_all_unlocked()?; + for expected_entry in expected { + let Some(current) = entries + .iter() + .find(|candidate| entry_id(candidate) == entry_id(expected_entry)) + else { + return Ok(false); + }; + if current != expected_entry { + return Ok(false); + } + } + for replacement in &replacements { + let identity = entry_id(replacement); + if !expected_ids.contains(&identity) + && entries + .iter() + .any(|candidate| entry_id(candidate) == identity) + { + return Ok(false); + } + } + + entries.retain(|candidate| !expected_ids.contains(&entry_id(candidate))); + entries.extend(replacements); + entries.sort_by_key(entry_id); + self.write_all_unlocked(&entries)?; + Ok(true) + }) + } + + /// Atomically appends companions only when the complete backing-store + /// snapshot is byte-for-byte unchanged. Unlike identity-scoped CAS, this + /// fences predicate cardinality: a concurrently inserted unknown identity + /// makes the expected snapshot stale. + pub fn append_if_snapshot_unchanged( + &self, + expected_snapshot: &[CultCacheEnvelope], + additions: Vec, + ) -> Result { + if additions.is_empty() { + return Err(anyhow!("conditional snapshot append requires additions")); + } + unique_batch_ids(expected_snapshot, "expected snapshot")?; + let addition_ids = unique_batch_ids(&additions, "snapshot additions")?; + self.with_exclusive_lock(|| { + let mut current = self.read_all_unlocked()?; + let mut expected = expected_snapshot.to_vec(); + current.sort_by_key(entry_id); + expected.sort_by_key(entry_id); + if current != expected { + return Ok(false); + } + if current + .iter() + .any(|row| addition_ids.contains(&entry_id(row))) + { + return Ok(false); + } + current.extend(additions); + current.sort_by_key(entry_id); + self.write_all_unlocked(¤t)?; + Ok(true) + }) + } + + /// Atomically deletes an exact set of envelopes. Any changed or missing + /// member refuses the entire deletion without a partial write. + pub fn delete_batch_if_unchanged(&self, expected: &[CultCacheEnvelope]) -> Result { + if expected.is_empty() { + return Err(anyhow!( + "conditional delete requires a non-empty expected set" + )); + } + let expected_ids = unique_batch_ids(expected, "delete expected")?; + self.with_exclusive_lock(|| { + let mut entries = self.read_all_unlocked()?; + for expected_entry in expected { + let Some(current) = entries + .iter() + .find(|candidate| entry_id(candidate) == entry_id(expected_entry)) + else { + return Ok(false); + }; + if current != expected_entry { + return Ok(false); + } + } + entries.retain(|candidate| !expected_ids.contains(&entry_id(candidate))); + self.write_all_unlocked(&entries)?; + Ok(true) + }) + } + + /// Atomically replaces existing rows and appends companion rows only while + /// the complete backing-store snapshot remains byte-exact. + pub fn replace_and_append_if_snapshot_unchanged( + &self, + expected_snapshot: &[CultCacheEnvelope], + replacements: Vec, + ) -> Result { + if replacements.is_empty() { + return Err(anyhow!( + "conditional snapshot replacement requires replacements" + )); + } + unique_batch_ids(expected_snapshot, "expected snapshot")?; + let replacement_ids = unique_batch_ids(&replacements, "snapshot replacements")?; + self.with_exclusive_lock(|| { + let mut current = self.read_all_unlocked()?; + let mut expected = expected_snapshot.to_vec(); + current.sort_by_key(entry_id); + expected.sort_by_key(entry_id); + if current != expected { + return Ok(false); + } + current.retain(|row| !replacement_ids.contains(&entry_id(row))); + current.extend(replacements); + current.sort_by_key(entry_id); + self.write_all_unlocked(¤t)?; + Ok(true) + }) + } + + /// Atomically replaces and deletes rows only while the complete store + /// snapshot remains byte-exact. This is the single-file compaction + /// primitive: summary publication and history deletion share one write. + pub fn replace_and_delete_if_snapshot_unchanged( + &self, + expected_snapshot: &[CultCacheEnvelope], + replacements: Vec, + deletions: &[CultCacheEnvelope], + ) -> Result { + if replacements.is_empty() || deletions.is_empty() { + return Err(anyhow!( + "conditional compaction requires replacements and deletions" + )); + } + let expected_ids = unique_batch_ids(expected_snapshot, "expected snapshot")?; + let replacement_ids = unique_batch_ids(&replacements, "replacement")?; + let deletion_ids = unique_batch_ids(deletions, "deletion")?; + if !deletion_ids.is_subset(&expected_ids) { + return Err(anyhow!( + "compaction deletion is absent from expected snapshot" + )); + } + if !deletion_ids.is_disjoint(&replacement_ids) { + return Err(anyhow!("compaction cannot replace and delete one identity")); + } + self.with_exclusive_lock(|| { + let mut current = self.read_all_unlocked()?; + let mut expected = expected_snapshot.to_vec(); + current.sort_by_key(entry_id); + expected.sort_by_key(entry_id); + if current != expected { + return Ok(false); + } + for row in &replacements { + if !expected_ids.contains(&entry_id(row)) + && current + .iter() + .any(|candidate| entry_id(candidate) == entry_id(row)) + { + return Ok(false); + } + } + current.retain(|row| !deletion_ids.contains(&entry_id(row))); + for row in replacements { + current.retain(|candidate| entry_id(candidate) != entry_id(&row)); + current.push(row); + } + current.sort_by_key(entry_id); + self.write_all_unlocked(¤t)?; + Ok(true) + }) + } + fn read_all_unlocked(&self) -> Result> { if !self.path.exists() { return Ok(Vec::new()); @@ -179,57 +658,47 @@ impl SingleFileMessagePackBackingStore { if bytes.is_empty() { return Ok(Vec::new()); } - decode_store_snapshot(&bytes).or_else(|_| { - rmp_serde::from_slice(&bytes) - .with_context(|| format!("failed to decode MessagePack {}", self.path.display())) - }) + decode_store_snapshot(&bytes) + .or_else(|_| rmp_serde::from_slice(&bytes)) + .with_context(|| format!("failed to decode MessagePack {}", self.path.display())) + } + + /// Reads one filesystem snapshot without creating or opening the sibling + /// CultCache lock file. + /// + /// This is deliberately narrower than `pull_all`: use it only at a + /// provider-owned, read-only crossing where the producer replaces the + /// complete file atomically and the consumer has no authority to mutate + /// the provider directory. An absent file is an empty snapshot. Writers + /// and locally owned stores must continue to use the locked APIs. + pub fn pull_all_read_only_snapshot(&self) -> Result> { + self.read_all_unlocked() } fn write_all_unlocked(&self, entries: &[CultCacheEnvelope]) -> Result<()> { - self.write_all_unlocked_with_observer(entries, |_| Ok(())) - } - - fn write_all_unlocked_with_observer( - &self, - entries: &[CultCacheEnvelope], - mut observe: impl FnMut(SnapshotReplacementStage) -> Result<()>, - ) -> Result<()> { if let Some(parent) = self.path.parent() { fs::create_dir_all(parent) .with_context(|| format!("failed to create {}", parent.display()))?; } - remove_stale_candidates(&self.path)?; - let bytes = rmp_serde::to_vec(&encode_store_snapshot(entries)) + remove_abandoned_staging_files(&self.path)?; + let bytes = rmp_serde::to_vec(&encode_store_snapshot(entries)?) .context("failed to encode MessagePack")?; let tmp_path = temporary_path_for(&self.path); - let mut candidate = OpenOptions::new() + let mut staged = OpenOptions::new() .create_new(true) .write(true) .open(&tmp_path) - .with_context(|| format!("failed to open {}", tmp_path.display()))?; - candidate + .with_context(|| format!("failed to create {}", tmp_path.display()))?; + use std::io::Write; + staged .write_all(&bytes) .with_context(|| format!("failed to write {}", tmp_path.display()))?; - candidate + staged .sync_all() - .with_context(|| format!("failed to flush {}", tmp_path.display()))?; - drop(candidate); - - observe(SnapshotReplacementStage::CandidateSynced)?; - replace_file(&tmp_path, &self.path)?; - if let Err(error) = observe(SnapshotReplacementStage::TargetReplaced) { - return Err(anyhow::Error::new( - CommittedSnapshotDurabilityUncertain::new(self.path.clone(), error), - )); - } - - if let Some(parent) = self.path.parent() { - if let Err(error) = sync_parent_directory(parent) { - return Err(anyhow::Error::new( - CommittedSnapshotDurabilityUncertain::new(self.path.clone(), error), - )); - } - } + .with_context(|| format!("failed to sync {}", tmp_path.display()))?; + drop(staged); + replace_file_atomically(&tmp_path, &self.path)?; + sync_parent_directory(&self.path)?; Ok(()) } @@ -244,26 +713,133 @@ impl SingleFileMessagePackBackingStore { } fn with_exclusive_lock(&self, action: impl FnOnce() -> Result) -> Result { - self.with_exclusive_lock_using_unlock(action, |lock| fs2::FileExt::unlock(lock)) - } - - fn with_exclusive_lock_using_unlock( - &self, - action: impl FnOnce() -> Result, - unlock: impl FnOnce(&File) -> std::io::Result<()>, - ) -> Result { let lock = self.open_lock_file()?; fs2::FileExt::lock_exclusive(&lock) .with_context(|| format!("failed to lock {}", self.lock_path().display()))?; - let action_result = action(); + let result = action(); + fs2::FileExt::unlock(&lock) + .with_context(|| format!("failed to unlock {}", self.lock_path().display()))?; + result + } - // Closing the file handle releases the OS lock even when an explicit - // unlock reports an error. The action outcome remains authoritative: - // in particular, never replace a committed-durability-uncertain error - // with secondary lock-cleanup noise. - let _unlock_result = unlock(&lock); - drop(lock); - action_result + /// Replaces the named entries only if each still matches its expectation. + /// Unlike the snapshot form this constrains only the identities named, so + /// unrelated concurrent writes do not lose the race. + pub fn compare_exchange( + &self, + expected: &[CultCacheExpectedEnvelope], + replacements: &[CultCacheEnvelope], + ) -> Result { + self.with_exclusive_lock(|| { + let mut entries = self.read_all_unlocked()?; + for condition in expected { + let current = entries + .iter() + .find(|entry| entry.r#type == condition.r#type && entry.key == condition.key); + if current != condition.current.as_ref() { + return Ok(false); + } + if let Some(envelope) = condition.current.as_ref() + && (envelope.r#type != condition.r#type || envelope.key != condition.key) + { + return Err(anyhow!( + "CultCache compare-exchange expectation identity differs from its envelope" + )); + } + } + let mut replacement_ids = BTreeSet::new(); + for replacement in replacements { + if replacement.key.trim().is_empty() || replacement.r#type.trim().is_empty() { + return Err(anyhow!( + "CultCache compare-exchange replacement identity is empty" + )); + } + if !replacement_ids.insert(entry_id(replacement)) { + return Err(anyhow!( + "CultCache compare-exchange contains a duplicate replacement" + )); + } + } + entries.retain(|entry| !replacement_ids.contains(&entry_id(entry))); + entries.extend_from_slice(replacements); + entries.sort_by_key(entry_id); + self.write_all_unlocked(&entries)?; + Ok(true) + }) + } + + /// Replaces the whole snapshot only if it still equals `expected`, waiting + /// for the exclusive lock. Returns false when another writer has moved the + /// snapshot on, which is a lost race rather than an error. + pub fn compare_exchange_snapshot( + &self, + expected: &[CultCacheEnvelope], + replacements: &[CultCacheEnvelope], + ) -> Result { + self.with_exclusive_lock(|| self.compare_exchange_snapshot_unlocked(expected, replacements)) + } + + /// Makes one nonblocking attempt at the same exchange. Once the lock is + /// held, comparison, validation, ordering and persistence are identical to + /// [`Self::compare_exchange_snapshot`]. + pub fn try_compare_exchange_snapshot( + &self, + expected: &[CultCacheEnvelope], + replacements: &[CultCacheEnvelope], + ) -> Result { + let Some(exchanged) = self.try_with_exclusive_lock(|| { + self.compare_exchange_snapshot_unlocked(expected, replacements) + })? + else { + return Ok(TryCompareExchangeSnapshotOutcome::LockContended); + }; + Ok(if exchanged { + TryCompareExchangeSnapshotOutcome::Exchanged + } else { + TryCompareExchangeSnapshotOutcome::Mismatch + }) + } + + fn compare_exchange_snapshot_unlocked( + &self, + expected: &[CultCacheEnvelope], + replacements: &[CultCacheEnvelope], + ) -> Result { + let current = self.read_all_unlocked()?; + if current != expected { + return Ok(false); + } + let mut replacement_ids = BTreeSet::new(); + for replacement in replacements { + if replacement.key.trim().is_empty() || replacement.r#type.trim().is_empty() { + return Err(anyhow!("CultCache snapshot replacement identity is empty")); + } + if !replacement_ids.insert(entry_id(replacement)) { + return Err(anyhow!( + "CultCache snapshot replacement contains a duplicate identity" + )); + } + } + let mut ordered = replacements.to_vec(); + ordered.sort_by_key(entry_id); + self.write_all_unlocked(&ordered)?; + Ok(true) + } + + fn try_with_exclusive_lock(&self, action: impl FnOnce() -> Result) -> Result> { + let lock = self.open_lock_file()?; + match fs2::FileExt::try_lock_exclusive(&lock) { + Ok(()) => {} + Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => return Ok(None), + Err(error) => { + return Err(error) + .with_context(|| format!("failed to lock {}", self.lock_path().display())); + } + } + let result = action(); + fs2::FileExt::unlock(&lock) + .with_context(|| format!("failed to unlock {}", self.lock_path().display()))?; + result.map(Some) } fn open_lock_file(&self) -> Result { @@ -286,14 +862,71 @@ impl SingleFileMessagePackBackingStore { .path .file_name() .map(|value| value.to_os_string()) - .unwrap_or_else(|| "cultcache.msgpack".into()); + .unwrap_or_else(|| "cultcache.cc".into()); lock_name.push(".lock"); self.path.with_file_name(lock_name) } } +struct ReadOnlySharedLockGuard +where + U: FnOnce(&File) -> Result<()>, + D: FnOnce(anyhow::Error), +{ + lock: Option, + unlock: Option, + on_unlock_failure: Option, +} + +impl ReadOnlySharedLockGuard +where + U: FnOnce(&File) -> Result<()>, + D: FnOnce(anyhow::Error), +{ + fn new(lock: File, unlock: U, on_unlock_failure: D) -> Self { + Self { + lock: Some(lock), + unlock: Some(unlock), + on_unlock_failure: Some(on_unlock_failure), + } + } +} + +impl Drop for ReadOnlySharedLockGuard +where + U: FnOnce(&File) -> Result<()>, + D: FnOnce(anyhow::Error), +{ + fn drop(&mut self) { + let (Some(lock), Some(unlock)) = (self.lock.as_ref(), self.unlock.take()) else { + return; + }; + if let Err(error) = unlock(lock) + && let Some(on_unlock_failure) = self.on_unlock_failure.take() + { + on_unlock_failure(error); + } + } +} + +fn unique_batch_ids( + entries: &[CultCacheEnvelope], + label: &str, +) -> Result> { + let ids = entries.iter().map(entry_id).collect::>(); + if ids.len() != entries.len() { + return Err(anyhow!( + "conditional batch {label} set contains duplicate identities" + )); + } + Ok(ids) +} + impl CacheBackingStore for SingleFileMessagePackBackingStore { fn pull_all(&self) -> Result> { + if !self.path.exists() && !self.lock_path().exists() { + return Ok(Vec::new()); + } self.with_shared_lock(|| self.read_all_unlocked()) } @@ -324,6 +957,947 @@ impl CacheBackingStore for SingleFileMessagePackBackingStore { } } +const REDB_ENVELOPES: TableDefinition<&[u8], &[u8]> = TableDefinition::new("cultcache_envelopes"); + +/// Transactional keyed CultCache storage. Each polymorphic `(type, key)` +/// identity is an independent redb row whose value is the exact MessagePack +/// serialization of its `CultCacheEnvelope`. +#[derive(Clone)] +pub struct RedbMessagePackBackingStore { + path: PathBuf, +} + +impl std::fmt::Debug for RedbMessagePackBackingStore { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("RedbMessagePackBackingStore") + .field("path", &self.path) + .finish_non_exhaustive() + } +} + +impl RedbMessagePackBackingStore { + pub fn new(path: impl Into) -> Result { + let path = path.into(); + if let Some(parent) = path + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + { + fs::create_dir_all(parent) + .with_context(|| format!("failed to create {}", parent.display()))?; + } + let store = Self { path }; + store.with_database(|database| { + let write = database.begin_write()?; + { + write.open_table(REDB_ENVELOPES)?; + } + write.commit()?; + Ok(()) + })?; + Ok(store) + } + + pub fn path(&self) -> &Path { + &self.path + } + + fn open_database(&self) -> Result { + Database::create(&self.path) + .with_context(|| format!("failed to open redb CultCache {}", self.path.display())) + } + + /// redb permits one open writable database handle per path. The external + /// CultCache lock therefore owns the complete open/transaction/close + /// interval, preserving the fresh-handle and cross-process construction + /// model used by the other backing store. + fn with_database(&self, action: impl FnOnce(&Database) -> Result) -> Result { + let lock_path = self.lock_path(); + if let Some(parent) = lock_path + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + { + fs::create_dir_all(parent) + .with_context(|| format!("failed to create {}", parent.display()))?; + } + let lock = OpenOptions::new() + .create(true) + .read(true) + .truncate(false) + .write(true) + .open(&lock_path) + .with_context(|| format!("failed to open {}", lock_path.display()))?; + fs2::FileExt::lock_exclusive(&lock) + .with_context(|| format!("failed to lock {}", lock_path.display()))?; + let result = self.open_database().and_then(|database| action(&database)); + fs2::FileExt::unlock(&lock) + .with_context(|| format!("failed to unlock {}", lock_path.display()))?; + result + } + + fn lock_path(&self) -> PathBuf { + let mut lock_name = self + .path + .file_name() + .map(|value| value.to_os_string()) + .unwrap_or_else(|| "cultcache.cc".into()); + lock_name.push(".lock"); + self.path.with_file_name(lock_name) + } + + pub fn compare_and_swap_entry( + &self, + expected: &CultCacheEnvelope, + replacement: CultCacheEnvelope, + ) -> Result { + if entry_id(expected) != entry_id(&replacement) { + return Err(anyhow!("entry CAS replacement must preserve identity")); + } + self.with_database(|database| { + let write = database.begin_write()?; + let matched = { + let mut table = write.open_table(REDB_ENVELOPES)?; + if read_redb_entry(&table, expected)? != Some(expected.clone()) { + false + } else { + insert_redb_entry(&mut table, &replacement)?; + true + } + }; + if matched { + write.commit()?; + } else { + write.abort()?; + } + Ok(matched) + }) + } + + pub fn insert_entry_if_absent(&self, entry: CultCacheEnvelope) -> Result { + self.with_database(|database| { + let write = database.begin_write()?; + let inserted = { + let mut table = write.open_table(REDB_ENVELOPES)?; + let key = redb_identity(&entry)?; + if table.get(key.as_slice())?.is_some() { + false + } else { + insert_redb_entry(&mut table, &entry)?; + true + } + }; + if inserted { + write.commit()?; + } else { + write.abort()?; + } + Ok(inserted) + }) + } + + pub fn compare_and_swap_batch( + &self, + expected: &[CultCacheEnvelope], + replacements: Vec, + ) -> Result { + if replacements.is_empty() { + return Err(anyhow!( + "conditional batch requires a non-empty replacement set" + )); + } + let expected_ids = unique_batch_ids(expected, "expected")?; + let replacement_ids = unique_batch_ids(&replacements, "replacement")?; + if !expected_ids.is_subset(&replacement_ids) { + return Err(anyhow!( + "conditional batch must replace every expected identity" + )); + } + self.with_database(|database| { + let write = database.begin_write()?; + let matched = { + let mut table = write.open_table(REDB_ENVELOPES)?; + let mut valid = true; + for row in expected { + if read_redb_entry(&table, row)? != Some(row.clone()) { + valid = false; + break; + } + } + if valid { + for row in &replacements { + if !expected_ids.contains(&entry_id(row)) + && read_redb_entry(&table, row)?.is_some() + { + valid = false; + break; + } + } + } + if valid { + for row in &replacements { + insert_redb_entry(&mut table, row)?; + } + } + valid + }; + if matched { + write.commit()?; + } else { + write.abort()?; + } + Ok(matched) + }) + } + + pub fn append_if_snapshot_unchanged( + &self, + expected_snapshot: &[CultCacheEnvelope], + additions: Vec, + ) -> Result { + if additions.is_empty() { + return Err(anyhow!("conditional snapshot append requires additions")); + } + unique_batch_ids(expected_snapshot, "expected snapshot")?; + let addition_ids = unique_batch_ids(&additions, "snapshot additions")?; + self.with_database(|database| { + let write = database.begin_write()?; + let matched = { + let mut table = write.open_table(REDB_ENVELOPES)?; + let mut current = read_all_redb(&table)?; + let mut expected = expected_snapshot.to_vec(); + current.sort_by_key(entry_id); + expected.sort_by_key(entry_id); + let valid = current == expected + && !current + .iter() + .any(|row| addition_ids.contains(&entry_id(row))); + if valid { + for row in &additions { + insert_redb_entry(&mut table, row)?; + } + } + valid + }; + if matched { + write.commit()?; + } else { + write.abort()?; + } + Ok(matched) + }) + } + + pub fn replace_and_append_if_snapshot_unchanged( + &self, + expected_snapshot: &[CultCacheEnvelope], + replacements: Vec, + ) -> Result { + if replacements.is_empty() { + return Err(anyhow!( + "conditional snapshot replacement requires replacements" + )); + } + unique_batch_ids(expected_snapshot, "expected snapshot")?; + unique_batch_ids(&replacements, "snapshot replacements")?; + self.with_database(|database| { + let write = database.begin_write()?; + let matched = { + let mut table = write.open_table(REDB_ENVELOPES)?; + let mut current = read_all_redb(&table)?; + let mut expected = expected_snapshot.to_vec(); + current.sort_by_key(entry_id); + expected.sort_by_key(entry_id); + let valid = current == expected; + if valid { + for row in &replacements { + insert_redb_entry(&mut table, row)?; + } + } + valid + }; + if matched { + write.commit()?; + } else { + write.abort()?; + } + Ok(matched) + }) + } + + /// Atomically replaces and deletes rows only while the complete store + /// snapshot remains byte-exact. This is the compaction primitive for a + /// non-owning keyed handle: a stale reader cannot summarize one history + /// while deleting another. + pub fn replace_and_delete_if_snapshot_unchanged( + &self, + expected_snapshot: &[CultCacheEnvelope], + replacements: Vec, + deletions: &[CultCacheEnvelope], + ) -> Result { + if replacements.is_empty() || deletions.is_empty() { + return Err(anyhow!( + "conditional compaction requires replacements and deletions" + )); + } + let expected_ids = unique_batch_ids(expected_snapshot, "expected snapshot")?; + let replacement_ids = unique_batch_ids(&replacements, "replacement")?; + let deletion_ids = unique_batch_ids(deletions, "deletion")?; + if !deletion_ids.is_subset(&expected_ids) { + return Err(anyhow!( + "compaction deletion is absent from expected snapshot" + )); + } + if !deletion_ids.is_disjoint(&replacement_ids) { + return Err(anyhow!("compaction cannot replace and delete one identity")); + } + self.with_database(|database| { + let write = database.begin_write()?; + let matched = { + let mut table = write.open_table(REDB_ENVELOPES)?; + let mut current = read_all_redb(&table)?; + let mut expected = expected_snapshot.to_vec(); + current.sort_by_key(entry_id); + expected.sort_by_key(entry_id); + let mut valid = current == expected; + if valid { + for row in &replacements { + if !expected_ids.contains(&entry_id(row)) + && read_redb_entry(&table, row)?.is_some() + { + valid = false; + break; + } + } + } + if valid { + for identity in deletion_ids { + let key = rmp_serde::to_vec(&identity) + .context("failed to encode CultCache identity")?; + table.remove(key.as_slice())?; + } + for row in &replacements { + insert_redb_entry(&mut table, row)?; + } + } + valid + }; + if matched { + write.commit()?; + } else { + write.abort()?; + } + Ok(matched) + }) + } + + pub fn delete_batch_if_unchanged(&self, expected: &[CultCacheEnvelope]) -> Result { + if expected.is_empty() { + return Err(anyhow!( + "conditional delete requires a non-empty expected set" + )); + } + unique_batch_ids(expected, "delete expected")?; + self.with_database(|database| { + let write = database.begin_write()?; + let matched = { + let mut table = write.open_table(REDB_ENVELOPES)?; + let mut valid = true; + for row in expected { + if read_redb_entry(&table, row)? != Some(row.clone()) { + valid = false; + break; + } + } + if valid { + for row in expected { + let key = redb_identity(row)?; + table.remove(key.as_slice())?; + } + } + valid + }; + if matched { + write.commit()?; + } else { + write.abort()?; + } + Ok(matched) + }) + } +} + +struct OwnedRedbInner { + database: Database, + _owner_lock: File, + file_identity: String, +} + +/// A pinned redb store for one long-lived service owner. Unlike the transient +/// redb store, this handle holds the CultCache external lock, an open file +/// handle, and the redb database for its entire lifetime. Clones share that +/// exact ownership authority. +#[derive(Clone)] +pub struct OwnedRedbMessagePackBackingStore { + path: PathBuf, + inner: Arc, +} + +impl std::fmt::Debug for OwnedRedbMessagePackBackingStore { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("OwnedRedbMessagePackBackingStore") + .field("path", &self.path) + .field("file_identity", &self.inner.file_identity) + .finish_non_exhaustive() + } +} + +impl OwnedRedbMessagePackBackingStore { + pub fn new(path: impl Into) -> Result { + let path = path.into(); + if let Some(parent) = path + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + { + fs::create_dir_all(parent) + .with_context(|| format!("failed to create {}", parent.display()))?; + } + let lock_path = redb_lock_path(&path); + let owner_lock = OpenOptions::new() + .create(true) + .read(true) + .truncate(false) + .write(true) + .open(&lock_path) + .with_context(|| format!("failed to open {}", lock_path.display()))?; + fs2::FileExt::try_lock_exclusive(&owner_lock).with_context(|| { + format!( + "redb CultCache {} already has an active owner", + path.display() + ) + })?; + + // Give redb ownership of the already pinned file, structurally closing + // path substitution during database creation. The post-open identity + // comparison separately verifies that the pathname still names it. + let database_file = OpenOptions::new() + .create(true) + .read(true) + .truncate(false) + .write(true) + .open(&path) + .with_context(|| format!("failed to pin redb CultCache {}", path.display()))?; + let held_identity = file_identity_from_file(&database_file)?; + let database = Database::builder() + .create_file(database_file) + .with_context(|| format!("failed to open redb CultCache {}", path.display()))?; + let path_identity_file = File::open(&path)?; + let path_identity = file_identity_from_file(&path_identity_file)?; + if held_identity != path_identity { + bail!( + "redb CultCache path identity changed while opening: held {held_identity}, path {path_identity}" + ); + } + let write = database.begin_write()?; + { + write.open_table(REDB_ENVELOPES)?; + } + write.commit()?; + Ok(Self { + path, + inner: Arc::new(OwnedRedbInner { + database, + _owner_lock: owner_lock, + file_identity: held_identity, + }), + }) + } + + pub fn path(&self) -> &Path { + &self.path + } + + pub fn file_identity(&self) -> &str { + &self.inner.file_identity + } + + pub fn require_file_identity(&self, expected: &str) -> Result<()> { + if expected != self.file_identity() { + bail!( + "redb CultCache file identity mismatch: expected {expected}, owned {}", + self.file_identity() + ); + } + Ok(()) + } + + pub fn validate_path_identity(&self) -> Result<()> { + let current_file = File::open(&self.path) + .with_context(|| format!("owned redb path {} is missing", self.path.display()))?; + let current = file_identity_from_file(¤t_file)?; + self.require_file_identity(¤t).with_context(|| { + format!( + "owned redb path {} no longer names the pinned file", + self.path.display() + ) + }) + } + + pub fn compare_and_swap_entry( + &self, + expected: &CultCacheEnvelope, + replacement: CultCacheEnvelope, + ) -> Result { + if entry_id(expected) != entry_id(&replacement) { + return Err(anyhow!("entry CAS replacement must preserve identity")); + } + let write = self.inner.database.begin_write()?; + let matched = { + let mut table = write.open_table(REDB_ENVELOPES)?; + if read_redb_entry(&table, expected)? != Some(expected.clone()) { + false + } else { + insert_redb_entry(&mut table, &replacement)?; + true + } + }; + if matched { + write.commit()?; + } else { + write.abort()?; + } + Ok(matched) + } + + pub fn insert_entry_if_absent(&self, entry: CultCacheEnvelope) -> Result { + let write = self.inner.database.begin_write()?; + let inserted = { + let mut table = write.open_table(REDB_ENVELOPES)?; + let key = redb_identity(&entry)?; + if table.get(key.as_slice())?.is_some() { + false + } else { + insert_redb_entry(&mut table, &entry)?; + true + } + }; + if inserted { + write.commit()?; + } else { + write.abort()?; + } + Ok(inserted) + } + + pub fn compare_and_swap_batch( + &self, + expected: &[CultCacheEnvelope], + replacements: Vec, + ) -> Result { + if replacements.is_empty() { + return Err(anyhow!( + "conditional batch requires a non-empty replacement set" + )); + } + let expected_ids = unique_batch_ids(expected, "expected")?; + let replacement_ids = unique_batch_ids(&replacements, "replacement")?; + if !expected_ids.is_subset(&replacement_ids) { + return Err(anyhow!( + "conditional batch must replace every expected identity" + )); + } + let write = self.inner.database.begin_write()?; + let matched = { + let mut table = write.open_table(REDB_ENVELOPES)?; + let mut valid = true; + for row in expected { + if read_redb_entry(&table, row)? != Some(row.clone()) { + valid = false; + break; + } + } + if valid { + for row in &replacements { + if !expected_ids.contains(&entry_id(row)) + && read_redb_entry(&table, row)?.is_some() + { + valid = false; + break; + } + } + } + if valid { + for row in &replacements { + insert_redb_entry(&mut table, row)?; + } + } + valid + }; + if matched { + write.commit()?; + } else { + write.abort()?; + } + Ok(matched) + } + + pub fn append_if_snapshot_unchanged( + &self, + expected_snapshot: &[CultCacheEnvelope], + additions: Vec, + ) -> Result { + if additions.is_empty() { + return Err(anyhow!("conditional snapshot append requires additions")); + } + unique_batch_ids(expected_snapshot, "expected snapshot")?; + let addition_ids = unique_batch_ids(&additions, "snapshot additions")?; + let write = self.inner.database.begin_write()?; + let matched = { + let mut table = write.open_table(REDB_ENVELOPES)?; + let mut current = read_all_redb(&table)?; + let mut expected = expected_snapshot.to_vec(); + current.sort_by_key(entry_id); + expected.sort_by_key(entry_id); + let valid = current == expected + && !current + .iter() + .any(|row| addition_ids.contains(&entry_id(row))); + if valid { + for row in &additions { + insert_redb_entry(&mut table, row)?; + } + } + valid + }; + if matched { + write.commit()?; + } else { + write.abort()?; + } + Ok(matched) + } + + pub fn replace_and_append_if_snapshot_unchanged( + &self, + expected_snapshot: &[CultCacheEnvelope], + replacements: Vec, + ) -> Result { + if replacements.is_empty() { + return Err(anyhow!( + "conditional snapshot replacement requires replacements" + )); + } + unique_batch_ids(expected_snapshot, "expected snapshot")?; + unique_batch_ids(&replacements, "snapshot replacements")?; + let write = self.inner.database.begin_write()?; + let matched = { + let mut table = write.open_table(REDB_ENVELOPES)?; + let mut current = read_all_redb(&table)?; + let mut expected = expected_snapshot.to_vec(); + current.sort_by_key(entry_id); + expected.sort_by_key(entry_id); + let valid = current == expected; + if valid { + for row in &replacements { + insert_redb_entry(&mut table, row)?; + } + } + valid + }; + if matched { + write.commit()?; + } else { + write.abort()?; + } + Ok(matched) + } + + pub fn delete_batch_if_unchanged(&self, expected: &[CultCacheEnvelope]) -> Result { + if expected.is_empty() { + return Err(anyhow!( + "conditional delete requires a non-empty expected set" + )); + } + let expected_ids = unique_batch_ids(expected, "delete expected")?; + let write = self.inner.database.begin_write()?; + let matched = { + let mut table = write.open_table(REDB_ENVELOPES)?; + let mut valid = true; + for row in expected { + if read_redb_entry(&table, row)? != Some(row.clone()) { + valid = false; + break; + } + } + if valid { + for identity in expected_ids { + let key = rmp_serde::to_vec(&identity) + .context("failed to encode CultCache identity")?; + table.remove(key.as_slice())?; + } + } + valid + }; + if matched { + write.commit()?; + } else { + write.abort()?; + } + Ok(matched) + } + + /// Atomically replaces and deletes rows only while the complete store + /// snapshot remains byte-exact. This is the compaction primitive: a stale + /// reader cannot summarize one history while deleting another. + pub fn replace_and_delete_if_snapshot_unchanged( + &self, + expected_snapshot: &[CultCacheEnvelope], + replacements: Vec, + deletions: &[CultCacheEnvelope], + ) -> Result { + if replacements.is_empty() || deletions.is_empty() { + return Err(anyhow!( + "conditional compaction requires replacements and deletions" + )); + } + let expected_ids = unique_batch_ids(expected_snapshot, "expected snapshot")?; + let replacement_ids = unique_batch_ids(&replacements, "replacement")?; + let deletion_ids = unique_batch_ids(deletions, "deletion")?; + if !deletion_ids.is_subset(&expected_ids) { + return Err(anyhow!( + "compaction deletion is absent from expected snapshot" + )); + } + if !deletion_ids.is_disjoint(&replacement_ids) { + return Err(anyhow!("compaction cannot replace and delete one identity")); + } + let write = self.inner.database.begin_write()?; + let matched = { + let mut table = write.open_table(REDB_ENVELOPES)?; + let mut current = read_all_redb(&table)?; + let mut expected = expected_snapshot.to_vec(); + current.sort_by_key(entry_id); + expected.sort_by_key(entry_id); + let mut valid = current == expected; + if valid { + for row in &replacements { + if !expected_ids.contains(&entry_id(row)) + && read_redb_entry(&table, row)?.is_some() + { + valid = false; + break; + } + } + } + if valid { + for identity in deletion_ids { + let key = rmp_serde::to_vec(&identity) + .context("failed to encode CultCache identity")?; + table.remove(key.as_slice())?; + } + for row in &replacements { + insert_redb_entry(&mut table, row)?; + } + } + valid + }; + if matched { + write.commit()?; + } else { + write.abort()?; + } + Ok(matched) + } +} + +impl CacheBackingStore for OwnedRedbMessagePackBackingStore { + fn pull_all(&self) -> Result> { + let read = self.inner.database.begin_read()?; + let table = read.open_table(REDB_ENVELOPES)?; + read_all_redb(&table) + } + + fn push(&mut self, entry: &CultCacheEnvelope) -> Result<()> { + let write = self.inner.database.begin_write()?; + { + let mut table = write.open_table(REDB_ENVELOPES)?; + insert_redb_entry(&mut table, entry)?; + } + write.commit()?; + Ok(()) + } + + fn delete(&mut self, entry: &CultCacheEnvelope) -> Result<()> { + let write = self.inner.database.begin_write()?; + { + let mut table = write.open_table(REDB_ENVELOPES)?; + let key = redb_identity(entry)?; + table.remove(key.as_slice())?; + } + write.commit()?; + Ok(()) + } + + fn push_all(&mut self, entries: &[CultCacheEnvelope], _options: PushAllOptions) -> Result<()> { + unique_batch_ids(entries, "push all")?; + let write = self.inner.database.begin_write()?; + { + let mut table = write.open_table(REDB_ENVELOPES)?; + let keys = table + .iter()? + .map(|row| Ok(row?.0.value().to_vec())) + .collect::>>()?; + for key in keys { + table.remove(key.as_slice())?; + } + for entry in entries { + insert_redb_entry(&mut table, entry)?; + } + } + write.commit()?; + Ok(()) + } +} + +fn redb_lock_path(path: &Path) -> PathBuf { + let mut lock_name = path + .file_name() + .map(|value| value.to_os_string()) + .unwrap_or_else(|| "cultcache.cc".into()); + lock_name.push(".lock"); + path.with_file_name(lock_name) +} + +#[cfg(unix)] +fn file_identity_from_file(file: &File) -> Result { + use std::os::unix::fs::MetadataExt; + let metadata = file.metadata()?; + Ok(format!( + "unix:{:016x}:{:016x}", + metadata.dev(), + metadata.ino() + )) +} + +#[cfg(windows)] +fn file_identity_from_file(file: &File) -> Result { + use std::mem::zeroed; + use std::os::windows::io::AsRawHandle; + use windows_sys::Win32::Storage::FileSystem::{ + BY_HANDLE_FILE_INFORMATION, GetFileInformationByHandle, + }; + let mut information: BY_HANDLE_FILE_INFORMATION = unsafe { zeroed() }; + let success = + unsafe { GetFileInformationByHandle(file.as_raw_handle() as _, &mut information) }; + if success == 0 { + return Err(std::io::Error::last_os_error()) + .context("failed to read Windows file identity"); + } + let index = ((information.nFileIndexHigh as u64) << 32) | information.nFileIndexLow as u64; + Ok(format!( + "windows:{:08x}:{index:016x}", + information.dwVolumeSerialNumber + )) +} + +fn redb_identity(entry: &CultCacheEnvelope) -> Result> { + rmp_serde::to_vec(&entry_id(entry)).context("failed to encode CultCache identity") +} + +fn read_redb_entry( + table: &impl ReadableTable<&'static [u8], &'static [u8]>, + identity: &CultCacheEnvelope, +) -> Result> { + let key = redb_identity(identity)?; + table + .get(key.as_slice())? + .map(|value| { + rmp_serde::from_slice(value.value()).context("failed to decode redb CultCache envelope") + }) + .transpose() +} + +fn insert_redb_entry( + table: &mut redb::Table<&[u8], &[u8]>, + entry: &CultCacheEnvelope, +) -> Result<()> { + let key = redb_identity(entry)?; + let value = rmp_serde::to_vec(entry).context("failed to encode redb CultCache envelope")?; + table.insert(key.as_slice(), value.as_slice())?; + Ok(()) +} + +fn read_all_redb( + table: &impl ReadableTable<&'static [u8], &'static [u8]>, +) -> Result> { + let mut entries = table + .iter()? + .map(|row| { + let (_, value) = row?; + rmp_serde::from_slice(value.value()).context("failed to decode redb CultCache envelope") + }) + .collect::>>()?; + entries.sort_by_key(entry_id); + Ok(entries) +} + +impl CacheBackingStore for RedbMessagePackBackingStore { + fn pull_all(&self) -> Result> { + self.with_database(|database| { + let read = database.begin_read()?; + let table = read.open_table(REDB_ENVELOPES)?; + read_all_redb(&table) + }) + } + + fn push(&mut self, entry: &CultCacheEnvelope) -> Result<()> { + self.with_database(|database| { + let write = database.begin_write()?; + { + let mut table = write.open_table(REDB_ENVELOPES)?; + insert_redb_entry(&mut table, entry)?; + } + write.commit()?; + Ok(()) + }) + } + + fn delete(&mut self, entry: &CultCacheEnvelope) -> Result<()> { + self.with_database(|database| { + let write = database.begin_write()?; + { + let mut table = write.open_table(REDB_ENVELOPES)?; + let key = redb_identity(entry)?; + table.remove(key.as_slice())?; + } + write.commit()?; + Ok(()) + }) + } + + fn push_all(&mut self, entries: &[CultCacheEnvelope], _options: PushAllOptions) -> Result<()> { + unique_batch_ids(entries, "push all")?; + self.with_database(|database| { + let write = database.begin_write()?; + { + let mut table = write.open_table(REDB_ENVELOPES)?; + let keys = table + .iter()? + .map(|row| Ok(row?.0.value().to_vec())) + .collect::>>()?; + for key in keys { + table.remove(key.as_slice())?; + } + for entry in entries { + insert_redb_entry(&mut table, entry)?; + } + } + write.commit()?; + Ok(()) + }) + } +} + struct CultCacheStoreRegistration { store: Box, types: BTreeSet, @@ -332,7 +1906,7 @@ struct CultCacheStoreRegistration { pub struct CultCache { definitions: BTreeMap, schema_name_definitions: BTreeMap, - entries: BTreeMap, + entries: BTreeMap<(String, String), CultCacheEnvelope>, stores: Vec, } @@ -379,31 +1953,77 @@ impl CultCache { self.register_entry_type::() } + pub fn registered_entry_types(&self) -> Vec { + self.definitions.keys().cloned().collect() + } + pub fn register_registry(&mut self, registry: R) -> Result<&mut Self> { registry.register_entries(self)?; Ok(self) } + /// Makes the store home to `types`, or the generic store when `types` is + /// empty. A type has exactly one home store: a second generic store or a + /// type already claimed by another store is an error and attaches nothing. pub fn add_backing_store( &mut self, store: impl CacheBackingStore + 'static, types: impl IntoIterator>, - ) { + ) -> Result<()> { + let types: BTreeSet = types.into_iter().map(Into::into).collect(); + if types.is_empty() && self.stores.iter().any(|registration| registration.types.is_empty()) { + return Err(anyhow!( + "Backing store would be a second generic store; name the types it is home to" + )); + } + if let Some(claimed) = types + .iter() + .find(|type_id| self.stores.iter().any(|registration| registration.types.contains(*type_id))) + { + return Err(anyhow!( + "CultCache type {:?} is already routed to another backing store; it cannot also route to this one", + claimed + )); + } self.stores.push(CultCacheStoreRegistration { store: Box::new(store), - types: types.into_iter().map(Into::into).collect(), + types, }); + // A record's home cannot change after it is admitted. + let candidate = self.stores.len() - 1; + let held: BTreeSet = self.entries.keys().map(|(type_id, _)| type_id.clone()).collect(); + for type_id in &held { + let type_id = type_id.as_str(); + let after = self.resolve_route_index(type_id); + let before = self.stores[..candidate] + .iter() + .position(|registration| registration.types.contains(type_id)) + .or_else(|| self.stores[..candidate].iter().position(|registration| registration.types.is_empty())); + if before != after { + let error = anyhow!( + "Attaching this backing store would move {:?} from {} to {}; attach routed stores before the generic store", + type_id, + self.describe_store(before), + self.describe_store(after) + ); + self.stores.pop(); + return Err(error); + } + } + Ok(()) } - pub fn add_generic_backing_store(&mut self, store: impl CacheBackingStore + 'static) { - self.add_backing_store(store, Vec::::new()); + pub fn add_generic_backing_store(&mut self, store: impl CacheBackingStore + 'static) -> Result<()> { + self.add_backing_store(store, Vec::::new()) } + /// Reads every store into a staged image and replaces the cache's view + /// only when all of them loaded, so a refused load admits nothing. pub fn pull_all_backing_stores(&mut self) -> Result<()> { - self.entries.clear(); + let mut loaded = BTreeMap::new(); let known_types: BTreeSet = self.definitions.keys().cloned().collect(); let schema_name_definitions = self.schema_name_definitions.clone(); - for registration in &mut self.stores { + for (index, registration) in self.stores.iter().enumerate() { for mut entry in registration.store.pull_all()? { let Some(canonical_type) = resolve_registered_type( &known_types, @@ -415,13 +2035,31 @@ impl CultCache { entry.r#type )); }; + let home = self.resolve_route_index(&canonical_type); + if home != Some(index) { + return Err(anyhow!( + "CultCache {:?} record {:?} was loaded from {}, but its home is {}", + canonical_type, + entry.key, + self.describe_store(Some(index)), + self.describe_store(home) + )); + } entry.r#type = canonical_type; - self.entries.insert(entry_id(&entry), entry); + loaded.insert(entry_id(&entry), entry); } } + self.entries = loaded; Ok(()) } + /// Returns the exact envelopes in the currently loaded cache image. + /// Callers that need optimistic concurrency can validate typed documents + /// and capture CAS expectations from one coherent read. + pub fn snapshot_envelopes(&self) -> Vec { + self.entries.values().cloned().collect() + } + pub fn get(&self, key: &str) -> Result> { self.require_entry_type::()?; let Some(entry) = self.entries.get(&entry_id_parts(T::TYPE, key)) else { @@ -472,7 +2110,57 @@ impl CultCache { Ok(values) } + pub fn get_all_with_keys(&self) -> Result> { + self.require_entry_type::()?; + let mut values = Vec::new(); + for entry in self.entries.values() { + if entry.r#type != T::TYPE { + continue; + } + values.push(( + entry.key.clone(), + rmp_serde::from_slice(&entry.payload).with_context(|| { + format!( + "failed to decode CultCache entry {:?} at key {:?} as {}", + T::TYPE, + entry.key, + T::SCHEMA_NAME + ) + })?, + )); + } + Ok(values) + } + + pub fn soa(&self) -> Result> { + let keyed_rows = self.get_all_with_keys::()?; + let (keys, documents): (Vec<_>, Vec<_>) = keyed_rows.into_iter().unzip(); + let columns = T::soa_columns(&documents) + .into_iter() + .map(|(name, values)| (name.to_string(), values)) + .collect(); + Ok(CultSoaTable { + keys, + documents, + columns, + _document: PhantomData, + }) + } + pub fn put(&mut self, key: impl Into, value: &T) -> Result { + let (entry, parsed) = self.prepare_entry(key, value)?; + if let Some(home) = self.home_index(T::TYPE)? { + self.stores[home].store.push(&entry)?; + } + self.entries.insert(entry_id(&entry), entry); + Ok(parsed) + } + + pub fn prepare_entry( + &self, + key: impl Into, + value: &T, + ) -> Result<(CultCacheEnvelope, T)> { self.require_entry_type::()?; let key = key.into(); let payload = rmp_serde::to_vec(value).with_context(|| { @@ -483,6 +2171,36 @@ impl CultCache { T::SCHEMA_NAME ) })?; + self.finish_prepared_entry::(key, payload) + } + + /// Prepares a typed entry while encoding ordinary nested structs as + /// MessagePack maps. DatabaseEntry itself remains the stable numeric-slot + /// tuple; named nested fields prevent serde's omitted optional fields from + /// shifting later values into the wrong positions. + pub fn prepare_entry_named( + &self, + key: impl Into, + value: &T, + ) -> Result<(CultCacheEnvelope, T)> { + self.require_entry_type::()?; + let key = key.into(); + let payload = rmp_serde::to_vec_named(value).with_context(|| { + format!( + "failed to encode named CultCache entry {:?} at key {:?} as {}", + T::TYPE, + key, + T::SCHEMA_NAME + ) + })?; + self.finish_prepared_entry::(key, payload) + } + + fn finish_prepared_entry( + &self, + key: String, + payload: Vec, + ) -> Result<(CultCacheEnvelope, T)> { let parsed: T = rmp_serde::from_slice(&payload).with_context(|| { format!( "failed to validate CultCache entry {:?} at key {:?} as {}", @@ -498,22 +2216,106 @@ impl CultCache { stored_at: now_utc_second(), schema_id: Some(T::TYPE.to_string()), }; - let route = self.resolve_route_indices(T::TYPE); - let Some(primary_index) = route.first().copied() else { - return Err(anyhow!( - "No backing store is registered for entry type {:?}", - T::TYPE - )); - }; - self.stores[primary_index].store.push(&entry)?; - for mirror_index in route.iter().skip(1).copied() { - self.stores[mirror_index].store.push(&entry)?; + Ok((entry, parsed)) + } + + pub fn put_prepared_batch(&mut self, staged: Vec) -> Result<()> { + if staged.is_empty() { + return Err(anyhow!("CultCache batch must contain at least one entry")); + } + // Validate every staged record before any store is touched. + let mut store_index = None; + for entry in &staged { + if !self.definitions.contains_key(&entry.r#type) { + return Err(anyhow!( + "CultCache batch entry type {:?} is not registered", + entry.r#type + )); + } + if entry.key.trim().is_empty() { + return Err(anyhow!( + "CultCache envelope keys for type {:?} must be non-empty", + entry.r#type + )); + } + if entry.stored_at.trim().is_empty() { + return Err(anyhow!( + "CultCache envelope stored_at for type {:?} must be non-empty", + entry.r#type + )); + } + let Some(home) = self.home_index(&entry.r#type)? else { + continue; + }; + match store_index { + Some(expected) if expected != home => { + return Err(anyhow!("CultCache atomic batch cannot span backing stores")); + } + None => store_index = Some(home), + _ => {} + } + } + let mut next = self.entries.clone(); + for entry in &staged { + next.insert(entry_id(entry), entry.clone()); + } + if let Some(store_index) = store_index { + // The home store receives its own records only: those it already + // holds in this cache plus the staged ones. + let snapshot: Vec<_> = next + .values() + .filter(|entry| self.resolve_route_index(&entry.r#type) == Some(store_index)) + .cloned() + .collect(); + self.stores[store_index] + .store + .push_all(&snapshot, PushAllOptions::default())?; + } + self.entries = next; + Ok(()) + } + + pub fn put_envelope(&mut self, entry: CultCacheEnvelope) -> Result { + let parsed = self.validate_envelope::(&entry)?; + if let Some(home) = self.home_index(T::TYPE)? { + self.stores[home].store.push(&entry)?; } self.entries.insert(entry_id(&entry), entry); Ok(parsed) } - pub fn put_envelope(&mut self, entry: CultCacheEnvelope) -> Result { + /// Publishes an envelope for a registered dynamic document type without + /// claiming compile-time knowledge of its payload schema. This is the + /// CultMesh/schema-registry boundary: callers own payload validation before + /// admission; CultCache still enforces registered type, identity, timestamp, + /// routing, and persistence. + pub fn put_raw_envelope(&mut self, entry: CultCacheEnvelope) -> Result<()> { + if !self.definitions.contains_key(&entry.r#type) { + return Err(anyhow!( + "No entry type registered for CultCache envelope type {:?}", + entry.r#type + )); + } + if entry.key.trim().is_empty() { + return Err(anyhow!( + "CultCache envelope keys for type {:?} must be non-empty", + entry.r#type + )); + } + if entry.stored_at.trim().is_empty() { + return Err(anyhow!( + "CultCache envelope stored_at for type {:?} must be non-empty", + entry.r#type + )); + } + if let Some(home) = self.home_index(&entry.r#type)? { + self.stores[home].store.push(&entry)?; + } + self.entries.insert(entry_id(&entry), entry); + Ok(()) + } + + fn validate_envelope(&self, entry: &CultCacheEnvelope) -> Result { self.require_entry_type::()?; if entry.r#type != T::TYPE { return Err(anyhow!( @@ -535,27 +2337,14 @@ impl CultCache { )); } - let parsed: T = rmp_serde::from_slice(&entry.payload).with_context(|| { + rmp_serde::from_slice(&entry.payload).with_context(|| { format!( "failed to validate CultCache envelope {:?} at key {:?} as {}", T::TYPE, entry.key, T::SCHEMA_NAME ) - })?; - let route = self.resolve_route_indices(T::TYPE); - let Some(primary_index) = route.first().copied() else { - return Err(anyhow!( - "No backing store is registered for entry type {:?}", - T::TYPE - )); - }; - self.stores[primary_index].store.push(&entry)?; - for mirror_index in route.iter().skip(1).copied() { - self.stores[mirror_index].store.push(&entry)?; - } - self.entries.insert(entry_id(&entry), entry); - Ok(parsed) + }) } pub fn update(&mut self, key: &str, updater: F) -> Result @@ -573,16 +2362,8 @@ impl CultCache { let Some(entry) = self.entries.get(&id).cloned() else { return Ok(false); }; - let route = self.resolve_route_indices(T::TYPE); - let Some(primary_index) = route.first().copied() else { - return Err(anyhow!( - "No backing store is registered for entry type {:?}", - T::TYPE - )); - }; - self.stores[primary_index].store.delete(&entry)?; - for mirror_index in route.iter().skip(1).copied() { - self.stores[mirror_index].store.delete(&entry)?; + if let Some(home) = self.home_index(T::TYPE)? { + self.stores[home].store.delete(&entry)?; } self.entries.remove(&id); Ok(true) @@ -592,6 +2373,30 @@ impl CultCache { self.entries.values().cloned().collect() } + pub fn snapshot_soa(&self) -> CultCacheSoaTable { + CultCacheSoaTable::from_envelopes(self.snapshot()) + } + + /// Replaces the cache's view with the table's records. Every record is + /// checked (registered type, and a home store once any store is attached) + /// into a local image first, so a refused load admits nothing and leaves + /// the prior view intact. Loading writes no store. + pub fn load_soa(&mut self, table: CultCacheSoaTable) -> Result<()> { + let mut loaded = BTreeMap::new(); + for entry in table.into_envelopes()? { + if !self.definitions.contains_key(&entry.r#type) { + return Err(anyhow!( + "No schema is registered for SoA entry type {:?}", + entry.r#type + )); + } + self.home_index(&entry.r#type)?; + loaded.insert(entry_id(&entry), entry); + } + self.entries = loaded; + Ok(()) + } + fn require_entry_type(&self) -> Result<()> { match self.definitions.get(T::TYPE) { Some(schema_name) if *schema_name == T::SCHEMA_NAME => Ok(()), @@ -602,25 +2407,40 @@ impl CultCache { } } - fn resolve_route_indices(&self, type_id: &str) -> Vec { - let type_specific: Vec = self - .stores - .iter() - .enumerate() - .filter_map(|(index, registration)| { - registration.types.contains(type_id).then_some(index) - }) - .collect(); - if !type_specific.is_empty() { - return type_specific; - } + /// The store that claims the type, else the generic store. Registration + /// guarantees at most one of each. + fn resolve_route_index(&self, type_id: &str) -> Option { self.stores .iter() - .enumerate() - .filter_map(|(index, registration)| registration.types.is_empty().then_some(index)) - .collect() + .position(|registration| registration.types.contains(type_id)) + .or_else(|| { + self.stores + .iter() + .position(|registration| registration.types.is_empty()) + }) } + /// `None` for a cache with no stores, which is in memory. Once any store + /// is attached, a type with no home store is an error. + fn home_index(&self, type_id: &str) -> Result> { + if self.stores.is_empty() { + return Ok(None); + } + self.resolve_route_index(type_id) + .map(Some) + .ok_or_else(|| anyhow!("No backing store is registered for entry type {:?}", type_id)) + } + + fn describe_store(&self, index: Option) -> String { + match index { + None => "memory".to_string(), + Some(index) if self.stores[index].types.is_empty() => "the generic store".to_string(), + Some(index) => format!( + "the store routed to {:?}", + self.stores[index].types.iter().collect::>() + ), + } + } } impl Default for CultCache { @@ -629,47 +2449,40 @@ impl Default for CultCache { } } -fn entry_id(entry: &CultCacheEnvelope) -> String { +fn entry_id(entry: &CultCacheEnvelope) -> (String, String) { entry_id_parts(&entry.r#type, &entry.key) } -fn entry_id_parts(r#type: &str, key: &str) -> String { - format!("{type}::{key}", type = r#type) -} - -fn resolve_registered_type( - known_types: &BTreeSet, - schema_name_definitions: &BTreeMap, - persisted_type: &str, -) -> Option { - if known_types.contains(persisted_type) { - return Some(persisted_type.to_string()); - } - - schema_name_definitions.get(persisted_type).cloned() +fn entry_id_parts(r#type: &str, key: &str) -> (String, String) { + (r#type.to_string(), key.to_string()) } fn now_utc_second() -> String { chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Secs, true) } -fn encode_store_snapshot(entries: &[CultCacheEnvelope]) -> PersistedStoreSnapshot { - let mut schema_names = BTreeSet::::new(); +fn encode_store_snapshot(entries: &[CultCacheEnvelope]) -> Result { + let mut schema_types = BTreeMap::::new(); for entry in entries { - schema_names.insert( - entry - .schema_id - .clone() - .unwrap_or_else(|| entry.r#type.clone()), - ); + let schema_id = entry + .schema_id + .clone() + .unwrap_or_else(|| entry.r#type.clone()); + if let Some(existing_type) = schema_types.insert(schema_id.clone(), entry.r#type.clone()) { + ensure!( + existing_type == entry.r#type, + "CultCache schema {schema_id:?} cannot identify both {existing_type:?} and {:?}", + entry.r#type + ); + } } - let catalog = schema_names + let catalog = schema_types .into_iter() - .map(|schema_id| { + .map(|(schema_id, document_type)| { PersistedSchemaCatalogEntry( schema_id.clone(), - schema_id.clone(), + document_type, format!("{schema_id}.v1"), schema_id.clone(), format!( @@ -697,7 +2510,11 @@ fn encode_store_snapshot(entries: &[CultCacheEnvelope]) -> PersistedStoreSnapsho }) .collect(); - PersistedStoreSnapshot("cultcache.store.v1".to_string(), catalog, records) + Ok(PersistedStoreSnapshot( + "cultcache.store.v1".to_string(), + catalog, + records, + )) } fn decode_store_snapshot(bytes: &[u8]) -> Result> { @@ -719,13 +2536,20 @@ fn decode_store_snapshot(bytes: &[u8]) -> Result> { let r#type = match catalog.get(&record.1) { Some(schema_name) => schema_name.clone(), None => { - let schema_version = infer_schema_version_from_payload(&record.3).ok_or_else(|| { - anyhow!( - "CultCache record {:?} references missing schema {:?}", - record.0, - record.1 - ) - })?; + // A record can outlive its catalog entry when another runtime + // wrote it, or when the record is older than the catalog it + // was stamped against. The payload is a MessagePack array + // whose first field is the schema version, so the type can be + // recovered from the record itself rather than failing the + // whole snapshot. + let schema_version = + infer_schema_version_from_payload(&record.3).ok_or_else(|| { + anyhow!( + "CultCache record {:?} references missing schema {:?}", + record.0, + record.1 + ) + })?; infer_schema_name(&schema_version).ok_or_else(|| { anyhow!( "CultCache record {:?} references missing schema {:?}", @@ -746,6 +2570,21 @@ fn decode_store_snapshot(bytes: &[u8]) -> Result> { .collect() } +/// A record recovered from its payload carries a schema name, while the +/// registry is keyed by entry type. Accept either, so a snapshot whose catalog +/// entry is gone still resolves to the registered definition. +fn resolve_registered_type( + known_types: &BTreeSet, + schema_name_definitions: &BTreeMap, + persisted_type: &str, +) -> Option { + if known_types.contains(persisted_type) { + return Some(persisted_type.to_string()); + } + + schema_name_definitions.get(persisted_type).cloned() +} + fn infer_schema_version_from_payload(payload: &[u8]) -> Option { let mut offset = 0usize; read_array_header(payload, &mut offset)?; @@ -816,188 +2655,162 @@ fn temporary_path_for(path: &Path) -> PathBuf { let mut file_name = path .file_name() .map(|value| value.to_os_string()) - .unwrap_or_else(|| "cultcache.msgpack".into()); + .unwrap_or_else(|| "cultcache.cc".into()); file_name.push(format!(".{}.tmp", uuid::Uuid::new_v4())); path.with_file_name(file_name) } -fn is_canonical_candidate_uuid(value: &[u8]) -> bool { - if value.len() != 36 || !value.is_ascii() { - return false; - } - let Ok(value) = std::str::from_utf8(value) else { - return false; +fn remove_abandoned_staging_files(destination: &Path) -> Result<()> { + let Some(parent) = destination.parent() else { + return Ok(()); }; - let Ok(uuid) = uuid::Uuid::parse_str(value) else { - return false; + let Some(destination_name) = destination.file_name().and_then(|value| value.to_str()) else { + return Ok(()); }; - uuid.hyphenated().to_string() == value -} - -#[cfg(unix)] -fn is_owned_candidate_name(target_name: &OsStr, candidate_name: &OsStr) -> bool { - use std::os::unix::ffi::OsStrExt; - - let target = target_name.as_bytes(); - let candidate = candidate_name.as_bytes(); - let Some(remainder) = candidate.strip_prefix(target) else { - return false; - }; - let Some(remainder) = remainder.strip_prefix(b".") else { - return false; - }; - let Some(uuid) = remainder.strip_suffix(b".tmp") else { - return false; - }; - is_canonical_candidate_uuid(uuid) -} - -#[cfg(windows)] -fn is_owned_candidate_name(target_name: &OsStr, candidate_name: &OsStr) -> bool { - use std::os::windows::ffi::OsStrExt; - - let target = target_name.encode_wide().collect::>(); - let candidate = candidate_name.encode_wide().collect::>(); - let Some(remainder) = candidate.strip_prefix(target.as_slice()) else { - return false; - }; - let Some(remainder) = remainder.strip_prefix(&[b'.' as u16]) else { - return false; - }; - let suffix = [b'.' as u16, b't' as u16, b'm' as u16, b'p' as u16]; - let Some(uuid_wide) = remainder.strip_suffix(&suffix) else { - return false; - }; - let Ok(uuid) = uuid_wide - .iter() - .map(|value| u8::try_from(*value)) - .collect::, _>>() - else { - return false; - }; - is_canonical_candidate_uuid(&uuid) -} - -#[cfg(not(any(unix, windows)))] -fn is_owned_candidate_name(target_name: &OsStr, candidate_name: &OsStr) -> bool { - let (Some(target), Some(candidate)) = (target_name.to_str(), candidate_name.to_str()) else { - return false; - }; - let Some(remainder) = candidate.strip_prefix(target) else { - return false; - }; - let Some(remainder) = remainder.strip_prefix('.') else { - return false; - }; - let Some(uuid) = remainder.strip_suffix(".tmp") else { - return false; - }; - is_canonical_candidate_uuid(uuid.as_bytes()) -} - -fn stale_candidate_paths(path: &Path) -> Result> { - let parent = path.parent().unwrap_or_else(|| Path::new(".")); - if !parent.exists() { - return Ok(Vec::new()); - } - let target_name = path - .file_name() - .unwrap_or_else(|| OsStr::new("cultcache.msgpack")); - let mut candidates = Vec::new(); - for entry in - fs::read_dir(parent).with_context(|| format!("failed to inspect {}", parent.display()))? - { - let entry = entry.with_context(|| format!("failed to inspect {}", parent.display()))?; - let name = entry.file_name(); - if is_owned_candidate_name(target_name, &name) { - candidates.push(entry.path()); + let prefix = format!("{destination_name}."); + for entry in fs::read_dir(parent).with_context(|| { + format!( + "failed to inspect {} for abandoned staging files", + parent.display() + ) + })? { + let entry = entry?; + let Some(name) = entry.file_name().to_str().map(str::to_owned) else { + continue; + }; + let Some(candidate) = name + .strip_prefix(&prefix) + .and_then(|value| value.strip_suffix(".tmp")) + else { + continue; + }; + if uuid::Uuid::parse_str(candidate).is_err() || !entry.file_type()?.is_file() { + continue; } - } - Ok(candidates) -} - -fn remove_stale_candidates(path: &Path) -> Result<()> { - for candidate in stale_candidate_paths(path)? { - fs::remove_file(&candidate) - .with_context(|| format!("failed to remove stale candidate {}", candidate.display()))?; + fs::remove_file(entry.path()).with_context(|| { + format!( + "failed to remove abandoned staging file {}", + entry.path().display() + ) + })?; } Ok(()) } -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum SnapshotReplacementStage { - CandidateSynced, - TargetReplaced, -} - #[cfg(unix)] -fn replace_file(candidate: &Path, target: &Path) -> Result<()> { - fs::rename(candidate, target).with_context(|| { +fn replace_file_atomically(staged: &Path, destination: &Path) -> Result<()> { + fs::rename(staged, destination).with_context(|| { format!( "failed to atomically replace {} with {}", - target.display(), - candidate.display() + destination.display(), + staged.display() ) }) } #[cfg(windows)] -fn replace_file(candidate: &Path, target: &Path) -> Result<()> { +fn replace_file_atomically(staged: &Path, destination: &Path) -> Result<()> { use std::os::windows::ffi::OsStrExt; use windows_sys::Win32::Storage::FileSystem::{ MOVEFILE_REPLACE_EXISTING, MOVEFILE_WRITE_THROUGH, MoveFileExW, }; - let candidate_wide = candidate + let staged_wide = staged .as_os_str() .encode_wide() .chain(std::iter::once(0)) .collect::>(); - let target_wide = target + let destination_wide = destination .as_os_str() .encode_wide() .chain(std::iter::once(0)) .collect::>(); - let replaced = unsafe { - MoveFileExW( - candidate_wide.as_ptr(), - target_wide.as_ptr(), - MOVEFILE_REPLACE_EXISTING | MOVEFILE_WRITE_THROUGH, - ) - }; - if replaced == 0 { - return Err(std::io::Error::last_os_error()).with_context(|| { - format!( - "failed to atomically replace {} with {}", - target.display(), - candidate.display() + // A concurrent reader of the destination (another process opening the + // store, an indexer, an antivirus scan) makes the replace fail with a + // sharing or access error for as long as it holds the handle. That is + // transient by construction; a daemon that treats it as fatal dies on + // the first overlap (Muninn serve did, 2026-09-11). Retry briefly. + const ERROR_ACCESS_DENIED: i32 = 5; + const ERROR_SHARING_VIOLATION: i32 = 32; + const ERROR_LOCK_VIOLATION: i32 = 33; + const ATTEMPTS: u32 = 25; + const BACKOFF: std::time::Duration = std::time::Duration::from_millis(20); + let mut attempt = 0; + loop { + let success = unsafe { + MoveFileExW( + staged_wide.as_ptr(), + destination_wide.as_ptr(), + MOVEFILE_REPLACE_EXISTING | MOVEFILE_WRITE_THROUGH, ) - }); + }; + if success != 0 { + return Ok(()); + } + let error = std::io::Error::last_os_error(); + attempt += 1; + let transient = matches!( + error.raw_os_error(), + Some(ERROR_ACCESS_DENIED | ERROR_SHARING_VIOLATION | ERROR_LOCK_VIOLATION) + ); + if !transient || attempt >= ATTEMPTS { + return Err(error).with_context(|| { + format!( + "failed to atomically replace {} with {} after {attempt} attempt(s)", + destination.display(), + staged.display() + ) + }); + } + std::thread::sleep(BACKOFF); } - Ok(()) } -#[cfg(not(any(unix, windows)))] -fn replace_file(candidate: &Path, target: &Path) -> Result<()> { - fs::rename(candidate, target).with_context(|| { - format!( - "failed to replace {} with {}", - target.display(), - candidate.display() - ) - }) +#[cfg(all(test, windows))] +mod windows_replace_tests { + use super::replace_file_atomically; + use std::os::windows::fs::OpenOptionsExt; + + /// A reader holding the destination open with no sharing makes the + /// replace fail for as long as it holds on; it must succeed once the + /// reader lets go, not report the first refusal as the outcome. + #[test] + fn a_replace_waits_out_a_reader_that_holds_the_destination() { + let dir = std::env::temp_dir().join(format!("cultcache-replace-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let destination = dir.join("store.cc"); + let staged = dir.join("store.cc.tmp"); + std::fs::write(&destination, b"old").unwrap(); + std::fs::write(&staged, b"new").unwrap(); + let held = std::fs::OpenOptions::new() + .read(true) + .share_mode(0) + .open(&destination) + .unwrap(); + let releaser = std::thread::spawn(move || { + std::thread::sleep(std::time::Duration::from_millis(120)); + drop(held); + }); + replace_file_atomically(&staged, &destination).expect("the replace outlasts the reader"); + releaser.join().unwrap(); + assert_eq!(std::fs::read(&destination).unwrap(), b"new"); + let _ = std::fs::remove_dir_all(&dir); + } } #[cfg(unix)] -fn sync_parent_directory(parent: &Path) -> Result<()> { +fn sync_parent_directory(path: &Path) -> Result<()> { + let Some(parent) = path.parent() else { + return Ok(()); + }; File::open(parent) - .with_context(|| format!("failed to open directory {}", parent.display()))? - .sync_all() - .with_context(|| format!("failed to flush directory {}", parent.display())) + .and_then(|directory| directory.sync_all()) + .with_context(|| format!("failed to sync {}", parent.display())) } -#[cfg(not(unix))] -fn sync_parent_directory(_parent: &Path) -> Result<()> { +#[cfg(windows)] +fn sync_parent_directory(_path: &Path) -> Result<()> { + // MoveFileExW with MOVEFILE_WRITE_THROUGH owns publication durability. Ok(()) } @@ -1005,6 +2818,37 @@ fn sync_parent_directory(_parent: &Path) -> Result<()> { mod tests { use super::*; use pretty_assertions::assert_eq; + use std::sync::Arc; + use std::sync::atomic::{AtomicBool, Ordering}; + use std::sync::mpsc; + use std::time::Duration; + + struct RefusingBatchStore { + inner: SingleFileMessagePackBackingStore, + refuse_batch: Arc, + } + + impl CacheBackingStore for RefusingBatchStore { + fn pull_all(&self) -> Result> { + self.inner.pull_all() + } + fn push(&mut self, entry: &CultCacheEnvelope) -> Result<()> { + self.inner.push(entry) + } + fn delete(&mut self, entry: &CultCacheEnvelope) -> Result<()> { + self.inner.delete(entry) + } + fn push_all( + &mut self, + entries: &[CultCacheEnvelope], + options: PushAllOptions, + ) -> Result<()> { + if self.refuse_batch.load(Ordering::SeqCst) { + return Err(anyhow!("injected batch refusal")); + } + self.inner.push_all(entries, options) + } + } #[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] #[cultcache(type = "settings")] @@ -1024,6 +2868,24 @@ mod tests { body: String, } + #[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] + #[cultcache(type = "binary-record")] + struct BinaryRecord { + #[cultcache(key = 0, bytes)] + required: Vec, + #[cultcache(key = 1, bytes, default)] + optional: Vec, + } + + #[derive(Clone, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)] + struct NestedOptionalValue { + label: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + count: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + symbol: Option, + } + #[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] #[cultcache(type = "runtime-policy", schema = "tests.schema_stamped_entry")] struct SchemaStamped { @@ -1035,269 +2897,94 @@ mod tests { value: String, } + #[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] + #[cultcache(type = "nested-note")] + struct NestedNote { + #[cultcache(key = 0)] + value: NestedOptionalValue, + } + + cultcache_soa!(Settings { + "theme" => |row: &Settings| row.theme.clone(), + "retries" => |row: &Settings| row.retries, + }); + cultcache_registry!(TestEntries { Settings, Note }); - fn test_envelope(value: &str) -> CultCacheEnvelope { - CultCacheEnvelope { - key: "state".to_string(), - r#type: "test-state".to_string(), - payload: value.as_bytes().to_vec(), - stored_at: "2026-07-15T00:00:00Z".to_string(), - schema_id: Some("tests.test_state.v1".to_string()), - } - } - #[test] - fn interruption_before_replacement_preserves_committed_snapshot() -> Result<()> { + fn captured_snapshot_cannot_outlive_a_shared_consequence_gate() -> Result<()> { let temp = tempfile::tempdir()?; - let store_path = temp.path().join("cache.msgpack"); + let store_path = temp.path().join("authority.cc"); let mut store = SingleFileMessagePackBackingStore::new(&store_path); - let old = test_envelope("old"); - let new = test_envelope("new"); - store.push_all(std::slice::from_ref(&old), PushAllOptions::default())?; - - let result = store.write_all_unlocked_with_observer(std::slice::from_ref(&new), |stage| { - assert_eq!(stage, SnapshotReplacementStage::CandidateSynced); - assert!( - store_path.exists(), - "the committed pathname must never be removed" - ); - Err(anyhow!("simulated interruption before replacement")) - }); - - assert!(result.is_err()); - let recovered = store.pull_all()?; - assert_eq!(recovered.len(), 1); - assert_eq!(recovered[0].payload, old.payload); - assert_eq!(stale_candidate_paths(&store_path)?.len(), 1); - - store.push_all(std::slice::from_ref(&new), PushAllOptions::default())?; - let recovered = store.pull_all()?; - assert_eq!(recovered.len(), 1); - assert_eq!(recovered[0].payload, new.payload); - assert!(stale_candidate_paths(&store_path)?.is_empty()); - Ok(()) - } - - #[test] - fn interruption_after_replacement_recovers_new_snapshot() -> Result<()> { - let temp = tempfile::tempdir()?; - let store_path = temp.path().join("cache.msgpack"); - let mut store = SingleFileMessagePackBackingStore::new(&store_path); - let old = test_envelope("old"); - let new = test_envelope("new"); - store.push_all(std::slice::from_ref(&old), PushAllOptions::default())?; - - let result = store.write_all_unlocked_with_observer(std::slice::from_ref(&new), |stage| { - if stage == SnapshotReplacementStage::TargetReplaced { - return Err(anyhow!("simulated interruption after replacement")); - } - Ok(()) - }); - - let error = result.expect_err("the injected post-replacement failure must surface"); - let uncertain = error - .downcast_ref::() - .expect("post-replacement failures must be typed as committed but uncertain"); - assert_eq!(uncertain.path(), store_path.as_path()); - assert!(uncertain.detail().contains("simulated interruption")); - let recovered = store.pull_all()?; - assert_eq!(recovered.len(), 1); - assert_eq!(recovered[0].payload, new.payload); - assert!(stale_candidate_paths(&store_path)?.is_empty()); - Ok(()) - } - - #[test] - fn first_write_interruption_before_replacement_leaves_store_uncommitted() -> Result<()> { - let temp = tempfile::tempdir()?; - let store_path = temp.path().join("cache.msgpack"); - let store = SingleFileMessagePackBackingStore::new(&store_path); - let new = test_envelope("new"); - - let result = store.write_all_unlocked_with_observer(std::slice::from_ref(&new), |stage| { - assert_eq!(stage, SnapshotReplacementStage::CandidateSynced); - Err(anyhow!("simulated interruption before first replacement")) - }); - - assert!(result.is_err()); - assert!(!store_path.exists()); - assert!(store.pull_all()?.is_empty()); - assert_eq!(stale_candidate_paths(&store_path)?.len(), 1); - Ok(()) - } - - #[test] - fn first_write_interruption_after_replacement_exposes_complete_snapshot() -> Result<()> { - let temp = tempfile::tempdir()?; - let store_path = temp.path().join("cache.msgpack"); - let store = SingleFileMessagePackBackingStore::new(&store_path); - let new = test_envelope("new"); - - let result = store.write_all_unlocked_with_observer(std::slice::from_ref(&new), |stage| { - if stage == SnapshotReplacementStage::TargetReplaced { - return Err(anyhow!("simulated interruption after first replacement")); - } - Ok(()) - }); - - let error = result.expect_err("the injected post-replacement failure must surface"); - assert!( - error - .downcast_ref::() - .is_some() - ); - let recovered = store.pull_all()?; - assert_eq!(recovered.len(), 1); - assert_eq!(recovered[0].payload, new.payload); - assert!(stale_candidate_paths(&store_path)?.is_empty()); - Ok(()) - } - - #[test] - fn unlock_failure_never_masks_the_action_outcome() -> Result<()> { - let temp = tempfile::tempdir()?; - let store_path = temp.path().join("cache.msgpack"); - let store = SingleFileMessagePackBackingStore::new(&store_path); - - let completed = store.with_exclusive_lock_using_unlock( - || Ok(42_u32), - |_| Err(std::io::Error::other("simulated unlock failure")), - )?; - assert_eq!(completed, 42); - - let uncertain: Result<()> = store.with_exclusive_lock_using_unlock( - || { - Err(anyhow::Error::new( - CommittedSnapshotDurabilityUncertain::new( - store_path.clone(), - "primary post-commit failure", - ), - )) - }, - |_| Err(std::io::Error::other("secondary unlock failure")), - ); - let error = uncertain.expect_err("the action error must survive unlock cleanup"); - let error = error - .downcast_ref::() - .expect("the primary typed outcome must not be masked"); - assert!(error.detail().contains("primary post-commit failure")); - assert!(!error.detail().contains("secondary unlock failure")); - - // Both injected unlock failures still close their lock handles. A new - // exclusive operation must therefore acquire the lock normally. - store.with_exclusive_lock(|| Ok(()))?; - Ok(()) - } - - #[test] - fn stale_cleanup_preserves_non_uuid_lookalikes() -> Result<()> { - let temp = tempfile::tempdir()?; - let store_path = temp.path().join("cache.msgpack"); - let lookalike = temp.path().join("cache.msgpack.not-a-uuid.tmp"); - fs::write(&lookalike, b"belongs to somebody else")?; - - let mut store = SingleFileMessagePackBackingStore::new(&store_path); - store.push_all(&[test_envelope("new")], PushAllOptions::default())?; - - assert_eq!(fs::read(&lookalike)?, b"belongs to somebody else"); - assert!(stale_candidate_paths(&store_path)?.is_empty()); - Ok(()) - } - - #[cfg(unix)] - #[test] - fn stale_candidate_symlink_is_removed_without_clobbering_its_target() -> Result<()> { - use std::os::unix::fs::symlink; - - let temp = tempfile::tempdir()?; - let store_path = temp.path().join("cache.msgpack"); - let victim_path = temp.path().join("victim.txt"); - fs::write(&victim_path, b"untouched")?; - let stale_path = temporary_path_for(&store_path); - symlink(&victim_path, &stale_path)?; - - let mut store = SingleFileMessagePackBackingStore::new(&store_path); - store.push_all(&[test_envelope("new")], PushAllOptions::default())?; - - assert_eq!(fs::read(&victim_path)?, b"untouched"); - assert!(!stale_path.exists()); - assert!(stale_candidate_paths(&store_path)?.is_empty()); - Ok(()) - } - - #[cfg(unix)] - #[test] - fn non_utf8_target_names_recover_their_own_candidates() -> Result<()> { - use std::ffi::OsString; - use std::os::unix::ffi::OsStringExt; - - let temp = tempfile::tempdir()?; - let store_name = OsString::from_vec(b"cache-\xff.msgpack".to_vec()); - let store_path = temp.path().join(store_name); - let mut store = SingleFileMessagePackBackingStore::new(&store_path); - let new = test_envelope("new"); - - let interrupted = store - .write_all_unlocked_with_observer(std::slice::from_ref(&new), |_| { - Err(anyhow!("simulated interruption")) - }); - assert!(interrupted.is_err()); - assert_eq!(stale_candidate_paths(&store_path)?.len(), 1); - - store.push_all(std::slice::from_ref(&new), PushAllOptions::default())?; - assert!(stale_candidate_paths(&store_path)?.is_empty()); - assert_eq!(store.pull_all()?[0].payload, new.payload); - Ok(()) - } - - #[test] - fn two_contending_writers_leave_one_complete_snapshot() -> Result<()> { - use std::sync::{Arc, Barrier}; - - let temp = tempfile::tempdir()?; - let store_path = temp.path().join("cache.msgpack"); - let barrier = Arc::new(Barrier::new(3)); - let spawn_writer = |marker: u8| { - let store_path = store_path.clone(); - let barrier = Arc::clone(&barrier); - std::thread::spawn(move || -> Result<()> { - let mut entries = Vec::new(); - for index in 0..128 { - let mut entry = test_envelope(std::str::from_utf8(&[marker]).unwrap()); - entry.key = format!("state-{index:03}"); - entry.payload = vec![marker; 4096]; - entries.push(entry); - } - let mut store = SingleFileMessagePackBackingStore::new(store_path); - barrier.wait(); - store.push_all(&entries, PushAllOptions::default()) - }) + let released = CultCacheEnvelope { + key: "brake".into(), + r#type: "brake".into(), + payload: b"released".to_vec(), + stored_at: "2026-07-20T00:00:00Z".into(), + schema_id: Some("brake".into()), }; + store.push(&released)?; - let writer_a = spawn_writer(b'a'); - let writer_b = spawn_writer(b'b'); - barrier.wait(); - writer_a.join().expect("writer A panicked")?; - writer_b.join().expect("writer B panicked")?; + let writer_path = store_path.clone(); + let (started_tx, started_rx) = mpsc::channel(); + let (finished_tx, finished_rx) = mpsc::channel(); + store.with_read_only_shared_snapshot(|captured| { + assert_eq!(captured, vec![released.clone()]); + std::thread::spawn(move || { + started_tx.send(()).unwrap(); + let engaged = CultCacheEnvelope { + payload: b"engaged".to_vec(), + ..released + }; + let mut writer = SingleFileMessagePackBackingStore::new(writer_path); + writer.push(&engaged).unwrap(); + finished_tx.send(()).unwrap(); + }); + started_rx.recv_timeout(Duration::from_secs(1))?; + assert!( + finished_rx + .recv_timeout(Duration::from_millis(100)) + .is_err() + ); + Ok(()) + })?; + finished_rx.recv_timeout(Duration::from_secs(1))?; + assert_eq!(store.pull_all()?[0].payload, b"engaged"); + Ok(()) + } - let recovered = SingleFileMessagePackBackingStore::new(&store_path).pull_all()?; - assert_eq!(recovered.len(), 128); - let winner = recovered[0].payload[0]; - assert!(winner == b'a' || winner == b'b'); - assert!( - recovered - .iter() - .all(|entry| entry.payload == vec![winner; 4096]) + #[test] + fn post_action_unlock_failure_is_diagnostic_not_action_result() -> Result<()> { + let temp = tempfile::tempdir()?; + let store_path = temp.path().join("authority.cc"); + let mut store = SingleFileMessagePackBackingStore::new(&store_path); + store.push(&CultCacheEnvelope { + key: "brake".into(), + r#type: "brake".into(), + payload: b"released".to_vec(), + stored_at: "2026-07-20T00:00:00Z".into(), + schema_id: Some("brake".into()), + })?; + + let mut diagnostic = None; + let result = store.with_read_only_shared_snapshot_using( + |_| Ok(42), + |_| Err(anyhow!("injected post-action unlock failure")), + |error| diagnostic = Some(error.to_string()), + ); + + assert_eq!(result?, 42); + assert_eq!( + diagnostic.as_deref(), + Some("injected post-action unlock failure") ); - assert!(stale_candidate_paths(&store_path)?.is_empty()); Ok(()) } #[test] fn familiar_cultcache_flow_persists_and_reloads_typed_documents() -> Result<()> { let temp = tempfile::tempdir()?; - let store_path = temp.path().join("cache.msgpack"); + let store_path = temp.path().join("cache.cc"); let settings = Settings { theme: "ash".to_string(), retries: 3, @@ -1305,14 +2992,14 @@ mod tests { let mut cache = CultCache::new(); cache.register_entry_type::()?; - cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&store_path)); + cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&store_path))?; cache.pull_all_backing_stores()?; cache.put("app", &settings)?; assert_eq!(cache.get_required::("app")?, settings); let mut reloaded = CultCache::new(); reloaded.register_entry_type::()?; - reloaded.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&store_path)); + reloaded.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&store_path))?; reloaded.pull_all_backing_stores()?; assert_eq!(reloaded.get_required::("app")?, settings); Ok(()) @@ -1321,10 +3008,10 @@ mod tests { #[test] fn entry_identity_is_polymorphic_by_type_and_key() -> Result<()> { let temp = tempfile::tempdir()?; - let store_path = temp.path().join("cache.msgpack"); + let store_path = temp.path().join("cache.cc"); let mut cache = CultCache::new(); cache.register_registry(TestEntries)?; - cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&store_path)); + cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&store_path))?; cache.put( "shared", @@ -1347,19 +3034,759 @@ mod tests { Ok(()) } + #[test] + fn prepared_batch_publishes_heterogeneous_entries_in_one_snapshot() -> Result<()> { + let temp = tempfile::tempdir()?; + let store_path = temp.path().join("batch.cc"); + let mut cache = CultCache::new(); + cache.register_registry(TestEntries)?; + cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&store_path))?; + + let (settings, _) = cache.prepare_entry( + "app", + &Settings { + theme: "iron".to_string(), + retries: 4, + }, + )?; + let (note, _) = cache.prepare_entry( + "receipt", + &Note { + title: "committed".to_string(), + body: "same snapshot".to_string(), + }, + )?; + cache.put_prepared_batch(vec![settings, note])?; + + let mut reloaded = CultCache::new(); + reloaded.register_registry(TestEntries)?; + reloaded.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&store_path))?; + reloaded.pull_all_backing_stores()?; + assert_eq!(reloaded.get_required::("app")?.theme, "iron"); + assert_eq!(reloaded.get_required::("receipt")?.title, "committed"); + Ok(()) + } + + #[test] + fn named_preparation_preserves_omitted_nested_optional_slots() -> Result<()> { + let temp = tempfile::tempdir()?; + let store_path = temp.path().join("named.cc"); + let mut cache = CultCache::new(); + cache.register_entry_type::()?; + cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&store_path))?; + cache.pull_all_backing_stores()?; + let note = NestedNote { + value: NestedOptionalValue { + label: "nested".into(), + count: None, + symbol: Some("kept-in-its-own-field".into()), + }, + }; + assert!(cache.prepare_entry("compact", ¬e).is_err()); + let (entry, parsed) = cache.prepare_entry_named("named", ¬e)?; + assert_eq!(parsed, note); + cache.put_prepared_batch(vec![entry])?; + assert_eq!(cache.get_required::("named")?, note); + Ok(()) + } + + #[test] + fn pulling_an_absent_store_does_not_create_its_parent_or_lock() -> Result<()> { + let temp = tempfile::tempdir()?; + let missing_parent = temp.path().join("missing-body"); + let store_path = missing_parent.join("cache.cc"); + let store = SingleFileMessagePackBackingStore::new(&store_path); + + assert!(store.pull_all()?.is_empty()); + assert!(!missing_parent.exists()); + assert!(!store_path.with_file_name("cache.cc.lock").exists()); + Ok(()) + } + + #[test] + fn provider_snapshot_read_does_not_create_or_open_a_lock() -> Result<()> { + let temp = tempfile::tempdir()?; + let store_path = temp.path().join("provider.cc"); + let lock_path = temp.path().join("provider.cc.lock"); + let mut store = SingleFileMessagePackBackingStore::new(&store_path); + let expected = test_envelope("provider.delivery", "one", b"payload"); + store.push(&expected)?; + std::fs::remove_file(&lock_path)?; + + assert_eq!(store.pull_all_read_only_snapshot()?, vec![expected]); + assert!(!lock_path.exists()); + Ok(()) + } + + #[test] + fn next_exclusive_write_removes_only_exact_abandoned_staging_files() -> Result<()> { + let temp = tempfile::tempdir()?; + let store_path = temp.path().join("cache.cc"); + let abandoned = temporary_path_for(&store_path); + let unknown = temp.path().join("cache.cc.not-a-uuid.tmp"); + let foreign = temp + .path() + .join("other.cc.00000000-0000-0000-0000-000000000000.tmp"); + fs::write(&abandoned, b"partial staging bytes")?; + fs::write(&unknown, b"operator-owned unknown file")?; + fs::write(&foreign, b"foreign store staging")?; + + let mut store = SingleFileMessagePackBackingStore::new(&store_path); + let expected = test_envelope("model", "current", b"one"); + store.push(&expected)?; + + assert!(!abandoned.exists()); + assert_eq!(fs::read(&unknown)?, b"operator-owned unknown file"); + assert_eq!(fs::read(&foreign)?, b"foreign store staging"); + assert_eq!(store.pull_all()?, vec![expected]); + Ok(()) + } + + #[test] + fn failed_snapshot_publication_preserves_the_committed_destination() -> Result<()> { + let temp = tempfile::tempdir()?; + let destination = temp.path().join("cache.cc"); + let absent_staging = temp.path().join("absent.tmp"); + let committed = b"committed snapshot"; + fs::write(&destination, committed)?; + + assert!(replace_file_atomically(&absent_staging, &destination).is_err()); + assert_eq!(fs::read(&destination)?, committed); + Ok(()) + } + + #[test] + fn snapshot_publication_atomically_replaces_an_existing_destination() -> Result<()> { + let temp = tempfile::tempdir()?; + let destination = temp.path().join("cache.cc"); + let staged = temp.path().join("cache.cc.staged"); + fs::write(&destination, b"old snapshot")?; + fs::write(&staged, b"new snapshot")?; + + replace_file_atomically(&staged, &destination)?; + + assert_eq!(fs::read(&destination)?, b"new snapshot"); + assert!(!staged.exists()); + Ok(()) + } + + #[test] + #[ignore = "operator SIGKILL probe; requires CULTCACHE_INTERRUPTION_STORE"] + fn snapshot_publication_interruption_probe_writer() -> Result<()> { + let store_path = std::env::var_os("CULTCACHE_INTERRUPTION_STORE") + .map(PathBuf::from) + .ok_or_else(|| anyhow!("CULTCACHE_INTERRUPTION_STORE is required"))?; + let marker_path = std::env::var_os("CULTCACHE_INTERRUPTION_MARKER") + .map(PathBuf::from) + .ok_or_else(|| anyhow!("CULTCACHE_INTERRUPTION_MARKER is required"))?; + let mut store = SingleFileMessagePackBackingStore::new(store_path); + for sequence in 0_u64.. { + store.push(&test_envelope( + "interruption.probe", + "current", + &rmp_serde::to_vec(&sequence)?, + ))?; + fs::write(&marker_path, sequence.to_string())?; + } + unreachable!("the interruption writer is stopped by the operator") + } + + #[test] + #[ignore = "operator SIGKILL probe; requires CULTCACHE_INTERRUPTION_STORE"] + fn snapshot_publication_interruption_probe_reader() -> Result<()> { + let store_path = std::env::var_os("CULTCACHE_INTERRUPTION_STORE") + .map(PathBuf::from) + .ok_or_else(|| anyhow!("CULTCACHE_INTERRUPTION_STORE is required"))?; + let rows = SingleFileMessagePackBackingStore::new(store_path).pull_all()?; + assert_eq!(rows.len(), 1); + assert_eq!(rows[0].r#type, "interruption.probe"); + assert_eq!(rows[0].key, "current"); + let _: u64 = rmp_serde::from_slice(&rows[0].payload)?; + Ok(()) + } + + fn test_envelope(r#type: &str, key: &str, payload: &[u8]) -> CultCacheEnvelope { + CultCacheEnvelope { + key: key.to_string(), + r#type: r#type.to_string(), + payload: payload.to_vec(), + stored_at: "2026-07-13T00:00:00Z".to_string(), + schema_id: Some(r#type.to_string()), + } + } + + #[test] + fn redb_store_crud_and_reopen_preserve_polymorphic_rows() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = temp.path().join("keyed.cc"); + let model = test_envelope("model", "shared", b"one"); + let receipt = test_envelope("receipt", "shared", b"two"); + { + let mut store = RedbMessagePackBackingStore::new(&path)?; + store.push(&model)?; + store.push(&receipt)?; + assert_eq!(store.pull_all()?.len(), 2); + store.delete(&receipt)?; + } + let reopened = RedbMessagePackBackingStore::new(&path)?; + assert_eq!(reopened.pull_all()?, vec![model]); + Ok(()) + } + + #[test] + fn redb_exact_cas_refuses_stale_value_without_mutation() -> Result<()> { + let temp = tempfile::tempdir()?; + let mut store = RedbMessagePackBackingStore::new(temp.path().join("cas.cc"))?; + let current = test_envelope("model", "current", b"one"); + store.push(¤t)?; + let stale = test_envelope("model", "current", b"stale"); + let replacement = test_envelope("model", "current", b"two"); + assert!(!store.compare_and_swap_entry(&stale, replacement.clone())?); + assert_eq!(store.pull_all()?, vec![current.clone()]); + assert!(store.compare_and_swap_entry(¤t, replacement.clone())?); + assert_eq!(store.pull_all()?, vec![replacement]); + Ok(()) + } + + #[test] + fn redb_batch_is_atomic_and_stale_member_refuses_every_write() -> Result<()> { + let temp = tempfile::tempdir()?; + let mut store = RedbMessagePackBackingStore::new(temp.path().join("batch.cc"))?; + let first = test_envelope("model", "first", b"one"); + let second = test_envelope("model", "second", b"one"); + store.push(&first)?; + store.push(&second)?; + let stale_second = test_envelope("model", "second", b"stale"); + let first_two = test_envelope("model", "first", b"two"); + let second_two = test_envelope("model", "second", b"two"); + let receipt = test_envelope("receipt", "batch", b"committed"); + assert!(!store.compare_and_swap_batch( + &[first.clone(), stale_second], + vec![first_two.clone(), second_two.clone(), receipt.clone()], + )?); + assert_eq!(store.pull_all()?, vec![first.clone(), second.clone()]); + assert!(store.compare_and_swap_batch( + &[first, second], + vec![first_two.clone(), second_two.clone(), receipt.clone()], + )?); + assert_eq!(store.pull_all()?, vec![first_two, second_two, receipt]); + Ok(()) + } + + #[test] + fn redb_compaction_refuses_stale_snapshot_and_atomically_replaces_and_deletes() -> Result<()> { + let temp = tempfile::tempdir()?; + let mut store = RedbMessagePackBackingStore::new(temp.path().join("compact.cc"))?; + let old_head = test_envelope("retention", "current", b"one"); + let retired = test_envelope("receipt", "old", b"terminal"); + store.push(&old_head)?; + store.push(&retired)?; + let stale = store.pull_all()?; + let concurrent = test_envelope("receipt", "running", b"active"); + assert!(store.insert_entry_if_absent(concurrent.clone())?); + let new_head = test_envelope("retention", "current", b"two"); + assert!(!store.replace_and_delete_if_snapshot_unchanged( + &stale, + vec![new_head.clone()], + std::slice::from_ref(&retired), + )?); + assert!(store.pull_all()?.contains(&retired)); + let exact = store.pull_all()?; + assert!(store.replace_and_delete_if_snapshot_unchanged( + &exact, + vec![new_head.clone()], + &[retired], + )?); + assert_eq!(store.pull_all()?, vec![concurrent, new_head]); + Ok(()) + } + + #[test] + fn redb_entry_cas_does_not_depend_on_unrelated_snapshot_changes() -> Result<()> { + let temp = tempfile::tempdir()?; + let store = RedbMessagePackBackingStore::new(temp.path().join("independent.cc"))?; + let current = test_envelope("model", "current", b"one"); + assert!(store.insert_entry_if_absent(current.clone())?); + let unrelated = test_envelope("event", "later", b"noise"); + assert!(store.insert_entry_if_absent(unrelated.clone())?); + let replacement = test_envelope("model", "current", b"two"); + assert!(store.compare_and_swap_entry(¤t, replacement.clone())?); + assert_eq!(store.pull_all()?, vec![unrelated, replacement]); + Ok(()) + } + + #[test] + fn redb_fresh_handles_share_one_database_for_the_same_cultcache_path() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = temp.path().join("shared.cc"); + let mut first = RedbMessagePackBackingStore::new(&path)?; + let second = RedbMessagePackBackingStore::new(&path)?; + let initial = test_envelope("model", "current", b"one"); + first.push(&initial)?; + assert_eq!(second.pull_all()?, vec![initial.clone()]); + let replacement = test_envelope("model", "current", b"two"); + assert!(second.compare_and_swap_entry(&initial, replacement.clone())?); + assert_eq!(first.pull_all()?, vec![replacement]); + Ok(()) + } + + #[test] + fn owned_redb_clones_share_authority_and_fresh_owner_is_refused() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = temp.path().join("owned.cc"); + let owner = OwnedRedbMessagePackBackingStore::new(&path)?; + let clone = owner.clone(); + assert_eq!(owner.file_identity(), clone.file_identity()); + assert!(OwnedRedbMessagePackBackingStore::new(&path).is_err()); + drop(owner); + assert!(OwnedRedbMessagePackBackingStore::new(&path).is_err()); + drop(clone); + assert!(OwnedRedbMessagePackBackingStore::new(&path).is_ok()); + Ok(()) + } + + #[test] + fn owned_redb_file_identity_is_stable_across_writes_and_reopen() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = temp.path().join("identity.cc"); + let identity = { + let mut owner = OwnedRedbMessagePackBackingStore::new(&path)?; + let identity = owner.file_identity().to_string(); + owner.push(&test_envelope("model", "current", b"one"))?; + owner.validate_path_identity()?; + owner.require_file_identity(&identity)?; + identity + }; + let reopened = OwnedRedbMessagePackBackingStore::new(&path)?; + assert_eq!(reopened.file_identity(), identity); + assert_eq!(reopened.pull_all()?.len(), 1); + Ok(()) + } + + #[test] + fn owned_redb_identity_helper_refuses_mismatch() -> Result<()> { + let temp = tempfile::tempdir()?; + let owner = OwnedRedbMessagePackBackingStore::new(temp.path().join("mismatch.cc"))?; + assert!(owner.require_file_identity("not-the-owned-file").is_err()); + owner.require_file_identity(owner.file_identity())?; + Ok(()) + } + + #[test] + fn owned_redb_batch_refuses_stale_member_and_commits_success_atomically() -> Result<()> { + let temp = tempfile::tempdir()?; + let mut owner = OwnedRedbMessagePackBackingStore::new(temp.path().join("batch-owned.cc"))?; + let first = test_envelope("model", "first", b"one"); + let second = test_envelope("model", "second", b"one"); + owner.push(&first)?; + owner.push(&second)?; + let first_two = test_envelope("model", "first", b"two"); + let second_two = test_envelope("model", "second", b"two"); + let companion = test_envelope("receipt", "batch", b"committed"); + let stale_second = test_envelope("model", "second", b"stale"); + assert!(!owner.compare_and_swap_batch( + &[first.clone(), stale_second], + vec![first_two.clone(), second_two.clone(), companion.clone()], + )?); + assert_eq!(owner.pull_all()?, vec![first.clone(), second.clone()]); + assert!(owner.compare_and_swap_batch( + &[first, second], + vec![first_two.clone(), second_two.clone(), companion.clone()], + )?); + assert_eq!(owner.pull_all()?, vec![first_two, second_two, companion]); + Ok(()) + } + + #[test] + fn owned_redb_compaction_refuses_stale_snapshot_and_atomically_replaces_and_deletes() + -> Result<()> { + let temp = tempfile::tempdir()?; + let mut owner = + OwnedRedbMessagePackBackingStore::new(temp.path().join("compact-owned.cc"))?; + let old_head = test_envelope("retention", "current", b"one"); + let retired_a = test_envelope("claim", "a", b"terminal"); + let retired_b = test_envelope("receipt", "a", b"terminal"); + owner.push(&old_head)?; + owner.push(&retired_a)?; + owner.push(&retired_b)?; + let stale = owner.pull_all()?; + let concurrent = test_envelope("claim", "concurrent", b"running"); + assert!(owner.insert_entry_if_absent(concurrent.clone())?); + let new_head = test_envelope("retention", "current", b"two"); + assert!(!owner.replace_and_delete_if_snapshot_unchanged( + &stale, + vec![new_head.clone()], + &[retired_a.clone(), retired_b.clone()], + )?); + assert!(owner.pull_all()?.contains(&retired_a)); + let exact = owner.pull_all()?; + assert!(owner.replace_and_delete_if_snapshot_unchanged( + &exact, + vec![new_head.clone()], + &[retired_a, retired_b], + )?); + assert_eq!(owner.pull_all()?, vec![concurrent, new_head]); + Ok(()) + } + + #[test] + fn owned_redb_path_replacement_cannot_redirect_pinned_authority() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = temp.path().join("pinned.cc"); + let displaced = temp.path().join("displaced.cc"); + let mut owner = OwnedRedbMessagePackBackingStore::new(&path)?; + let identity = owner.file_identity().to_string(); + owner.push(&test_envelope("model", "before", b"one"))?; + fs::rename(&path, &displaced)?; + File::create(&path)?; + assert!(owner.validate_path_identity().is_err()); + assert_eq!(owner.file_identity(), identity); + owner.push(&test_envelope("model", "after", b"two"))?; + assert_eq!(owner.pull_all()?.len(), 2); + assert_ne!( + file_identity_from_file(&File::open(&path)?)?, + owner.file_identity() + ); + Ok(()) + } + + #[test] + fn redb_overlapping_fresh_handles_serialize_the_database_open_interval() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = temp.path().join("contended.cc"); + let first = RedbMessagePackBackingStore::new(&path)?; + let second = RedbMessagePackBackingStore::new(&path)?; + let barrier = std::sync::Arc::new(std::sync::Barrier::new(3)); + let first_barrier = barrier.clone(); + let first_thread = std::thread::spawn(move || -> Result<()> { + first_barrier.wait(); + for index in 0..32 { + first.insert_entry_if_absent(test_envelope( + "event", + &format!("first-{index}"), + b"first", + ))?; + } + Ok(()) + }); + let second_barrier = barrier.clone(); + let second_thread = std::thread::spawn(move || -> Result<()> { + second_barrier.wait(); + for index in 0..32 { + second.insert_entry_if_absent(test_envelope( + "event", + &format!("second-{index}"), + b"second", + ))?; + } + Ok(()) + }); + barrier.wait(); + first_thread.join().expect("first redb writer")?; + second_thread.join().expect("second redb writer")?; + assert_eq!( + RedbMessagePackBackingStore::new(path)?.pull_all()?.len(), + 64 + ); + Ok(()) + } + + #[test] + fn redb_accepts_a_bare_relative_cultcache_path() -> Result<()> { + let name = format!("cultcache-redb-relative-{}.cc", uuid::Uuid::new_v4()); + let path = PathBuf::from(&name); + let lock_path = PathBuf::from(format!("{name}.lock")); + let store = RedbMessagePackBackingStore::new(&path)?; + assert!(store.insert_entry_if_absent(test_envelope("model", "current", b"one"))?); + assert_eq!(store.pull_all()?.len(), 1); + fs::remove_file(path)?; + fs::remove_file(lock_path)?; + Ok(()) + } + + #[test] + fn conditional_batch_replaces_expected_and_inserts_companion_atomically() -> Result<()> { + let temp = tempfile::tempdir()?; + let store_path = temp.path().join("conditional-batch.cc"); + let mut store = SingleFileMessagePackBackingStore::new(&store_path); + let current = test_envelope("model", "current", b"revision-1"); + store.push(¤t)?; + let replacement = test_envelope("model", "current", b"revision-2"); + let companion = test_envelope("receipt", "migration-1", b"applied"); + + assert!(store.compare_and_swap_batch( + std::slice::from_ref(¤t), + vec![replacement.clone(), companion.clone()], + )?); + assert_eq!(store.pull_all()?, vec![replacement, companion]); + Ok(()) + } + + #[test] + fn conditional_batch_allows_atomic_first_import_when_identities_are_absent() -> Result<()> { + let temp = tempfile::tempdir()?; + let store_path = temp.path().join("conditional-import.cc"); + let store = SingleFileMessagePackBackingStore::new(&store_path); + let model = test_envelope("model", "current", b"revision-1"); + let receipt = test_envelope("receipt", "migration-1", b"applied"); + + assert!(store.compare_and_swap_batch(&[], vec![model.clone(), receipt.clone()])?); + assert_eq!(store.pull_all()?, vec![model, receipt]); + Ok(()) + } + + #[test] + fn conditional_batch_stale_expected_and_companion_collision_preserve_exact_bytes() -> Result<()> + { + let temp = tempfile::tempdir()?; + let store_path = temp.path().join("conditional-refusal.cc"); + let mut store = SingleFileMessagePackBackingStore::new(&store_path); + let current = test_envelope("model", "current", b"revision-1"); + let occupied = test_envelope("receipt", "migration-1", b"someone-else"); + store.push(¤t)?; + store.push(&occupied)?; + let before = fs::read(&store_path)?; + + let stale = test_envelope("model", "current", b"stale-copy"); + let replacement = test_envelope("model", "current", b"revision-2"); + let fresh_companion = test_envelope("receipt", "migration-2", b"applied"); + assert!( + !store.compare_and_swap_batch(&[stale], vec![replacement.clone(), fresh_companion],)? + ); + assert_eq!(fs::read(&store_path)?, before); + + assert!(!store.compare_and_swap_batch( + &[current], + vec![ + replacement, + test_envelope("receipt", "migration-1", b"collision") + ], + )?); + assert_eq!(fs::read(&store_path)?, before); + Ok(()) + } + + #[test] + fn conditional_batch_rejects_empty_replacements_and_duplicate_identities() -> Result<()> { + let temp = tempfile::tempdir()?; + let store = SingleFileMessagePackBackingStore::new(temp.path().join("invalid-batch.cc")); + let entry = test_envelope("model", "current", b"one"); + + assert!( + store + .compare_and_swap_batch(&[entry.clone()], Vec::new()) + .is_err() + ); + assert!( + store + .compare_and_swap_batch(&[], vec![entry.clone(), entry.clone()]) + .is_err() + ); + assert!( + store + .compare_and_swap_batch( + &[entry.clone(), entry.clone()], + vec![test_envelope("model", "current", b"two")], + ) + .is_err() + ); + assert!(!store.path().exists()); + Ok(()) + } + + #[test] + fn full_snapshot_replacement_and_append_refuses_concurrent_rows() -> Result<()> { + let temp = tempfile::tempdir()?; + let snapshot_path = temp.path().join("snapshot-replace-append.cc"); + let mut snapshot_store = SingleFileMessagePackBackingStore::new(&snapshot_path); + let current = test_envelope("session", "current", b"active"); + snapshot_store.push(¤t)?; + let stale_snapshot = snapshot_store.pull_all()?; + let concurrent = test_envelope("event", "concurrent", b"live"); + snapshot_store.push(&concurrent)?; + let completed = test_envelope("session", "current", b"completed"); + let receipt = test_envelope("receipt", "terminal", b"done"); + assert!(!snapshot_store.replace_and_append_if_snapshot_unchanged( + &stale_snapshot, + vec![completed.clone(), receipt.clone()], + )?); + assert_eq!( + snapshot_store.pull_all()?, + vec![concurrent.clone(), current.clone()] + ); + let exact_snapshot = snapshot_store.pull_all()?; + assert!(snapshot_store.replace_and_append_if_snapshot_unchanged( + &exact_snapshot, + vec![completed.clone(), receipt.clone()], + )?); + assert_eq!( + snapshot_store.pull_all()?, + vec![concurrent.clone(), receipt.clone(), completed.clone()] + ); + + let keyed_path = temp.path().join("keyed-replace-append.redb"); + let mut keyed_store = RedbMessagePackBackingStore::new(&keyed_path)?; + keyed_store.push(¤t)?; + let stale_snapshot = keyed_store.pull_all()?; + keyed_store.push(&concurrent)?; + assert!(!keyed_store.replace_and_append_if_snapshot_unchanged( + &stale_snapshot, + vec![completed.clone(), receipt.clone()], + )?); + let exact_snapshot = keyed_store.pull_all()?; + assert!(keyed_store.replace_and_append_if_snapshot_unchanged( + &exact_snapshot, + vec![completed.clone(), receipt.clone()], + )?); + assert_eq!( + keyed_store.pull_all()?, + vec![concurrent, receipt, completed] + ); + Ok(()) + } + + #[test] + fn single_file_compaction_atomically_replaces_summary_and_deletes_history() -> Result<()> { + let temp = tempfile::tempdir()?; + let store_path = temp.path().join("conditional-compaction.cc"); + let mut store = SingleFileMessagePackBackingStore::new(&store_path); + let history = test_envelope("history", "old", b"closed"); + let head = test_envelope("retention", "head", b"revision-1"); + store.push(&history)?; + store.push(&head)?; + let snapshot = store.pull_all()?; + let next_head = test_envelope("retention", "head", b"revision-2"); + + assert!(store.replace_and_delete_if_snapshot_unchanged( + &snapshot, + vec![next_head.clone()], + std::slice::from_ref(&history), + )?); + assert_eq!(store.pull_all()?, vec![next_head]); + + let before = fs::read(&store_path)?; + assert!(!store.replace_and_delete_if_snapshot_unchanged( + &snapshot, + vec![test_envelope("retention", "head", b"revision-3")], + std::slice::from_ref(&history), + )?); + assert_eq!(fs::read(&store_path)?, before); + Ok(()) + } + + #[test] + fn snapshot_append_fences_concurrent_unknown_identity() -> Result<()> { + let temp = tempfile::tempdir()?; + let store_path = temp.path().join("snapshot-append.cc"); + let mut store = SingleFileMessagePackBackingStore::new(&store_path); + let model = test_envelope("model", "current", b"revision-1"); + store.push(&model)?; + let expected = store.pull_all()?; + let duplicate_admission = test_envelope("admission", "unexpected", b"also-current"); + store.push(&duplicate_admission)?; + let before = fs::read(&store_path)?; + + assert!(!store.append_if_snapshot_unchanged( + &expected, + vec![test_envelope("readiness", "proof", b"ready")], + )?); + assert_eq!(fs::read(&store_path)?, before); + Ok(()) + } + + #[test] + fn snapshot_append_is_order_insensitive_and_refuses_collisions() -> Result<()> { + let temp = tempfile::tempdir()?; + let store_path = temp.path().join("snapshot-order.cc"); + let mut store = SingleFileMessagePackBackingStore::new(&store_path); + let a = test_envelope("a", "one", b"1"); + let b = test_envelope("b", "two", b"2"); + store.push(&a)?; + store.push(&b)?; + assert!(store.append_if_snapshot_unchanged( + &[b.clone(), a.clone()], + vec![test_envelope("proof", "three", b"3")], + )?); + let current = store.pull_all()?; + assert!(!store.append_if_snapshot_unchanged( + ¤t, + vec![test_envelope("a", "one", b"collision")], + )?); + assert!( + store + .append_if_snapshot_unchanged(¤t, Vec::new()) + .is_err() + ); + Ok(()) + } + + #[test] + fn type_and_key_identity_is_not_delimiter_ambiguous() -> Result<()> { + let temp = tempfile::tempdir()?; + let store_path = temp.path().join("delimiter-identity.cc"); + let mut store = SingleFileMessagePackBackingStore::new(&store_path); + let left = test_envelope("a::b", "c", b"left"); + let right = test_envelope("a", "b::c", b"right"); + store.push(&left)?; + store.push(&right)?; + let rows = store.pull_all()?; + assert_eq!(rows.len(), 2); + assert!(rows.contains(&left)); + assert!(rows.contains(&right)); + Ok(()) + } + + #[test] + fn refused_batch_preserves_prior_snapshot() -> Result<()> { + let temp = tempfile::tempdir()?; + let store_path = temp.path().join("refused-batch.cc"); + let refusal = Arc::new(AtomicBool::new(false)); + let mut cache = CultCache::new(); + cache.register_registry(TestEntries)?; + cache.add_generic_backing_store(RefusingBatchStore { + inner: SingleFileMessagePackBackingStore::new(&store_path), + refuse_batch: refusal.clone(), + })?; + cache.put( + "app", + &Settings { + theme: "before".to_string(), + retries: 1, + }, + )?; + let (note, _) = cache.prepare_entry( + "receipt", + &Note { + title: "after".to_string(), + body: "must not land".to_string(), + }, + )?; + refusal.store(true, Ordering::SeqCst); + assert!(cache.put_prepared_batch(vec![note]).is_err()); + assert!(cache.get::("receipt")?.is_none()); + + let mut reloaded = CultCache::new(); + reloaded.register_registry(TestEntries)?; + reloaded.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&store_path))?; + reloaded.pull_all_backing_stores()?; + assert_eq!(reloaded.get_required::("app")?.theme, "before"); + assert!(reloaded.get::("receipt")?.is_none()); + Ok(()) + } + #[test] fn type_specific_store_routes_before_generic_store() -> Result<()> { let temp = tempfile::tempdir()?; - let generic_path = temp.path().join("generic.msgpack"); - let settings_path = temp.path().join("settings.msgpack"); + let generic_path = temp.path().join("generic.cc"); + let settings_path = temp.path().join("settings.cc"); let mut cache = CultCache::new(); cache.register_entry_type::()?; cache.register_entry_type::()?; - cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&generic_path)); cache.add_backing_store( SingleFileMessagePackBackingStore::new(&settings_path), ["settings"], - ); + )?; + cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&generic_path))?; cache.put( "app", @@ -1384,12 +3811,231 @@ mod tests { } #[test] - fn update_and_delete_follow_the_cache_api() -> Result<()> { + fn second_generic_store_is_rejected() -> Result<()> { let temp = tempfile::tempdir()?; - let store_path = temp.path().join("cache.msgpack"); + let mut cache = CultCache::new(); + cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(temp.path().join("a.cc")))?; + let error = cache + .add_generic_backing_store(SingleFileMessagePackBackingStore::new(temp.path().join("b.cc"))) + .unwrap_err(); + assert!(error.to_string().contains("second generic store"), "{error}"); + assert_eq!(cache.stores.len(), 1); + Ok(()) + } + + #[test] + fn type_claimed_twice_is_rejected() -> Result<()> { + let temp = tempfile::tempdir()?; + let mut cache = CultCache::new(); + cache.add_backing_store(SingleFileMessagePackBackingStore::new(temp.path().join("a.cc")), ["settings"])?; + let error = cache + .add_backing_store( + SingleFileMessagePackBackingStore::new(temp.path().join("b.cc")), + ["note", "settings"], + ) + .unwrap_err(); + assert!(error.to_string().contains("already routed"), "{error}"); + // A refused registration claims nothing, so "note" is still free. + cache.add_backing_store(SingleFileMessagePackBackingStore::new(temp.path().join("c.cc")), ["note"])?; + assert_eq!(cache.stores.len(), 2); + Ok(()) + } + + #[test] + fn attach_that_would_move_a_held_record_home_is_refused() -> Result<()> { + let temp = tempfile::tempdir()?; + let generic_path = temp.path().join("generic.cc"); + let settings_path = temp.path().join("settings.cc"); let mut cache = CultCache::new(); cache.register_entry_type::()?; - cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&store_path)); + cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&generic_path))?; + cache.put("app", &Settings { theme: "v1-generic".to_string(), retries: 1 })?; + + let error = cache + .add_backing_store(SingleFileMessagePackBackingStore::new(&settings_path), ["settings"]) + .unwrap_err(); + assert!( + error + .to_string() + .contains("would move \"settings\" from the generic store to the store routed to [\"settings\"]"), + "{error}" + ); + assert_eq!(cache.stores.len(), 1); + // Nothing attached: the next write still lands in the generic store. + cache.put("app", &Settings { theme: "v2-generic".to_string(), retries: 2 })?; + let generic_entries = SingleFileMessagePackBackingStore::new(&generic_path).pull_all()?; + assert_eq!(generic_entries.len(), 1); + assert_eq!(generic_entries[0].r#type, "settings"); + assert!(!settings_path.exists()); + Ok(()) + } + + #[test] + fn load_from_a_store_that_is_not_the_record_home_is_refused() -> Result<()> { + let temp = tempfile::tempdir()?; + let generic_path = temp.path().join("generic.cc"); + let settings_path = temp.path().join("settings.cc"); + let mut writer = CultCache::new(); + writer.register_entry_type::()?; + writer.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&generic_path))?; + writer.put("app", &Settings { theme: "stray".to_string(), retries: 0 })?; + let generic_bytes = fs::read(&generic_path)?; + + let mut cache = CultCache::new(); + cache.register_entry_type::()?; + cache.add_backing_store(SingleFileMessagePackBackingStore::new(&settings_path), ["settings"])?; + cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&generic_path))?; + // The cache already holds a record, in its home store, before the refused load. + cache.put("held", &Settings { theme: "home".to_string(), retries: 0 })?; + let before = cache.snapshot(); + let settings_bytes = fs::read(&settings_path)?; + let error = cache.pull_all_backing_stores().unwrap_err(); + assert!( + error.to_string().contains( + "\"settings\" record \"app\" was loaded from the generic store, but its home is the store routed to [\"settings\"]" + ), + "{error}" + ); + assert_eq!(cache.snapshot(), before); + assert_eq!(cache.get_required::("held")?.theme, "home"); + assert_eq!(fs::read(&generic_path)?, generic_bytes); + assert_eq!(fs::read(&settings_path)?, settings_bytes); + Ok(()) + } + + #[test] + fn refused_pull_leaves_prior_view_intact() -> Result<()> { + let temp = tempfile::tempdir()?; + let store_path = temp.path().join("generic.cc"); + let mut cache = CultCache::new(); + cache.register_entry_type::()?; + cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&store_path))?; + cache.put("app", &Settings { theme: "before".to_string(), retries: 1 })?; + let before = cache.snapshot(); + + // A record of a type this cache never registered lands in the store behind its back. + SingleFileMessagePackBackingStore::new(&store_path).push(&CultCacheEnvelope { + key: "stray".into(), + r#type: "unregistered".into(), + payload: rmp_serde::to_vec(&("x",))?, + stored_at: "2026-09-14T00:00:00Z".into(), + schema_id: Some("unregistered".into()), + })?; + let error = cache.pull_all_backing_stores().unwrap_err(); + assert!(error.to_string().contains("\"unregistered\""), "{error}"); + assert_eq!(cache.snapshot(), before); + assert_eq!(cache.get_required::("app")?.theme, "before"); + Ok(()) + } + + #[test] + fn refused_soa_load_leaves_prior_view_intact() -> Result<()> { + let mut source = CultCache::new(); + source.register_registry(TestEntries)?; + source.put("app", &Settings { theme: "ash".to_string(), retries: 3 })?; + source.put("memo", &Note { title: "machine".to_string(), body: "awake".to_string() })?; + let table = source.snapshot_soa(); + + // A type with no registration. + let mut unregistered = CultCache::new(); + unregistered.register_entry_type::()?; + unregistered.put("held", &Settings { theme: "held".to_string(), retries: 0 })?; + let before = unregistered.snapshot(); + let error = unregistered.load_soa(table.clone()).unwrap_err(); + assert!(error.to_string().contains("\"note\""), "{error}"); + assert_eq!(unregistered.snapshot(), before); + + // A registered type with no home once a store is attached. + let temp = tempfile::tempdir()?; + let mut homeless = CultCache::new(); + homeless.register_registry(TestEntries)?; + homeless.add_backing_store(SingleFileMessagePackBackingStore::new(temp.path().join("settings.cc")), ["settings"])?; + homeless.put("held", &Settings { theme: "held".to_string(), retries: 0 })?; + let before = homeless.snapshot(); + let error = homeless.load_soa(table).unwrap_err(); + assert!(error.to_string().contains("No backing store is registered for entry type \"note\""), "{error}"); + assert_eq!(homeless.snapshot(), before); + Ok(()) + } + + #[test] + fn single_home_batch_is_not_refused_by_other_homes_records() -> Result<()> { + let temp = tempfile::tempdir()?; + let generic_path = temp.path().join("generic.cc"); + let settings_path = temp.path().join("settings.cc"); + let mut cache = CultCache::new(); + cache.register_registry(TestEntries)?; + cache.add_backing_store(SingleFileMessagePackBackingStore::new(&settings_path), ["settings"])?; + cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&generic_path))?; + cache.put("memo", &Note { title: "generic".to_string(), body: "home".to_string() })?; + cache.put("old", &Settings { theme: "kept".to_string(), retries: 0 })?; + + let (staged, _) = cache.prepare_entry("app", &Settings { theme: "batch".to_string(), retries: 1 })?; + cache.put_prepared_batch(vec![staged])?; + + let mut settings_keys: Vec<_> = SingleFileMessagePackBackingStore::new(&settings_path) + .pull_all()? + .into_iter() + .map(|entry| format!("{}:{}", entry.r#type, entry.key)) + .collect(); + settings_keys.sort(); + assert_eq!(settings_keys, vec!["settings:app".to_string(), "settings:old".to_string()]); + let generic_keys: Vec<_> = SingleFileMessagePackBackingStore::new(&generic_path) + .pull_all()? + .into_iter() + .map(|entry| format!("{}:{}", entry.r#type, entry.key)) + .collect(); + assert_eq!(generic_keys, vec!["note:memo".to_string()]); + assert_eq!(cache.get_required::("app")?.theme, "batch"); + Ok(()) + } + + #[test] + fn batch_with_empty_key_or_stored_at_is_refused_before_any_store() -> Result<()> { + let temp = tempfile::tempdir()?; + let store_path = temp.path().join("generic.cc"); + let mut cache = CultCache::new(); + cache.register_registry(TestEntries)?; + cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&store_path))?; + cache.put("app", &Settings { theme: "before".to_string(), retries: 1 })?; + let before = cache.snapshot(); + let store_bytes = fs::read(&store_path)?; + + let (empty_key, _) = cache.prepare_entry("", &Note { title: "t".to_string(), body: "b".to_string() })?; + let error = cache.put_prepared_batch(vec![empty_key]).unwrap_err(); + assert!(error.to_string().contains("keys for type \"note\" must be non-empty"), "{error}"); + + let (mut empty_stored_at, _) = cache.prepare_entry("n", &Note { title: "t".to_string(), body: "b".to_string() })?; + empty_stored_at.stored_at = " ".to_string(); + let error = cache.put_prepared_batch(vec![empty_stored_at]).unwrap_err(); + assert!(error.to_string().contains("stored_at for type \"note\" must be non-empty"), "{error}"); + + assert_eq!(cache.snapshot(), before); + assert_eq!(fs::read(&store_path)?, store_bytes); + Ok(()) + } + + #[test] + fn zero_store_cache_writes_and_deletes_in_memory() -> Result<()> { + let mut cache = CultCache::new(); + cache.register_entry_type::()?; + cache.put("x", &Settings { theme: "mem".to_string(), retries: 0 })?; + assert_eq!(cache.get_required::("x")?.theme, "mem"); + let (staged, _) = cache.prepare_entry("y", &Settings { theme: "batch".to_string(), retries: 0 })?; + cache.put_prepared_batch(vec![staged])?; + assert_eq!(cache.get_required::("y")?.theme, "batch"); + assert!(cache.delete::("x")?); + assert!(cache.get::("x")?.is_none()); + Ok(()) + } + + #[test] + fn update_and_delete_follow_the_cache_api() -> Result<()> { + let temp = tempfile::tempdir()?; + let store_path = temp.path().join("cache.cc"); + let mut cache = CultCache::new(); + cache.register_entry_type::()?; + cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&store_path))?; let updated = cache.update::("app", |current| { let mut current = current.unwrap_or(Settings { @@ -1408,7 +4054,7 @@ mod tests { #[test] fn pull_rejects_unregistered_persisted_entry_type() -> Result<()> { let temp = tempfile::tempdir()?; - let store_path = temp.path().join("cache.msgpack"); + let store_path = temp.path().join("cache.cc"); let mut store = SingleFileMessagePackBackingStore::new(&store_path); store.push(&CultCacheEnvelope { key: "unknown".to_string(), @@ -1419,7 +4065,7 @@ mod tests { })?; let mut cache = CultCache::new(); - cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&store_path)); + cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&store_path))?; let error = cache.pull_all_backing_stores().unwrap_err(); assert!( error @@ -1432,10 +4078,10 @@ mod tests { #[test] fn payload_is_binary_messagepack_not_json_value() -> Result<()> { let temp = tempfile::tempdir()?; - let store_path = temp.path().join("cache.msgpack"); + let store_path = temp.path().join("cache.cc"); let mut cache = CultCache::new(); cache.register_entry_type::()?; - cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&store_path)); + cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&store_path))?; cache.put( "app", &Settings { @@ -1451,10 +4097,23 @@ mod tests { Ok(()) } + #[test] + fn byte_slots_use_messagepack_binary_and_round_trip() -> Result<()> { + let record = BinaryRecord { + required: vec![1, 2, 3], + optional: Vec::new(), + }; + + let payload = rmp_serde::to_vec(&record)?; + assert_eq!(payload, vec![0x92, 0xc4, 3, 1, 2, 3, 0xc4, 0]); + assert_eq!(rmp_serde::from_slice::(&payload)?, record); + Ok(()) + } + #[test] fn corrupted_payload_fails_during_typed_retrieval() -> Result<()> { let temp = tempfile::tempdir()?; - let store_path = temp.path().join("cache.msgpack"); + let store_path = temp.path().join("cache.cc"); let mut store = SingleFileMessagePackBackingStore::new(&store_path); store.push(&CultCacheEnvelope { key: "app".to_string(), @@ -1466,7 +4125,7 @@ mod tests { let mut cache = CultCache::new(); cache.register_entry_type::()?; - cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&store_path)); + cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&store_path))?; cache.pull_all_backing_stores()?; let error = cache.get_required::("app").unwrap_err(); assert!( @@ -1480,12 +4139,12 @@ mod tests { #[test] fn put_envelope_reuses_existing_messagepack_payload() -> Result<()> { let temp = tempfile::tempdir()?; - let origin_store = temp.path().join("origin.msgpack"); - let target_store = temp.path().join("target.msgpack"); + let origin_store = temp.path().join("origin.cc"); + let target_store = temp.path().join("target.cc"); let mut origin = CultCache::new(); origin.register_entry_type::()?; - origin.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&origin_store)); + origin.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&origin_store))?; origin.put( "app", &Settings { @@ -1498,7 +4157,7 @@ mod tests { let mut target = CultCache::new(); target.register_entry_type::()?; - target.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&target_store)); + target.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&target_store))?; let applied = target.put_envelope::(envelope.clone())?; assert_eq!( @@ -1516,6 +4175,145 @@ mod tests { Ok(()) } + #[test] + fn raw_envelope_path_preserves_a_registered_dynamic_payload() -> Result<()> { + let temp = tempfile::tempdir()?; + let store_path = temp.path().join("dynamic.cc"); + let envelope = CultCacheEnvelope { + key: "app".to_string(), + r#type: Settings::TYPE.to_string(), + payload: vec![0xc1], + stored_at: now_utc_second(), + schema_id: Some("dynamic.settings.v1".to_string()), + }; + + let mut cache = CultCache::new(); + cache.register_entry_type::()?; + cache.register_entry_type::()?; + cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&store_path))?; + cache.put_raw_envelope(envelope.clone())?; + + assert_eq!(cache.snapshot(), vec![envelope.clone()]); + assert_eq!( + SingleFileMessagePackBackingStore::new(&store_path).pull_all()?, + vec![envelope] + ); + assert!(cache.get_required::("app").is_err()); + let before_collision = fs::read(&store_path)?; + assert!( + cache + .put_raw_envelope(CultCacheEnvelope { + key: "note".to_string(), + r#type: Note::TYPE.to_string(), + payload: Vec::new(), + stored_at: now_utc_second(), + schema_id: Some("dynamic.settings.v1".to_string()), + }) + .is_err() + ); + assert_eq!(fs::read(&store_path)?, before_collision); + assert!( + cache + .put_raw_envelope(CultCacheEnvelope { + key: "foreign".to_string(), + r#type: "unregistered".to_string(), + payload: Vec::new(), + stored_at: now_utc_second(), + schema_id: None, + }) + .is_err() + ); + Ok(()) + } + + #[test] + fn soa_snapshot_round_trips_registered_entries() -> Result<()> { + let temp = tempfile::tempdir()?; + let store_path = temp.path().join("cache.cc"); + let mut cache = CultCache::new(); + cache.register_registry(TestEntries)?; + cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&store_path))?; + cache.put( + "app", + &Settings { + theme: "ash".to_string(), + retries: 3, + }, + )?; + cache.put( + "memo", + &Note { + title: "machine".to_string(), + body: "awake".to_string(), + }, + )?; + + let soa = cache.snapshot_soa(); + assert_eq!(soa.len(), 2); + assert_eq!(soa.keys.len(), soa.payloads.len()); + + let mut restored = CultCache::new(); + restored.register_registry(TestEntries)?; + restored.add_generic_backing_store(SingleFileMessagePackBackingStore::new( + temp.path().join("restored.cc"), + ))?; + restored.load_soa(soa)?; + assert_eq!(restored.get_required::("app")?.theme, "ash"); + assert_eq!(restored.get_required::("memo")?.body, "awake"); + Ok(()) + } + + #[test] + fn typed_soa_columns_match_csharp_cache_ergonomics() -> Result<()> { + let temp = tempfile::tempdir()?; + let store_path = temp.path().join("cache.cc"); + let mut cache = CultCache::new(); + cache.register_registry(TestEntries)?; + cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&store_path))?; + cache.put( + "settings:a", + &Settings { + theme: "ash".to_string(), + retries: 3, + }, + )?; + cache.put( + "settings:b", + &Settings { + theme: "bone".to_string(), + retries: 5, + }, + )?; + + let table = cache.soa::()?; + assert_eq!( + table.keys(), + &["settings:a".to_string(), "settings:b".to_string()] + ); + assert_eq!( + table.column::("theme")?.values(), + &["ash".to_string(), "bone".to_string()] + ); + assert_eq!(table.column::("retries")?.values(), &[3, 5]); + + let error = table.column::("retries").unwrap_err(); + assert!(error.to_string().contains("not alloc::string::String")); + Ok(()) + } + + #[test] + fn soa_rejects_column_length_drift() { + let table = CultCacheSoaTable { + keys: vec!["app".to_string()], + types: Vec::new(), + payloads: Vec::new(), + stored_ats: Vec::new(), + schema_ids: Vec::new(), + }; + let error = table.validate().unwrap_err(); + assert!(error.to_string().contains("column types has length 0")); + } + #[test] fn messagepack_store_recovers_schema_stamped_records_missing_catalog_entries() -> Result<()> { let temp = tempfile::tempdir()?; @@ -1525,6 +4323,8 @@ mod tests { name: "schema-stamped".to_string(), value: "still readable".to_string(), }; + // An empty catalog with a stale schema id is what a record written by + // another runtime, or one older than its catalog, looks like on disk. let snapshot = PersistedStoreSnapshot( "cultcache.store.v1".to_string(), Vec::new(), @@ -1535,11 +4335,11 @@ mod tests { rmp_serde::to_vec(&expected)?, )], ); - fs::write(&store_path, rmp_serde::to_vec(&snapshot)?)?; + std::fs::write(&store_path, rmp_serde::to_vec(&snapshot)?)?; let mut cache = CultCache::new(); cache.register_entry_type::()?; - cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&store_path)); + cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&store_path))?; cache.pull_all_backing_stores()?; assert_eq!(cache.get_required::("record-1")?, expected); @@ -1551,4 +4351,226 @@ mod tests { ); Ok(()) } + + #[test] + fn abandoned_staging_cleanup_removes_only_canonical_candidates() -> Result<()> { + let temp = tempfile::tempdir()?; + let store_path = temp.path().join("store.cc"); + let mut cache = CultCache::new(); + cache.register_entry_type::()?; + cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&store_path))?; + cache.put( + "primary", + &Settings { + theme: "dark".to_string(), + retries: 2, + }, + )?; + + // A crashed writer leaves a canonically named candidate behind. + let abandoned = temp + .path() + .join(format!("store.cc.{}.tmp", uuid::Uuid::new_v4())); + std::fs::write(&abandoned, b"partial snapshot")?; + // These only look like candidates. Cleanup must not touch them: the + // staging name is the store's own file name plus a canonical UUID, and + // anything else is somebody's file. + let lookalike = temp.path().join("store.cc.not-a-uuid.tmp"); + std::fs::write(&lookalike, b"keep me")?; + let bare_suffix = temp.path().join("store.cc.tmp"); + std::fs::write(&bare_suffix, b"keep me too")?; + let neighbour = temp.path().join("other.cc.00000000-0000-0000-0000-000000000000.tmp"); + std::fs::write(&neighbour, b"not this store")?; + + cache.put( + "primary", + &Settings { + theme: "light".to_string(), + retries: 3, + }, + )?; + + assert!(!abandoned.exists(), "canonical abandoned candidate survived"); + assert!(lookalike.exists(), "non-UUID lookalike was deleted"); + assert!(bare_suffix.exists(), "bare .tmp suffix was deleted"); + assert!(neighbour.exists(), "another store's candidate was deleted"); + + let mut reloaded = CultCache::new(); + reloaded.register_entry_type::()?; + reloaded.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&store_path))?; + reloaded.pull_all_backing_stores()?; + assert_eq!(reloaded.get_required::("primary")?.theme, "light"); + Ok(()) + } + + fn snapshot_envelope(key: &str, payload: &[u8]) -> CultCacheEnvelope { + CultCacheEnvelope { + key: key.to_string(), + r#type: "snapshot.record".to_string(), + payload: payload.to_vec(), + stored_at: "2026-01-01T00:00:00Z".to_string(), + // The store stamps the schema id from the type on write, so a + // fixture that omits it would not survive its own round trip. + schema_id: Some("snapshot.record".to_string()), + } + } + + #[test] + fn snapshot_exchange_commits_only_against_the_observed_snapshot() -> Result<()> { + let temp = tempfile::tempdir()?; + let store = SingleFileMessagePackBackingStore::new(temp.path().join("generation.cc")); + + let first = vec![snapshot_envelope("admitted", b"one")]; + assert!(store.compare_exchange_snapshot(&[], &first)?); + assert_eq!(store.pull_all_read_only_snapshot()?, first); + + // A writer holding a stale view loses the race instead of clobbering it. + let stale = vec![snapshot_envelope("admitted", b"stale")]; + assert!(!store.compare_exchange_snapshot(&[], &stale)?); + assert_eq!(store.pull_all_read_only_snapshot()?, first); + + let second = vec![snapshot_envelope("admitted", b"two")]; + assert!(store.compare_exchange_snapshot(&first, &second)?); + assert_eq!(store.pull_all_read_only_snapshot()?, second); + Ok(()) + } + + #[test] + fn try_snapshot_exchange_separates_a_lost_race_from_a_commit() -> Result<()> { + let temp = tempfile::tempdir()?; + let store = SingleFileMessagePackBackingStore::new(temp.path().join("generation.cc")); + + let first = vec![snapshot_envelope("admitted", b"one")]; + assert_eq!( + store.try_compare_exchange_snapshot(&[], &first)?, + TryCompareExchangeSnapshotOutcome::Exchanged + ); + assert_eq!( + store.try_compare_exchange_snapshot(&[], &first)?, + TryCompareExchangeSnapshotOutcome::Mismatch, + "a stale expectation is a lost race, not contention" + ); + assert_eq!(store.pull_all_read_only_snapshot()?, first); + Ok(()) + } + + #[test] + fn snapshot_exchange_refuses_replacements_that_cannot_be_addressed() -> Result<()> { + let temp = tempfile::tempdir()?; + let store = SingleFileMessagePackBackingStore::new(temp.path().join("generation.cc")); + + assert!( + store + .compare_exchange_snapshot(&[], &[snapshot_envelope(" ", b"one")]) + .is_err(), + "an unaddressable record must not reach the store" + ); + assert!( + store + .compare_exchange_snapshot( + &[], + &[ + snapshot_envelope("admitted", b"one"), + snapshot_envelope("admitted", b"two"), + ], + ) + .is_err(), + "a duplicate identity would make the snapshot ambiguous" + ); + assert_eq!(store.pull_all_read_only_snapshot()?, Vec::new()); + Ok(()) + } + + #[test] + fn per_entry_exchange_constrains_only_the_identities_it_names() -> Result<()> { + let temp = tempfile::tempdir()?; + let store = SingleFileMessagePackBackingStore::new(temp.path().join("generation.cc")); + + let admitted = snapshot_envelope("admitted", b"one"); + let unrelated = snapshot_envelope("unrelated", b"other"); + assert!(store.compare_exchange( + &[ + CultCacheExpectedEnvelope { + key: "admitted".into(), + r#type: "snapshot.record".into(), + current: None, + }, + CultCacheExpectedEnvelope { + key: "unrelated".into(), + r#type: "snapshot.record".into(), + current: None, + }, + ], + &[admitted.clone(), unrelated.clone()], + )?); + + // The snapshot moved, but an expectation naming only `admitted` still + // holds. A whole-snapshot exchange would have lost this race. + let next = snapshot_envelope("admitted", b"two"); + assert!(store.compare_exchange( + &[CultCacheExpectedEnvelope { + key: "admitted".into(), + r#type: "snapshot.record".into(), + current: Some(admitted.clone()), + }], + &[next.clone()], + )?); + assert_eq!( + store.pull_all_read_only_snapshot()?, + vec![next.clone(), unrelated.clone()] + ); + + // A stale expectation for that identity now loses. + assert!(!store.compare_exchange( + &[CultCacheExpectedEnvelope { + key: "admitted".into(), + r#type: "snapshot.record".into(), + current: Some(admitted), + }], + &[snapshot_envelope("admitted", b"three")], + )?); + assert_eq!(store.pull_all_read_only_snapshot()?, vec![next, unrelated]); + Ok(()) + } + + #[test] + fn per_entry_exchange_distinguishes_absent_from_stored() -> Result<()> { + let temp = tempfile::tempdir()?; + let store = SingleFileMessagePackBackingStore::new(temp.path().join("generation.cc")); + let admitted = snapshot_envelope("admitted", b"one"); + + // Expecting a stored value where nothing is stored must not commit. + assert!(!store.compare_exchange( + &[CultCacheExpectedEnvelope { + key: "admitted".into(), + r#type: "snapshot.record".into(), + current: Some(admitted.clone()), + }], + &[admitted.clone()], + )?); + assert_eq!(store.pull_all_read_only_snapshot()?, Vec::new()); + + // Expecting absence commits the first writer. + assert!(store.compare_exchange( + &[CultCacheExpectedEnvelope { + key: "admitted".into(), + r#type: "snapshot.record".into(), + current: None, + }], + &[admitted.clone()], + )?); + + // And the second writer expecting absence now loses, which is what + // makes this usable as a create-once claim. + assert!(!store.compare_exchange( + &[CultCacheExpectedEnvelope { + key: "admitted".into(), + r#type: "snapshot.record".into(), + current: None, + }], + &[snapshot_envelope("admitted", b"two")], + )?); + assert_eq!(store.pull_all_read_only_snapshot()?, vec![admitted]); + Ok(()) + } } diff --git a/packages/cultcache-ts/README.md b/packages/cultcache-ts/README.md index 064cc91b..d1730958 100644 --- a/packages/cultcache-ts/README.md +++ b/packages/cultcache-ts/README.md @@ -53,7 +53,7 @@ import { SingleFileMessagePackBackingStore, defineDocumentRegistry, defineDocumentType, -} from "cultcache-ts"; +} from "@gamecult/cultcache-ts"; const itemDocument = defineDocumentType({ type: "item", @@ -142,6 +142,25 @@ Document definitions may also carry persistence metadata for cross-language stores: `schemaId`, `schemaName`, `schemaVersion`, `contentHash`, `canonicalSchemaJson`, `compatibleSchemaIds`, and slot `members`. +## Backing Store Routing + +```ts +const cache = CultCache.builder() + .withRegistry(registry) + .withBackingStore(new SingleFileMessagePackBackingStore("settings.cc"), settingsDocument) + .withGenericStore(new SingleFileMessagePackBackingStore("cache.cc")) + .build(); +``` + +A document type has exactly one home store +(`src/GameCult.Caching/Contracts/cultcache-store-composition.md` in CultLib): + +- a store added with types owns those types; otherwise the generic store is the home +- a second generic store throws, and so does a type claimed by a second store; + a refused registration attaches nothing +- `put`, `putEnvelope` and `delete` touch exactly one store; nothing is mirrored +- a write whose type has no home store throws + ## Name, Index, and Global Semantics `name` and `indexes` can be declared on the document definition: @@ -161,11 +180,19 @@ const playerDocument = defineDocumentType({ You can also register them later if you feel like changing the machine after boot: ```ts -cache.registerNameLookup(playerDocument, "displayName"); -cache.registerIndex(playerDocument, "faction", "faction"); +await cache.registerNameLookup(playerDocument, "displayName"); +await cache.registerIndex(playerDocument, "faction", "faction"); ``` -Global documents are singleton-style per type: +Registration, `addBackingStore`, `pullAllBackingStores` and every write and +delete run one at a time through a per-cache queue and return promises; reads +are synchronous. A write validates the whole record (home store, global key, +name and index accessors) before any store is touched, so a refused write +changes neither store nor cache. Registering an accessor that throws on a held +record installs nothing. + +Global documents are singleton-style per type, always stored under +`CultCache.GLOBAL_KEY` (`__global__`); writing one under any other key throws: ```ts await cache.putGlobal(settingsDocument, { theme: "ash", retries: 3 }); diff --git a/packages/cultcache-ts/package.json b/packages/cultcache-ts/package.json index b9df4300..67f8d5ed 100644 --- a/packages/cultcache-ts/package.json +++ b/packages/cultcache-ts/package.json @@ -1,6 +1,6 @@ { - "name": "cultcache-ts", - "version": "0.13.5", + "name": "@gamecult/cultcache-ts", + "version": "0.14.0", "description": "CultCache-style typed persistence for TypeScript with schema-backed polymorphic MessagePack storage.", "main": "dist/index.js", "types": "dist/index.d.ts", diff --git a/packages/cultcache-ts/src/cult-cache.ts b/packages/cultcache-ts/src/cult-cache.ts index 157e0934..57766be9 100644 --- a/packages/cultcache-ts/src/cult-cache.ts +++ b/packages/cultcache-ts/src/cult-cache.ts @@ -13,32 +13,84 @@ import type { CultCacheStoreRegistration, } from "./types"; -type StoreRoute = { - primary?: CacheBackingStore; - mirrors: CacheBackingStore[]; -}; +type Accessor = (value: unknown) => string | undefined; type RegisteredDefinition = { readonly definition: AnyCultCacheDocumentDefinition; readonly global: boolean; readonly formatter: CultCacheDocumentFormatter; readonly catalogEntry: CultCacheSchemaCatalogEntry; - nameAccessor?: (value: unknown) => string | undefined; - readonly indexAccessors: Map string | undefined>; + nameAccessor?: Accessor; + indexAccessors: ReadonlyMap; +}; + +// The name and index values a record produced when it was admitted. Removing the record +// reads these and runs no accessor, so removal has no fallible step. +type EntryLookups = { + readonly name?: string; + readonly indexes: ReadonlyMap; }; type HydratedEntry = CultCacheEnvelope & { value: unknown; + lookups: EntryLookups; }; +// Everything a pull replaces. Built whole, then swapped in with one assignment. +type HydratedState = { + readonly entries: Map; + readonly typeEntryIds: Map>; + readonly nameToKeyMaps: Map>; + readonly indexToKeyMaps: Map>; + readonly globalKeys: Map; + // Load compatibility shim: a global persisted under a key other than GLOBAL_KEY is held as that + // type's global under GLOBAL_KEY, and its persisted envelope is kept here so the first write or + // delete of that global removes it from the home store. Remove once no store holds a global under + // a key other than GLOBAL_KEY. + readonly legacyGlobals: Map; +}; + +// Lookups for every held record of one type under candidate accessors, derived before anything is installed. +type DerivedTypeLookups = { + readonly entryLookups: Array<[HydratedEntry, EntryLookups]>; + readonly nameMap?: Map; + readonly indexMaps: Map>; +}; + +function emptyHydratedState(): HydratedState { + return { + entries: new Map(), + typeEntryIds: new Map(), + nameToKeyMaps: new Map(), + indexToKeyMaps: new Map(), + globalKeys: new Map(), + legacyGlobals: new Map(), + }; +} + +function getOrCreate(map: Map, key: K, create: () => V): V { + let value = map.get(key); + if (value === undefined) { + value = create(); + map.set(key, value); + } + return value; +} + type BackingStoreTypeReference = string | AnyCultCacheDocumentDefinition; type AccessorInput = string | ((value: any) => unknown); +// Assigned in CultCache's static block. The builder's cache is unreachable until build(), so no +// queued operation can be in flight and registration and attach may run synchronously. +let registerNow!: (cache: CultCache, definition: AnyCultCacheDocumentDefinition) => void; +let attachNow!: (cache: CultCache, store: CacheBackingStore, types: BackingStoreTypeReference[]) => void; + export class CultCacheBuilder { readonly #cache = new CultCache(); + #built = false; withDocumentType(definition: TDefinition): this { - this.#cache.registerDocumentType(definition); + registerNow(this.#unbuilt(), definition); return this; } @@ -47,7 +99,10 @@ export class CultCacheBuilder { | CultCacheDocumentRegistry | Iterable, ): this { - this.#cache.registerRegistry(registry); + const cache = this.#unbuilt(); + for (const definition of registryDefinitions(registry)) { + registerNow(cache, definition); + } return this; } @@ -55,167 +110,193 @@ export class CultCacheBuilder { store: CacheBackingStore, ...types: BackingStoreTypeReference[] ): this { - this.#cache.addBackingStore(store, ...types); + attachNow(this.#unbuilt(), store, types); return this; } withGenericStore(store: CacheBackingStore): this { - this.#cache.addGenericBackingStore(store); + attachNow(this.#unbuilt(), store, []); return this; } build(): CultCache { + this.#built = true; return this.#cache; } + + #unbuilt(): CultCache { + if (this.#built) { + throw new Error("CultCacheBuilder has already built its cache; configure the cache through its own methods."); + } + return this.#cache; + } +} + +function registryDefinitions( + registry: CultCacheDocumentRegistry | Iterable, +): Iterable { + return Symbol.iterator in Object(registry) && !("definitions" in Object(registry)) + ? registry as Iterable + : (registry as CultCacheDocumentRegistry).definitions; } export class CultCache { static readonly GLOBAL_KEY = "__global__"; + static { + registerNow = (cache, definition) => cache.#registerNow(definition); + attachNow = (cache, store, types) => cache.#attachNow(store, types); + } + readonly #definitions = new Map(); readonly #schemaIdDefinitions = new Map(); readonly #schemaNameDefinitions = new Map(); - readonly #entries = new Map(); - readonly #typeEntryIds = new Map>(); - readonly #nameToKeyMaps = new Map>(); - readonly #indexToKeyMaps = new Map>(); - readonly #globalKeys = new Map(); + #hydrated = emptyHydratedState(); readonly #stores: CultCacheStoreRegistration[] = []; + #tail: Promise = Promise.resolve(); static builder(): CultCacheBuilder { return new CultCacheBuilder(); } + // Attach, pull, registration and every write and delete run one at a time through this chain, + // so validation, the store call and applying to the cache never interleave with another + // mutation of this cache. Reads are not queued. Observer delivery is not scheduled here. + // The chain advances on its own promise; the caller gets the operation's promise, untouched, so + // an unawaited failure is still an unhandled rejection. A call made from inside an operation + // queues behind it; a store call that waits for that call waits on itself and never finishes. + #serial(operation: () => T | Promise): Promise { + let advance!: () => void; + const previous = this.#tail; + this.#tail = new Promise((resolve) => { + advance = resolve; + }); + return previous.then(async () => { + try { + return await operation(); + } finally { + advance(); + } + }); + } + registerDocumentType( definition: TDefinition, - ): TDefinition { - const existing = this.#definitions.get(definition.type); - if (existing && existing.definition !== definition) { - throw new Error(`CultCache already has a different definition registered for type "${definition.type}".`); - } - - const registered = existing ?? { - definition, - global: definition.global === true, - formatter: this.#createFormatter(definition), - catalogEntry: this.#createCatalogEntry(definition), - nameAccessor: undefined, - indexAccessors: new Map string | undefined>(), - }; - - registered.nameAccessor = definition.name - ? this.#compileAccessor(definition.type, "name", definition.name) - : undefined; - registered.indexAccessors.clear(); - for (const [indexName, accessor] of this.#enumerateDefinitionIndexes(definition)) { - registered.indexAccessors.set( - indexName, - this.#compileAccessor(definition.type, `index "${indexName}"`, accessor), - ); - } - - this.#definitions.set(definition.type, registered); - for (const schemaId of registered.catalogEntry.compatibleSchemaIds ?? [registered.catalogEntry.schemaId]) { - const existingBySchema = this.#schemaIdDefinitions.get(schemaId); - if (existingBySchema && existingBySchema !== registered) { - throw new Error(`CultCache schema id "${schemaId}" is already registered for type "${existingBySchema.definition.type}".`); - } - this.#schemaIdDefinitions.set(schemaId, registered); - } - const existingBySchemaName = this.#schemaNameDefinitions.get(registered.catalogEntry.schemaName); - if (existingBySchemaName && existingBySchemaName !== registered) { - throw new Error( - `CultCache schema name "${registered.catalogEntry.schemaName}" is already registered for type "${existingBySchemaName.definition.type}".`, - ); - } - this.#schemaNameDefinitions.set(registered.catalogEntry.schemaName, registered); - this.#rebuildDefinitionLookups(registered); - return definition; + ): Promise { + return this.#serial(() => { + this.#registerNow(definition); + return definition; + }); } registerRegistry( registry: | CultCacheDocumentRegistry | Iterable, - ): this { - const definitions = Symbol.iterator in Object(registry) && !("definitions" in Object(registry)) - ? registry as Iterable - : (registry as CultCacheDocumentRegistry).definitions; - - for (const definition of definitions) { - this.registerDocumentType(definition); - } - - return this; + ): Promise { + return this.#serial(() => { + for (const definition of registryDefinitions(registry)) { + this.#registerNow(definition); + } + }); } registerNameLookup( definition: TDefinition, accessor: CultCacheDocumentAccessor>, - ): TDefinition { - const registered = this.#requireDefinition(definition); - registered.nameAccessor = this.#compileAccessor(definition.type, "name", accessor); - this.#rebuildNameLookup(registered); - return definition; + ): Promise { + return this.#serial(() => { + const registered = this.#requireDefinition(definition); + const nameAccessor = this.#compileAccessor(definition.type, "name", accessor); + const derived = this.#deriveTypeLookups(definition.type, nameAccessor, registered.indexAccessors); + this.#installAccessors(registered, nameAccessor, registered.indexAccessors, derived); + return definition; + }); } registerIndex( definition: TDefinition, indexName: string, accessor: CultCacheDocumentAccessor>, - ): TDefinition { - if (!indexName || indexName.trim().length === 0) { - throw new Error(`CultCache index names for type "${definition.type}" must be non-empty.`); - } + ): Promise { + return this.#serial(() => { + if (!indexName || indexName.trim().length === 0) { + throw new Error(`CultCache index names for type "${definition.type}" must be non-empty.`); + } - const registered = this.#requireDefinition(definition); - registered.indexAccessors.set( - indexName, - this.#compileAccessor(definition.type, `index "${indexName}"`, accessor), - ); - this.#rebuildIndexLookup(registered, indexName); - return definition; + const registered = this.#requireDefinition(definition); + const indexAccessors = new Map(registered.indexAccessors); + indexAccessors.set(indexName, this.#compileAccessor(definition.type, `index "${indexName}"`, accessor)); + const derived = this.#deriveTypeLookups(definition.type, registered.nameAccessor, indexAccessors); + this.#installAccessors(registered, registered.nameAccessor, indexAccessors, derived); + return definition; + }); } addBackingStore( store: CacheBackingStore, ...types: BackingStoreTypeReference[] - ): void { - this.#stores.push({ - store, - types: [...new Set(types.map((value) => (typeof value === "string" ? value : value.type)))], - }); + ): Promise { + return this.#serial(() => this.#attachNow(store, types)); } - addGenericBackingStore(store: CacheBackingStore): void { - this.addBackingStore(store); + addGenericBackingStore(store: CacheBackingStore): Promise { + return this.addBackingStore(store); } - async pullAllBackingStores(): Promise { - this.#resetHydratedState(); + // The complete next state (entries, globals, name and index lookups) is built into a fresh + // HydratedState, running every check and every user accessor, and replaces the cache's state + // in one assignment only once the build finishes. A refused load admits nothing. + pullAllBackingStores(): Promise { + return this.#serial(async () => { + const next = emptyHydratedState(); + const persistedGlobalKeys = new Map(); - for (const registration of this.#stores) { - const entries = await registration.store.pullAll(); + for (const registration of this.#stores) { + const entries = await registration.store.pullAll(); - for (const entry of entries) { - const registered = this.#resolveDefinitionForEnvelope(entry); - if (!registered) { - throw new Error( - entry.schemaId - ? `No schema is registered for persisted schema id "${entry.schemaId}".` - : `No schema is registered for persisted document type "${entry.type}".`, - ); + for (const entry of entries) { + const registered = this.#resolveDefinitionForEnvelope(entry); + if (!registered) { + throw new Error( + entry.schemaId + ? `No schema is registered for persisted schema id "${entry.schemaId}".` + : `No schema is registered for persisted document type "${entry.type}".`, + ); + } + + const type = registered.definition.type; + const home = this.#resolveRegistration(type); + if (home?.store !== registration.store) { + throw new Error( + `CultCache "${type}" record "${entry.key}" was loaded from ${this.#describe(registration)}, ` + + `but its home is ${this.#describe(home)}.`, + ); + } + + const payload = this.#cloneBytes(entry.payload); + const value = registered.formatter.decode(payload); + let admitted: CultCacheEnvelope = { ...entry, type, payload }; + if (registered.global) { + const persistedKey = persistedGlobalKeys.get(type); + if (persistedKey !== undefined) { + throw new Error( + `CultCache global document type "${type}" has multiple persisted entries: "${persistedKey}" and "${entry.key}".`, + ); + } + persistedGlobalKeys.set(type, entry.key); + if (entry.key !== CultCache.GLOBAL_KEY) { + next.legacyGlobals.set(type, admitted); + admitted = { ...admitted, key: CultCache.GLOBAL_KEY }; + } + } + const hydrated = this.#admitEntry(next, registered, admitted, value); + this.#installEntry(next, registered, hydrated); } - - const payload = this.#cloneBytes(entry.payload); - const value = registered.formatter.decode(payload); - this.#applyHydratedEntry(registered, { - ...entry, - type: registered.definition.type, - payload, - }, value, "pull"); } - } + + this.#hydrated = next; + }); } get( @@ -223,7 +304,7 @@ export class CultCache { key: string, ): CultCacheDocumentValue | undefined { this.#requireDefinition(definition); - const entry = this.#entries.get(this.#entryId(definition.type, key)); + const entry = this.#hydrated.entries.get(this.#entryId(definition.type, key)); return entry ? (entry.value as CultCacheDocumentValue) : undefined; } @@ -244,7 +325,7 @@ export class CultCache { key: string, ): CultCacheEnvelope | undefined { this.#requireDefinition(definition); - const entry = this.#entries.get(this.#entryId(definition.type, key)); + const entry = this.#hydrated.entries.get(this.#entryId(definition.type, key)); return entry ? this.#toEnvelope(entry) : undefined; } @@ -263,12 +344,8 @@ export class CultCache { getGlobal( definition: TDefinition, ): CultCacheDocumentValue | undefined { - const registered = this.#requireDefinition(definition); - if (!registered.global) { - throw new Error(`CultCache document type "${definition.type}" is not marked as global.`); - } - - const globalKey = this.#globalKeys.get(definition.type); + this.#requireGlobal(definition); + const globalKey = this.#hydrated.globalKeys.get(definition.type); return globalKey ? this.get(definition, globalKey) : undefined; } @@ -286,12 +363,8 @@ export class CultCache { getGlobalEnvelope( definition: TDefinition, ): CultCacheEnvelope | undefined { - const registered = this.#requireDefinition(definition); - if (!registered.global) { - throw new Error(`CultCache document type "${definition.type}" is not marked as global.`); - } - - const globalKey = this.#globalKeys.get(definition.type); + this.#requireGlobal(definition); + const globalKey = this.#hydrated.globalKeys.get(definition.type); return globalKey ? this.getEnvelope(definition, globalKey) : undefined; } @@ -299,21 +372,7 @@ export class CultCache { definition: TDefinition, ): CultCacheDocumentValue[] { this.#requireDefinition(definition); - const values: CultCacheDocumentValue[] = []; - const entryIds = this.#typeEntryIds.get(definition.type); - - if (!entryIds) { - return values; - } - - for (const entryId of entryIds) { - const entry = this.#entries.get(entryId); - if (entry) { - values.push(entry.value as CultCacheDocumentValue); - } - } - - return values; + return this.#entriesForType(definition.type).map((entry) => entry.value as CultCacheDocumentValue); } getKeyByName( @@ -325,7 +384,7 @@ export class CultCache { return undefined; } - return this.#nameToKeyMaps.get(definition.type)?.get(name); + return this.#hydrated.nameToKeyMaps.get(definition.type)?.get(name); } getIdByName( @@ -349,7 +408,7 @@ export class CultCache { value: string, ): string | undefined { this.#requireDefinition(definition); - return this.#indexToKeyMaps.get(this.#indexId(definition.type, indexName))?.get(value); + return this.#hydrated.indexToKeyMaps.get(this.#indexId(definition.type, indexName))?.get(value); } getIdByIndex( @@ -369,12 +428,180 @@ export class CultCache { return key ? this.get(definition, key) : undefined; } - async put( + put( + definition: TDefinition, + key: string, + value: CultCacheDocumentValue, + ): Promise> { + return this.#serial(() => this.#putNow(definition, key, value)); + } + + putEnvelope( + definition: TDefinition, + envelope: CultCacheEnvelope, + ): Promise> { + return this.#serial(async () => { + const registered = this.#requireDefinition(definition); + if (envelope.type !== definition.type) { + throw new Error( + `CultCache envelope type "${envelope.type}" does not match definition "${definition.type}".`, + ); + } + if (!envelope.key || envelope.key.trim().length === 0) { + throw new Error(`CultCache envelope key for type "${definition.type}" must be non-empty.`); + } + if (!envelope.storedAt || envelope.storedAt.trim().length === 0) { + throw new Error(`CultCache envelope storedAt for type "${definition.type}" must be non-empty.`); + } + this.#requireGlobalKey(registered, envelope.key); + + const payload = this.#cloneBytes(envelope.payload); + const parsed = registered.formatter.decode(payload) as CultCacheDocumentValue; + const entry: CultCacheEnvelope = { + key: envelope.key, + type: envelope.type, + payload, + storedAt: envelope.storedAt, + schemaId: envelope.schemaId ?? registered.catalogEntry.schemaId, + catalogEntry: envelope.catalogEntry ?? registered.catalogEntry, + }; + await this.#writeNow(registered, entry, parsed); + return parsed; + }); + } + + putGlobal( + definition: TDefinition, + value: CultCacheDocumentValue, + ): Promise> { + return this.#serial(() => { + this.#requireGlobal(definition); + return this.#putNow(definition, CultCache.GLOBAL_KEY, value); + }); + } + + update( + definition: TDefinition, + key: string, + updater: ( + current: CultCacheDocumentValue | undefined, + ) => CultCacheDocumentValue, + ): Promise> { + return this.#serial(() => this.#putNow(definition, key, updater(this.get(definition, key)))); + } + + updateGlobal( + definition: TDefinition, + updater: ( + current: CultCacheDocumentValue | undefined, + ) => CultCacheDocumentValue, + ): Promise> { + return this.#serial(() => + this.#putNow(definition, CultCache.GLOBAL_KEY, updater(this.getGlobal(definition))), + ); + } + + delete( + definition: TDefinition, + key: string, + ): Promise { + return this.#serial(() => this.#deleteNow(definition, key)); + } + + deleteGlobal( + definition: TDefinition, + ): Promise { + return this.#serial(async () => { + this.#requireGlobal(definition); + const globalKey = this.#hydrated.globalKeys.get(definition.type); + return globalKey ? this.#deleteNow(definition, globalKey) : false; + }); + } + + snapshot(): CultCacheEnvelope[] { + return [...this.#hydrated.entries.values()].map((entry) => this.#toEnvelope(entry)); + } + + #registerNow(definition: AnyCultCacheDocumentDefinition): void { + const existing = this.#definitions.get(definition.type); + if (existing && existing.definition !== definition) { + throw new Error(`CultCache already has a different definition registered for type "${definition.type}".`); + } + + const registered: RegisteredDefinition = existing ?? { + definition, + global: definition.global === true, + formatter: this.#createFormatter(definition), + catalogEntry: this.#createCatalogEntry(definition), + nameAccessor: undefined, + indexAccessors: new Map(), + }; + + const nameAccessor = definition.name + ? this.#compileAccessor(definition.type, "name", definition.name) + : undefined; + const indexAccessors = new Map(); + for (const [indexName, accessor] of this.#enumerateDefinitionIndexes(definition)) { + indexAccessors.set(indexName, this.#compileAccessor(definition.type, `index "${indexName}"`, accessor)); + } + + const schemaIds = registered.catalogEntry.compatibleSchemaIds ?? [registered.catalogEntry.schemaId]; + for (const schemaId of schemaIds) { + const existingBySchema = this.#schemaIdDefinitions.get(schemaId); + if (existingBySchema && existingBySchema !== registered) { + throw new Error(`CultCache schema id "${schemaId}" is already registered for type "${existingBySchema.definition.type}".`); + } + } + const existingBySchemaName = this.#schemaNameDefinitions.get(registered.catalogEntry.schemaName); + if (existingBySchemaName && existingBySchemaName !== registered) { + throw new Error( + `CultCache schema name "${registered.catalogEntry.schemaName}" is already registered for type "${existingBySchemaName.definition.type}".`, + ); + } + const derived = this.#deriveTypeLookups(definition.type, nameAccessor, indexAccessors); + + // Every check and accessor has run; install. + this.#definitions.set(definition.type, registered); + for (const schemaId of schemaIds) { + this.#schemaIdDefinitions.set(schemaId, registered); + } + this.#schemaNameDefinitions.set(registered.catalogEntry.schemaName, registered); + this.#installAccessors(registered, nameAccessor, indexAccessors, derived); + } + + #attachNow(store: CacheBackingStore, types: BackingStoreTypeReference[]): void { + const claimed = [...new Set(types.map((value) => (typeof value === "string" ? value : value.type)))]; + if (claimed.length === 0 && this.#stores.some((registration) => registration.types.length === 0)) { + throw new Error("Backing store would be a second generic store; name the types it is home to."); + } + for (const type of claimed) { + if (this.#stores.some((registration) => registration.types.includes(type))) { + throw new Error(`CultCache type "${type}" is already routed to another backing store; it cannot also route to this one.`); + } + } + const candidate: CultCacheStoreRegistration = { store, types: claimed }; + for (const type of this.#hydrated.typeEntryIds.keys()) { + const before = this.#resolveRegistration(type); + this.#stores.push(candidate); + const after = this.#resolveRegistration(type); + this.#stores.pop(); + if (before !== after) { + throw new Error( + `Attaching this backing store would move "${type}" from ${this.#describe(before)} to ${this.#describe(after)}; ` + + "attach routed stores before the generic store.", + ); + } + } + this.#stores.push(candidate); + } + + async #putNow( definition: TDefinition, key: string, value: CultCacheDocumentValue, ): Promise> { const registered = this.#requireDefinition(definition); + this.#requireGlobalKey(registered, key); const payload = registered.formatter.encode(value); const parsed = registered.formatter.decode(payload) as CultCacheDocumentValue; const entry: CultCacheEnvelope = { @@ -385,140 +612,67 @@ export class CultCache { schemaId: registered.catalogEntry.schemaId, catalogEntry: registered.catalogEntry, }; - - const route = this.#resolveRoute(definition.type); - if (!route.primary) { - throw new Error(`No backing store is registered for document type "${definition.type}".`); - } - - if (registered.global) { - const existingGlobalKey = this.#globalKeys.get(definition.type); - if (existingGlobalKey && existingGlobalKey !== key) { - await this.delete(definition, existingGlobalKey); - } - } - - await route.primary.push(entry); - await Promise.all(route.mirrors.map(async (mirror) => mirror.push(entry))); - this.#applyHydratedEntry(registered, entry, parsed, "put"); + await this.#writeNow(registered, entry, parsed); return parsed; } - async putEnvelope( - definition: TDefinition, - envelope: CultCacheEnvelope, - ): Promise> { - const registered = this.#requireDefinition(definition); - if (envelope.type !== definition.type) { - throw new Error( - `CultCache envelope type "${envelope.type}" does not match definition "${definition.type}".`, - ); + // Validates the whole write (home, global singleton, name and index accessors) before the store + // is touched; once the store accepts, applying to the cache has no fallible step. + async #writeNow(registered: RegisteredDefinition, entry: CultCacheEnvelope, value: unknown): Promise { + const home = this.#homeStore(entry.type); + const hydrated = this.#admitEntry(this.#hydrated, registered, entry, value); + const legacy = this.#hydrated.legacyGlobals.get(entry.type); + await home?.push(entry); + if (home && legacy) { + await this.#retireLegacyGlobal(home, legacy, entry); } - if (!envelope.key || envelope.key.trim().length === 0) { - throw new Error(`CultCache envelope key for type "${definition.type}" must be non-empty.`); - } - if (!envelope.storedAt || envelope.storedAt.trim().length === 0) { - throw new Error(`CultCache envelope storedAt for type "${definition.type}" must be non-empty.`); - } - - const payload = this.#cloneBytes(envelope.payload); - const parsed = registered.formatter.decode(payload) as CultCacheDocumentValue; - const entry: CultCacheEnvelope = { - key: envelope.key, - type: envelope.type, - payload, - storedAt: envelope.storedAt, - schemaId: envelope.schemaId ?? registered.catalogEntry.schemaId, - catalogEntry: envelope.catalogEntry ?? registered.catalogEntry, - }; - - const route = this.#resolveRoute(definition.type); - if (!route.primary) { - throw new Error(`No backing store is registered for document type "${definition.type}".`); - } - - if (registered.global) { - const existingGlobalKey = this.#globalKeys.get(definition.type); - if (existingGlobalKey && existingGlobalKey !== entry.key) { - await this.delete(definition, existingGlobalKey); - } - } - - await route.primary.push(entry); - await Promise.all(route.mirrors.map(async (mirror) => mirror.push(entry))); - this.#applyHydratedEntry(registered, entry, parsed, "put"); - return parsed; + this.#hydrated.legacyGlobals.delete(entry.type); + this.#installEntry(this.#hydrated, registered, hydrated); } - async putGlobal( - definition: TDefinition, - value: CultCacheDocumentValue, - ): Promise> { - const registered = this.#requireDefinition(definition); - if (!registered.global) { - throw new Error(`CultCache document type "${definition.type}" is not marked as global.`); + // The first write of a global loaded from a legacy key removes that record in the same write. If the + // removal fails, the new record is taken back out, so the store keeps only the legacy record. + async #retireLegacyGlobal(home: CacheBackingStore, legacy: CultCacheEnvelope, written: CultCacheEnvelope): Promise { + try { + await home.delete(legacy); + } catch (error) { + await home.delete(written).catch(() => undefined); + throw error; } - - return this.put(definition, CultCache.GLOBAL_KEY, value); } - async update( - definition: TDefinition, - key: string, - updater: ( - current: CultCacheDocumentValue | undefined, - ) => CultCacheDocumentValue, - ): Promise> { - const current = this.get(definition, key); - return this.put(definition, key, updater(current)); - } - - async updateGlobal( - definition: TDefinition, - updater: ( - current: CultCacheDocumentValue | undefined, - ) => CultCacheDocumentValue, - ): Promise> { - const current = this.getGlobal(definition); - return this.putGlobal(definition, updater(current)); - } - - async delete( + async #deleteNow( definition: TDefinition, key: string, ): Promise { const registered = this.#requireDefinition(definition); - const entry = this.#entries.get(this.#entryId(definition.type, key)); + const entry = this.#hydrated.entries.get(this.#entryId(definition.type, key)); if (!entry) { return false; } - const route = this.#resolveRoute(definition.type); - if (!route.primary) { - throw new Error(`No backing store is registered for document type "${definition.type}".`); - } - - const envelope = this.#toEnvelope(entry); - await route.primary.delete(envelope); - await Promise.all(route.mirrors.map(async (mirror) => mirror.delete(envelope))); - this.#removeHydratedEntry(registered, entry); + const home = this.#homeStore(definition.type); + await home?.delete(this.#hydrated.legacyGlobals.get(definition.type) ?? this.#toEnvelope(entry)); + this.#hydrated.legacyGlobals.delete(definition.type); + this.#removeEntry(this.#hydrated, registered, entry); return true; } - async deleteGlobal( - definition: TDefinition, - ): Promise { + #requireGlobal(definition: AnyCultCacheDocumentDefinition): RegisteredDefinition { const registered = this.#requireDefinition(definition); if (!registered.global) { throw new Error(`CultCache document type "${definition.type}" is not marked as global.`); } - - const globalKey = this.#globalKeys.get(definition.type); - return globalKey ? this.delete(definition, globalKey) : false; + return registered; } - snapshot(): CultCacheEnvelope[] { - return [...this.#entries.values()].map((entry) => this.#toEnvelope(entry)); + // A global is only ever stored under GLOBAL_KEY; any other key is refused before a store is touched. + #requireGlobalKey(registered: RegisteredDefinition, key: string): void { + if (registered.global && key !== CultCache.GLOBAL_KEY) { + throw new Error( + `CultCache global document type "${registered.definition.type}" must use key "${CultCache.GLOBAL_KEY}", not "${key}".`, + ); + } } #createFormatter(definition: AnyCultCacheDocumentDefinition): CultCacheDocumentFormatter { @@ -603,7 +757,7 @@ export class CultCache { type: string, label: string, accessor: AccessorInput, - ): (value: unknown) => string | undefined { + ): Accessor { if (typeof accessor === "function") { return (value: unknown) => this.#normalizeIndexValue(type, label, accessor(value)); } @@ -660,159 +814,195 @@ export class CultCache { return this.#schemaNameDefinitions.get(entry.type) ?? this.#definitions.get(entry.type); } - #resolveRoute(type: string): StoreRoute { - const typeSpecific = this.#stores.filter((registration) => registration.types.includes(type)); - if (typeSpecific.length > 0) { - return { - primary: typeSpecific[0]?.store, - mirrors: typeSpecific.slice(1).map((registration) => registration.store), - }; - } - - const generic = this.#stores.filter((registration) => registration.types.length === 0); - return { - primary: generic[0]?.store, - mirrors: generic.slice(1).map((registration) => registration.store), - }; + // The registration that claims the type, else the generic one. Registration guarantees at most one of each. + #resolveRegistration(type: string): CultCacheStoreRegistration | undefined { + return this.#stores.find((registration) => registration.types.includes(type)) + ?? this.#stores.find((registration) => registration.types.length === 0); } - #applyHydratedEntry( + // A cache with no stores is in memory: undefined. Once any store is attached, a type with no home is an error. + #homeStore(type: string): CacheBackingStore | undefined { + if (this.#stores.length === 0) { + return undefined; + } + const home = this.#resolveRegistration(type); + if (!home) { + throw new Error(`No backing store is registered for document type "${type}".`); + } + return home.store; + } + + #describe(registration: CultCacheStoreRegistration | undefined): string { + if (!registration) { + return "memory"; + } + return registration.types.length === 0 + ? "the generic store" + : `the store routed to ${registration.types.join(", ")}`; + } + + #lookupsFor( + nameAccessor: Accessor | undefined, + indexAccessors: ReadonlyMap, + value: unknown, + ): EntryLookups { + const name = nameAccessor?.(value); + const indexes = new Map(); + for (const [indexName, accessor] of indexAccessors) { + const indexValue = accessor(value); + if (indexValue !== undefined) { + indexes.set(indexName, indexValue); + } + } + return { name, indexes }; + } + + // Fallible half of admission: the global singleton check and every user accessor. Touches no state. + #admitEntry( + state: HydratedState, registered: RegisteredDefinition, entry: CultCacheEnvelope, value: unknown, - source: "pull" | "put", - ): void { - const entryId = this.#entryId(entry.type, entry.key); - const existing = this.#entries.get(entryId); - if (existing) { - this.#removeHydratedEntry(registered, existing); - } - + ): HydratedEntry { if (registered.global) { - const currentGlobalKey = this.#globalKeys.get(entry.type); - if (currentGlobalKey && currentGlobalKey !== entry.key) { + const currentGlobalKey = state.globalKeys.get(entry.type); + if (currentGlobalKey !== undefined && currentGlobalKey !== entry.key) { throw new Error( - source === "pull" - ? `CultCache global document type "${entry.type}" has multiple persisted entries.` - : `CultCache global document type "${entry.type}" already has a different key "${currentGlobalKey}".`, + `CultCache global document type "${entry.type}" has multiple persisted entries: "${currentGlobalKey}" and "${entry.key}".`, ); } } - const hydrated: HydratedEntry = { + return { ...entry, payload: this.#cloneBytes(entry.payload), schemaId: entry.schemaId ?? registered.catalogEntry.schemaId, catalogEntry: entry.catalogEntry ?? registered.catalogEntry, value, + lookups: this.#lookupsFor(registered.nameAccessor, registered.indexAccessors, value), }; - this.#entries.set(entryId, hydrated); - this.#getOrCreateTypeEntrySet(entry.type).add(entryId); + } + // Infallible half of admission: replaces any held record at the same key and installs the admitted one. + #installEntry(state: HydratedState, registered: RegisteredDefinition, hydrated: HydratedEntry): void { + const entryId = this.#entryId(hydrated.type, hydrated.key); + const existing = state.entries.get(entryId); + if (existing) { + this.#removeEntry(state, registered, existing); + } + + state.entries.set(entryId, hydrated); + getOrCreate(state.typeEntryIds, hydrated.type, () => new Set()).add(entryId); if (registered.global) { - this.#globalKeys.set(entry.type, entry.key); + state.globalKeys.set(hydrated.type, hydrated.key); } - - if (registered.nameAccessor) { - const name = registered.nameAccessor(value); - if (name !== undefined) { - this.#getOrCreateNameMap(entry.type).set(name, entry.key); - } + if (hydrated.lookups.name !== undefined) { + getOrCreate(state.nameToKeyMaps, hydrated.type, () => new Map()).set(hydrated.lookups.name, hydrated.key); } - - for (const [indexName, accessor] of registered.indexAccessors) { - const indexValue = accessor(value); - if (indexValue !== undefined) { - this.#getOrCreateIndexMap(entry.type, indexName).set(indexValue, entry.key); - } + for (const [indexName, indexValue] of hydrated.lookups.indexes) { + getOrCreate(state.indexToKeyMaps, this.#indexId(hydrated.type, indexName), () => new Map()) + .set(indexValue, hydrated.key); } } - #removeHydratedEntry( + // Reads the lookups stored at admission; runs no accessor. + #removeEntry( + state: HydratedState, registered: RegisteredDefinition, entry: HydratedEntry, ): void { const entryId = this.#entryId(entry.type, entry.key); - this.#entries.delete(entryId); + state.entries.delete(entryId); - const typeEntryIds = this.#typeEntryIds.get(entry.type); + const typeEntryIds = state.typeEntryIds.get(entry.type); if (typeEntryIds) { typeEntryIds.delete(entryId); if (typeEntryIds.size === 0) { - this.#typeEntryIds.delete(entry.type); + state.typeEntryIds.delete(entry.type); } } - if (registered.global && this.#globalKeys.get(entry.type) === entry.key) { - this.#globalKeys.delete(entry.type); + if (registered.global && state.globalKeys.get(entry.type) === entry.key) { + state.globalKeys.delete(entry.type); } - if (registered.nameAccessor) { - const name = registered.nameAccessor(entry.value); - if (name !== undefined) { - this.#nameToKeyMaps.get(entry.type)?.delete(name); + if (entry.lookups.name !== undefined) { + const names = state.nameToKeyMaps.get(entry.type); + if (names?.get(entry.lookups.name) === entry.key) { + names.delete(entry.lookups.name); } } - - for (const [indexName, accessor] of registered.indexAccessors) { - const value = accessor(entry.value); - if (value !== undefined) { - this.#indexToKeyMaps.get(this.#indexId(entry.type, indexName))?.delete(value); + for (const [indexName, indexValue] of entry.lookups.indexes) { + const index = state.indexToKeyMaps.get(this.#indexId(entry.type, indexName)); + if (index?.get(indexValue) === entry.key) { + index.delete(indexValue); } } } - #rebuildDefinitionLookups(registered: RegisteredDefinition): void { - this.#rebuildNameLookup(registered); + // Runs the candidate accessors over every held record of the type into local maps. + #deriveTypeLookups( + type: string, + nameAccessor: Accessor | undefined, + indexAccessors: ReadonlyMap, + ): DerivedTypeLookups { + const entryLookups: Array<[HydratedEntry, EntryLookups]> = []; + const nameMap = nameAccessor ? new Map() : undefined; + const indexMaps = new Map>(); + for (const indexName of indexAccessors.keys()) { + indexMaps.set(indexName, new Map()); + } + for (const entry of this.#entriesForType(type)) { + const lookups = this.#lookupsFor(nameAccessor, indexAccessors, entry.value); + entryLookups.push([entry, lookups]); + if (lookups.name !== undefined) { + nameMap?.set(lookups.name, entry.key); + } + for (const [indexName, indexValue] of lookups.indexes) { + indexMaps.get(indexName)?.set(indexValue, entry.key); + } + } + + return { entryLookups, nameMap, indexMaps }; + } + + // Installs accessors and their derived lookups together. Nothing here can throw. + #installAccessors( + registered: RegisteredDefinition, + nameAccessor: Accessor | undefined, + indexAccessors: ReadonlyMap, + derived: DerivedTypeLookups, + ): void { + const type = registered.definition.type; for (const indexName of registered.indexAccessors.keys()) { - this.#rebuildIndexLookup(registered, indexName); - } - } - - #rebuildNameLookup(registered: RegisteredDefinition): void { - this.#nameToKeyMaps.delete(registered.definition.type); - - if (!registered.nameAccessor) { - return; + this.#hydrated.indexToKeyMaps.delete(this.#indexId(type, indexName)); } - const map = this.#getOrCreateNameMap(registered.definition.type); - for (const entry of this.#entriesForType(registered.definition.type)) { - const name = registered.nameAccessor(entry.value); - if (name !== undefined) { - map.set(name, entry.key); - } + registered.nameAccessor = nameAccessor; + registered.indexAccessors = indexAccessors; + for (const [entry, lookups] of derived.entryLookups) { + entry.lookups = lookups; } - } - - #rebuildIndexLookup(registered: RegisteredDefinition, indexName: string): void { - const lookupId = this.#indexId(registered.definition.type, indexName); - this.#indexToKeyMaps.delete(lookupId); - - const accessor = registered.indexAccessors.get(indexName); - if (!accessor) { - return; + if (derived.nameMap) { + this.#hydrated.nameToKeyMaps.set(type, derived.nameMap); + } else { + this.#hydrated.nameToKeyMaps.delete(type); } - - const map = this.#getOrCreateIndexMap(registered.definition.type, indexName); - for (const entry of this.#entriesForType(registered.definition.type)) { - const value = accessor(entry.value); - if (value !== undefined) { - map.set(value, entry.key); - } + for (const [indexName, map] of derived.indexMaps) { + this.#hydrated.indexToKeyMaps.set(this.#indexId(type, indexName), map); } } #entriesForType(type: string): HydratedEntry[] { - const entryIds = this.#typeEntryIds.get(type); + const entryIds = this.#hydrated.typeEntryIds.get(type); if (!entryIds) { return []; } const entries: HydratedEntry[] = []; for (const entryId of entryIds) { - const entry = this.#entries.get(entryId); + const entry = this.#hydrated.entries.get(entryId); if (entry) { entries.push(entry); } @@ -821,34 +1011,6 @@ export class CultCache { return entries; } - #getOrCreateTypeEntrySet(type: string): Set { - let set = this.#typeEntryIds.get(type); - if (!set) { - set = new Set(); - this.#typeEntryIds.set(type, set); - } - return set; - } - - #getOrCreateNameMap(type: string): Map { - let map = this.#nameToKeyMaps.get(type); - if (!map) { - map = new Map(); - this.#nameToKeyMaps.set(type, map); - } - return map; - } - - #getOrCreateIndexMap(type: string, indexName: string): Map { - const lookupId = this.#indexId(type, indexName); - let map = this.#indexToKeyMaps.get(lookupId); - if (!map) { - map = new Map(); - this.#indexToKeyMaps.set(lookupId, map); - } - return map; - } - #toEnvelope(entry: HydratedEntry): CultCacheEnvelope { return { key: entry.key, @@ -860,14 +1022,6 @@ export class CultCache { }; } - #resetHydratedState(): void { - this.#entries.clear(); - this.#typeEntryIds.clear(); - this.#nameToKeyMaps.clear(); - this.#indexToKeyMaps.clear(); - this.#globalKeys.clear(); - } - #entryId(type: string, key: string): string { return `${type}::${key}`; } diff --git a/packages/cultcache-ts/src/index.ts b/packages/cultcache-ts/src/index.ts index 97687de8..92f7b481 100644 --- a/packages/cultcache-ts/src/index.ts +++ b/packages/cultcache-ts/src/index.ts @@ -27,3 +27,8 @@ export type { InspectedCatalogMember, InspectedRecord, } from "./cult-cache-inspector"; +export { + defineSwarmDocuments, + type DefineDocumentType, + type SwarmDocumentCatalog, +} from "./swarm-documents"; diff --git a/packages/cultcache-ts/src/swarm-documents.ts b/packages/cultcache-ts/src/swarm-documents.ts new file mode 100644 index 00000000..b1c0b40d --- /dev/null +++ b/packages/cultcache-ts/src/swarm-documents.ts @@ -0,0 +1,561 @@ +// GameCult swarm document-type catalog. +// +// These are CultCache document definitions for documents that cross service +// boundaries: Eve surfaces and provider advertisements, Ghostlight, Heimdall, +// Idunn, Muninn, Sleipnir, Aetheria, VoidBot, Stonks. The catalog lived in Odin +// because Odin was written first, not because Odin owns it; every service that +// decodes another service documents needs the same definitions, and a second +// copy would be a second definition of one wire format. +// +// defineDocumentType is injected rather than imported. That is how it worked in +// Odin, it keeps this file clear of the generic signature in ./document, and it +// lets a caller supply a runtime other than this package. + +/** An undecoded document payload. These definitions predate typing and name fields defensively. */ +export type DocumentValue = any; + +export type DefineDocumentType = (definition: Record) => unknown; +export type SwarmDocumentCatalog = Record; + +function parseObjectDocument(label: string) { + return { + parse(value: unknown) { + if (!value || typeof value !== "object") { + throw new Error(`${label} must be an object`); + } + return value; + }, + }; +} + +export function defineSwarmDocuments(defineDocumentType?: DefineDocumentType | null): SwarmDocumentCatalog { + if (!defineDocumentType) { + return { + interfaceBindingDefinition: null, + evePluginAdvertisementDefinition: null, + ghostlightSchemaCatalogDefinition: null, + heimdallCommandBoundaryDefinition: null, + heimdallTransportProfileDefinition: null, + idunnCommandBoundaryDefinition: null, + idunnDaemonHealthDefinition: null, + idunnTransportProfileDefinition: null, + interfaceLayoutDefinition: null, + muninnCaptureStreamCommandDefinition: null, + muninnCaptureStreamDefinition: null, + muninnCommandBoundaryDefinition: null, + muninnMediaAudioPacketDefinition: null, + muninnMediaReceiverFeedbackDefinition: null, + muninnMediaVideoAccessUnitDefinition: null, + muninnHidControllerStateDefinition: null, + muninnMoveControllerStateDefinition: null, + muninnMoveIdentityDefinition: null, + muninnMoveLightCommandDefinition: null, + muninnMoveMarkerCandidateDefinition: null, + muninnObsStreamCatalogDefinition: null, + muninnQuestAccessDefinition: null, + muninnTelemetrySurfaceDefinition: null, + muninnTransportProfileDefinition: null, + SleipnirInputMappingDefinition: null, + aetheriaAssetManifestDefinition: null, + aetheriaGravityViewportDefinition: null, + aetheriaObjectsViewportDefinition: null, + aetheriaRenderSplatsViewportDefinition: null, + operatorStateDefinition: null, + providerAdvertisementDefinition: null, + voidbotProviderCatalogDefinition: null, + stonksCommandBoundaryDefinition: null, + stonksMarketSnapshotDefinition: null, + stonksRequestEventDefinition: null, + stonksTransportProfileDefinition: null, + streamPixelsCommandBoundaryDefinition: null, + streamPixelsTransportProfileDefinition: null, + surfaceDefinition: null, + viliCommandBoundaryDefinition: null, + viliTransportProfileDefinition: null, + weksaCommandBoundaryDefinition: null, + weksaOperatorStateDefinition: null, + weksaTransportProfileDefinition: null, + voidbotSwarmSnapshotDefinition: null, + }; + } + + const surfaceDefinition = defineDocumentType({ + type: "gamecult.eve.surface_state", + schemaName: "gamecult.eve.surface_state", + schemaId: "gamecult.eve.surface_state.v1", + schemaVersion: "gamecult.eve.surface_state.v1", + global: false, + name: (value: DocumentValue) => value?.providerId || "surface", + schema: { parse: (value: DocumentValue) => value }, + members: [ + { slot: 0, memberName: "providerId", typeName: "string", isName: true }, + { slot: 1, memberName: "title", typeName: "string" }, + { slot: 2, memberName: "version", typeName: "long" }, + { slot: 3, memberName: "updatedAt", typeName: "string" }, + { slot: 4, memberName: "surface", typeName: "object" }, + ], + }); + const interfaceBindingDefinition = defineDocumentType({ + type: "gamecult.eve.interface_binding", + schemaName: "gamecult.eve.interface_binding", + schemaId: "gamecult.eve.interface_binding.v1", + schemaVersion: "gamecult.eve.interface_binding.v1", + global: false, + name: (value: DocumentValue) => value?.bindingId || value?.providerId || "interface", + schema: parseObjectDocument("Eve interface binding"), + }); + const providerAdvertisementDefinition = defineDocumentType({ + type: "gamecult.eve.provider_advertisement", + schemaName: "gamecult.eve.provider_advertisement", + schemaId: "gamecult.eve.provider_advertisement.v1", + schemaVersion: "gamecult.eve.provider_advertisement.v1", + global: false, + name: (value: DocumentValue) => value?.providerId || value?.provider?.id || "provider", + schema: parseObjectDocument("Eve provider advertisement"), + }); + const evePluginAdvertisementDefinition = defineDocumentType({ + type: "gamecult.eve.plugin_advertisement", + schemaName: "gamecult.eve.plugin_advertisement", + schemaId: "gamecult.eve.plugin_advertisement.v1", + schemaVersion: "gamecult.eve.plugin_advertisement.v1", + global: false, + name: (value: DocumentValue) => value?.pluginId || value?.plugin?.id || "plugin", + schema: parseObjectDocument("Eve plugin advertisement"), + }); + const ghostlightSchemaCatalogDefinition = defineDocumentType({ + type: "ghostlight.schema_catalog", + schemaName: "ghostlight.schema_catalog", + schemaId: "ghostlight.schema_catalog.v1", + schemaVersion: "ghostlight.schema_catalog.v1", + global: false, + name: (value: DocumentValue) => value?.providerId || "gamecult.ghostlight.dungeon", + schema: parseObjectDocument("Ghostlight public schema catalog"), + }); + const heimdallCommandBoundaryDefinition = defineDocumentType({ + type: "heimdall.command_boundary", + schemaName: "heimdall.command_boundary", + schemaId: "heimdall.command_boundary.v1", + schemaVersion: "heimdall.command_boundary.v1", + global: false, + name: (value: DocumentValue) => value?.boundaryId || value?.daemonId || "heimdall", + schema: parseObjectDocument("Heimdall command boundary"), + }); + const heimdallTransportProfileDefinition = defineDocumentType({ + type: "heimdall.transport_profile", + schemaName: "heimdall.transport_profile", + schemaId: "heimdall.transport_profile.v1", + schemaVersion: "heimdall.transport_profile.v1", + global: false, + name: (value: DocumentValue) => value?.profileId || value?.daemonId || "heimdall", + schema: parseObjectDocument("Heimdall transport profile"), + }); + const interfaceLayoutDefinition = defineDocumentType({ + type: "odin.interface_layout", + schemaName: "odin.interface_layout", + schemaId: "odin.interface_layout.v1", + schemaVersion: "odin.interface_layout.v1", + global: false, + name: (value: DocumentValue) => value?.layoutId || value?.providerId || "odin.providers", + schema: parseObjectDocument("Odin interface layout"), + members: [ + { slot: 0, memberName: "layoutId", typeName: "string", isName: true }, + { slot: 1, memberName: "updatedAt", typeName: "string" }, + { slot: 2, memberName: "tiles", typeName: "object" }, + ], + }); + const idunnDaemonHealthDefinition = defineDocumentType({ + type: "idunn.daemon_health", + schemaName: "idunn.daemon_health", + schemaId: "idunn.daemon_health", + schemaVersion: "idunn.daemon_health.v1", + global: false, + name: (value: DocumentValue) => value?.daemonId || value?.daemon_id || "daemon", + schema: parseObjectDocument("Idunn daemon health"), + }); + const idunnCommandBoundaryDefinition = defineDocumentType({ + type: "idunn.command_boundary", + schemaName: "idunn.command_boundary", + schemaId: "idunn.command_boundary.v1", + schemaVersion: "idunn.command_boundary.v1", + global: false, + name: (value: DocumentValue) => value?.boundary_id || value?.boundaryId || value?.daemon_id || value?.daemonId || "daemon", + schema: parseObjectDocument("Idunn command boundary"), + }); + const idunnTransportProfileDefinition = defineDocumentType({ + type: "idunn.daemon_transport_profile", + schemaName: "idunn.daemon_transport_profile", + schemaId: "idunn.daemon_transport_profile.v1", + schemaVersion: "idunn.daemon_transport_profile.v1", + global: false, + name: (value: DocumentValue) => value?.profile_id || value?.profileId || value?.daemon_id || value?.daemonId || "daemon", + schema: parseObjectDocument("Idunn daemon transport profile"), + }); + const muninnCaptureStreamDefinition = defineDocumentType({ + type: "muninn.capture_stream", + schemaName: "muninn.capture_stream", + schemaId: "muninn.capture_stream.v1", + schemaVersion: "muninn.capture_stream.v1", + global: false, + name: (value: DocumentValue) => value?.stream_id || value?.streamId || "capture-stream", + schema: parseObjectDocument("Muninn capture stream"), + }); + const muninnCaptureStreamCommandDefinition = defineDocumentType({ + type: "muninn.capture_stream_command", + schemaName: "muninn.capture_stream_command", + schemaId: "muninn.capture_stream_command.v1", + schemaVersion: "muninn.capture_stream_command.v1", + global: false, + name: (value: DocumentValue) => value?.command_id || value?.commandId || value?.stream_id || value?.streamId || "capture-stream-command", + schema: parseObjectDocument("Muninn capture stream command"), + }); + const muninnCommandBoundaryDefinition = defineDocumentType({ + type: "muninn.command_boundary", + schemaName: "muninn.command_boundary", + schemaId: "muninn.command_boundary.v1", + schemaVersion: "muninn.command_boundary.v1", + global: false, + name: (value: DocumentValue) => value?.boundary_id || value?.boundaryId || value?.daemon_id || value?.daemonId || "muninn", + schema: parseObjectDocument("Muninn command boundary"), + }); + const muninnMediaVideoAccessUnitDefinition = defineDocumentType({ + type: "muninn.media_video_access_unit", + schemaName: "muninn.media_video_access_unit", + schemaId: "muninn.media_video_access_unit.v1", + schemaVersion: "muninn.media_video_access_unit.v1", + global: false, + name: (value: DocumentValue) => { + const streamId = value?.stream_id || value?.streamId || "stream"; + const sessionId = value?.session_id || value?.sessionId || "session"; + const frameId = value?.frame_id ?? value?.frameId ?? "frame"; + const chunkIndex = value?.chunk_index ?? value?.chunkIndex ?? "chunk"; + return `${streamId}:${sessionId}:video:${frameId}:${chunkIndex}`; + }, + schema: parseObjectDocument("Muninn media video access unit"), + }); + const muninnMediaAudioPacketDefinition = defineDocumentType({ + type: "muninn.media_audio_packet", + schemaName: "muninn.media_audio_packet", + schemaId: "muninn.media_audio_packet.v1", + schemaVersion: "muninn.media_audio_packet.v1", + global: false, + name: (value: DocumentValue) => { + const streamId = value?.stream_id || value?.streamId || "stream"; + const sessionId = value?.session_id || value?.sessionId || "session"; + const packetId = value?.packet_id ?? value?.packetId ?? "packet"; + return `${streamId}:${sessionId}:audio:${packetId}`; + }, + schema: parseObjectDocument("Muninn media audio packet"), + }); + const muninnMediaReceiverFeedbackDefinition = defineDocumentType({ + type: "muninn.media_receiver_feedback", + schemaName: "muninn.media_receiver_feedback", + schemaId: "muninn.media_receiver_feedback.v1", + schemaVersion: "muninn.media_receiver_feedback.v1", + global: false, + name: (value: DocumentValue) => { + const streamId = value?.stream_id || value?.streamId || "stream"; + const sessionId = value?.session_id || value?.sessionId || "session"; + const receiverId = value?.receiver_id || value?.receiverId || "receiver"; + return `${streamId}:${sessionId}:feedback:${receiverId}`; + }, + schema: parseObjectDocument("Muninn media receiver feedback"), + }); + const muninnMoveControllerStateDefinition = defineDocumentType({ + type: "muninn.move_controller_state", + schemaName: "muninn.move_controller_state", + schemaId: "muninn.move_controller_state.v1", + schemaVersion: "muninn.move_controller_state.v1", + global: false, + name: (value: DocumentValue) => value?.stream_id || value?.streamId || value?.move_id || value?.moveId || "move-controller-state", + schema: parseObjectDocument("Muninn Move controller state"), + }); + const muninnHidControllerStateDefinition = defineDocumentType({ + type: "muninn.hid_controller_state", + schemaName: "muninn.hid_controller_state", + schemaId: "muninn.hid_controller_state.v1", + schemaVersion: "muninn.hid_controller_state.v1", + global: false, + name: (value: DocumentValue) => value?.stream_id || value?.streamId || value?.device_id || value?.deviceId || "hid-controller-state", + schema: parseObjectDocument("Muninn HID controller state"), + }); + const muninnMoveIdentityDefinition = defineDocumentType({ + type: "muninn.move_identity", + schemaName: "muninn.move_identity", + schemaId: "muninn.move_identity.v1", + schemaVersion: "muninn.move_identity.v1", + global: false, + name: (value: DocumentValue) => value?.identity_id || value?.identityId || value?.move_id || value?.moveId || "move-identity", + schema: parseObjectDocument("Muninn Move identity"), + }); + const muninnMoveLightCommandDefinition = defineDocumentType({ + type: "muninn.move_light_command", + schemaName: "muninn.move_light_command", + schemaId: "muninn.move_light_command.v1", + schemaVersion: "muninn.move_light_command.v1", + global: false, + name: (value: DocumentValue) => value?.command_id || value?.commandId || value?.move_id || value?.moveId || "move-light-command", + schema: parseObjectDocument("Muninn Move light command"), + }); + const muninnMoveMarkerCandidateDefinition = defineDocumentType({ + type: "muninn.move_marker_candidate", + schemaName: "muninn.move_marker_candidate", + schemaId: "muninn.move_marker_candidate.v1", + schemaVersion: "muninn.move_marker_candidate.v1", + global: false, + name: (value: DocumentValue) => value?.stream_id || value?.streamId || value?.camera_id || value?.cameraId || "move-marker-candidate", + schema: parseObjectDocument("Muninn Move marker candidate"), + }); + const muninnObsStreamCatalogDefinition = defineDocumentType({ + type: "muninn.obs_stream_catalog", + schemaName: "muninn.obs_stream_catalog", + schemaId: "muninn.obs_stream_catalog.v1", + schemaVersion: "muninn.obs_stream_catalog.v1", + global: false, + name: (value: DocumentValue) => value?.catalog_id || value?.catalogId || value?.host_id || value?.hostId || "muninn", + schema: parseObjectDocument("Muninn OBS stream catalog"), + }); + const muninnQuestAccessDefinition = defineDocumentType({ + type: "muninn.quest_access", + schemaName: "muninn.quest_access", + schemaId: "muninn.quest_access.v1", + schemaVersion: "muninn.quest_access.v1", + global: false, + name: (value: DocumentValue) => value?.access_id || value?.accessId || value?.serial || "quest-access", + schema: parseObjectDocument("Muninn Quest access"), + }); + const muninnTelemetrySurfaceDefinition = defineDocumentType({ + type: "muninn.telemetry_surface", + schemaName: "muninn.telemetry_surface", + schemaId: "muninn.telemetry_surface.v1", + schemaVersion: "muninn.telemetry_surface.v1", + global: false, + name: (value: DocumentValue) => value?.surface_id || value?.surfaceId || value?.host_id || value?.hostId || "muninn-telemetry", + schema: parseObjectDocument("Muninn telemetry surface"), + }); + const muninnTransportProfileDefinition = defineDocumentType({ + type: "muninn.transport_profile", + schemaName: "muninn.transport_profile", + schemaId: "muninn.transport_profile.v1", + schemaVersion: "muninn.transport_profile.v1", + global: false, + name: (value: DocumentValue) => value?.profile_id || value?.profileId || value?.daemon_id || value?.daemonId || "muninn", + schema: parseObjectDocument("Muninn transport profile"), + }); + const SleipnirInputMappingDefinition = defineDocumentType({ + type: "sleipnir.input_mapping", + schemaName: "sleipnir.input_mapping", + schemaId: "sleipnir.input_mapping.v1", + schemaVersion: "sleipnir.input_mapping.v1", + global: false, + name: (value: DocumentValue) => value?.providerId || value?.provider_id || "sleipnir", + schema: parseObjectDocument("Sleipnir input mapping"), + }); + const aetheriaAssetManifestDefinition = defineDocumentType({ + type: "gamecult.aetheria.asset_manifest", + schemaName: "gamecult.aetheria.asset_manifest", + schemaId: "gamecult.aetheria.asset_manifest.v1", + schemaVersion: "gamecult.aetheria.asset_manifest.v1", + global: false, + name: (value: DocumentValue) => value?.runId || value?.run_id || "aetheria-assets", + schema: parseObjectDocument("Aetheria asset manifest"), + }); + const aetheriaGravityViewportDefinition = defineDocumentType({ + type: "gamecult.fields.gravity", + schemaName: "gamecult.fields.gravity", + schemaId: "gamecult.fields.gravity.v1", + schemaVersion: "gamecult.fields.gravity.v1", + global: false, + name: (value: DocumentValue) => value?.viewportId || value?.viewport_id || "aetheria-gravity", + schema: parseObjectDocument("Aetheria gravity viewport"), + }); + const aetheriaObjectsViewportDefinition = defineDocumentType({ + type: "gamecult.fields.objects", + schemaName: "gamecult.fields.objects", + schemaId: "gamecult.fields.objects.v1", + schemaVersion: "gamecult.fields.objects.v1", + global: false, + name: (value: DocumentValue) => value?.viewportId || value?.viewport_id || "aetheria-objects", + schema: parseObjectDocument("Aetheria objects viewport"), + }); + const aetheriaRenderSplatsViewportDefinition = defineDocumentType({ + type: "gamecult.fields.splats", + schemaName: "gamecult.fields.splats", + schemaId: "gamecult.fields.splats.v1", + schemaVersion: "gamecult.fields.splats.v1", + global: false, + name: (value: DocumentValue) => value?.viewportId || value?.viewport_id || "aetheria-render-splats", + schema: parseObjectDocument("Aetheria render splats viewport"), + }); + const operatorStateDefinition = defineDocumentType({ + type: "gamecult.vili.operator_state", + schemaName: "gamecult.vili.operator_state", + schemaId: "gamecult.vili.operator_state.v1", + schemaVersion: "gamecult.vili.operator_state.v1", + global: false, + name: (value: DocumentValue) => value?.service || value?.providerId || "operator", + schema: parseObjectDocument("Vili operator state"), + }); + const viliCommandBoundaryDefinition = defineDocumentType({ + type: "gamecult.vili.command_boundary", + schemaName: "gamecult.vili.command_boundary", + schemaId: "gamecult.vili.command_boundary.v1", + schemaVersion: "gamecult.vili.command_boundary.v1", + global: false, + name: (value: DocumentValue) => value?.boundaryId || value?.daemonId || "vili", + schema: parseObjectDocument("Vili command boundary"), + }); + const viliTransportProfileDefinition = defineDocumentType({ + type: "gamecult.vili.transport_profile", + schemaName: "gamecult.vili.transport_profile", + schemaId: "gamecult.vili.transport_profile.v1", + schemaVersion: "gamecult.vili.transport_profile.v1", + global: false, + name: (value: DocumentValue) => value?.profileId || value?.daemonId || "vili", + schema: parseObjectDocument("Vili transport profile"), + }); + const weksaOperatorStateDefinition = defineDocumentType({ + type: "weksa.operator_state", + schemaName: "weksa.operator_state", + schemaId: "weksa.operator_state.v0", + schemaVersion: "weksa.operator_state.v0", + global: false, + name: (value: DocumentValue) => value?.daemon_id || value?.daemonId || value?.provider_id || value?.providerId || "weksa", + schema: parseObjectDocument("Weksa operator state"), + }); + const weksaCommandBoundaryDefinition = defineDocumentType({ + type: "weksa.command_boundary", + schemaName: "weksa.command_boundary", + schemaId: "weksa.command_boundary.v1", + schemaVersion: "weksa.command_boundary.v1", + global: false, + name: (value: DocumentValue) => value?.boundary_id || value?.boundaryId || value?.daemon_id || value?.daemonId || "weksa", + schema: parseObjectDocument("Weksa command boundary"), + }); + const weksaTransportProfileDefinition = defineDocumentType({ + type: "weksa.transport_profile", + schemaName: "weksa.transport_profile", + schemaId: "weksa.transport_profile.v1", + schemaVersion: "weksa.transport_profile.v1", + global: false, + name: (value: DocumentValue) => value?.profile_id || value?.profileId || value?.daemon_id || value?.daemonId || "weksa", + schema: parseObjectDocument("Weksa transport profile"), + }); + const stonksRequestEventDefinition = defineDocumentType({ + type: "stonks.request_event", + schemaName: "stonks.request_event", + schemaId: "stonks.request_event.v1", + schemaVersion: "stonks.request_event.v1", + global: false, + name: (value: DocumentValue) => value?.id || "request", + schema: parseObjectDocument("Stonks request event"), + }); + const stonksMarketSnapshotDefinition = defineDocumentType({ + type: "stonks.market_snapshot", + schemaName: "stonks.market_snapshot", + schemaId: "stonks.market_snapshot.v1", + schemaVersion: "stonks.market_snapshot.v1", + global: true, + schema: parseObjectDocument("Stonks market snapshot"), + }); + const stonksCommandBoundaryDefinition = defineDocumentType({ + type: "stonks.command_boundary", + schemaName: "stonks.command_boundary", + schemaId: "stonks.command_boundary.v1", + schemaVersion: "stonks.command_boundary.v1", + global: false, + name: (value: DocumentValue) => value?.boundaryId || value?.daemonId || "stonks", + schema: parseObjectDocument("Stonks command boundary"), + }); + const stonksTransportProfileDefinition = defineDocumentType({ + type: "stonks.transport_profile", + schemaName: "stonks.transport_profile", + schemaId: "stonks.transport_profile.v1", + schemaVersion: "stonks.transport_profile.v1", + global: false, + name: (value: DocumentValue) => value?.profileId || value?.daemonId || "stonks", + schema: parseObjectDocument("Stonks transport profile"), + }); + const streamPixelsCommandBoundaryDefinition = defineDocumentType({ + type: "streampixels.command_boundary", + schemaName: "streampixels.command_boundary", + schemaId: "streampixels.command_boundary.v1", + schemaVersion: "streampixels.command_boundary.v1", + global: false, + name: (value: DocumentValue) => value?.boundaryId || value?.daemonId || "streampixels", + schema: parseObjectDocument("StreamPixels command boundary"), + }); + const streamPixelsTransportProfileDefinition = defineDocumentType({ + type: "streampixels.transport_profile", + schemaName: "streampixels.transport_profile", + schemaId: "streampixels.transport_profile.v1", + schemaVersion: "streampixels.transport_profile.v1", + global: false, + name: (value: DocumentValue) => value?.profileId || value?.daemonId || "streampixels", + schema: parseObjectDocument("StreamPixels transport profile"), + }); + const voidbotSwarmSnapshotDefinition = defineDocumentType({ + type: "voidbot.swarm_state_snapshot", + schemaName: "voidbot.swarm_state_snapshot", + schemaId: "voidbot.swarm_state_snapshot.v1", + schemaVersion: "voidbot.swarm_state_snapshot.v1", + global: true, + schema: parseObjectDocument("VoidBot swarm snapshot"), + }); + const voidbotProviderCatalogDefinition = defineDocumentType({ + type: "voidbot.provider_advertisement_catalog", + schemaName: "voidbot.provider_advertisement_catalog", + schemaId: "voidbot.provider_advertisement_catalog.v0", + schemaVersion: "voidbot.provider_advertisement_catalog.v0", + global: true, + schema: parseObjectDocument("VoidBot provider advertisement catalog"), + }); + + return { + interfaceBindingDefinition, + evePluginAdvertisementDefinition, + ghostlightSchemaCatalogDefinition, + heimdallCommandBoundaryDefinition, + heimdallTransportProfileDefinition, + interfaceLayoutDefinition, + idunnCommandBoundaryDefinition, + idunnDaemonHealthDefinition, + idunnTransportProfileDefinition, + muninnCaptureStreamCommandDefinition, + muninnCaptureStreamDefinition, + muninnCommandBoundaryDefinition, + muninnMediaAudioPacketDefinition, + muninnMediaReceiverFeedbackDefinition, + muninnMediaVideoAccessUnitDefinition, + muninnHidControllerStateDefinition, + muninnMoveControllerStateDefinition, + muninnMoveIdentityDefinition, + muninnMoveLightCommandDefinition, + muninnMoveMarkerCandidateDefinition, + muninnObsStreamCatalogDefinition, + muninnQuestAccessDefinition, + muninnTelemetrySurfaceDefinition, + muninnTransportProfileDefinition, + SleipnirInputMappingDefinition, + aetheriaAssetManifestDefinition, + aetheriaGravityViewportDefinition, + aetheriaObjectsViewportDefinition, + aetheriaRenderSplatsViewportDefinition, + operatorStateDefinition, + providerAdvertisementDefinition, + stonksCommandBoundaryDefinition, + stonksMarketSnapshotDefinition, + stonksRequestEventDefinition, + stonksTransportProfileDefinition, + streamPixelsCommandBoundaryDefinition, + streamPixelsTransportProfileDefinition, + surfaceDefinition, + viliCommandBoundaryDefinition, + viliTransportProfileDefinition, + weksaCommandBoundaryDefinition, + weksaOperatorStateDefinition, + weksaTransportProfileDefinition, + voidbotProviderCatalogDefinition, + voidbotSwarmSnapshotDefinition, + }; +} + diff --git a/packages/cultcache-ts/test/cult-cache.test.ts b/packages/cultcache-ts/test/cult-cache.test.ts index fb206eb6..5932b2f8 100644 --- a/packages/cultcache-ts/test/cult-cache.test.ts +++ b/packages/cultcache-ts/test/cult-cache.test.ts @@ -1,9 +1,10 @@ import assert from "node:assert/strict"; import { exec, execFile } from "node:child_process"; +import { EventEmitter } from "node:events"; import { existsSync } from "node:fs"; import { access, mkdtemp, readFile, writeFile } from "node:fs/promises"; import { homedir, tmpdir } from "node:os"; -import { dirname, join, resolve } from "node:path"; +import { dirname, join, resolve, delimiter } from "node:path"; import { test } from "node:test"; import { promisify } from "node:util"; @@ -24,7 +25,9 @@ const pythonCommand = process.env.PYTHON ?? "python"; const cultCacheTsRoot = resolve(__dirname, "../.."); const cultLibRoot = findAncestor(cultCacheTsRoot, "CultLib.sln") ?? resolve(cultCacheTsRoot, "..", "CultLib"); const cultcachePyRoot = resolve(cultLibRoot, "packages", "cultcache-py"); -const cultcachePySrc = resolve(cultcachePyRoot, "src"); +const cultcachePySrc = ["cultcache-py", "cultnet-py", "cultmesh-py"] + .map((name) => resolve(cultLibRoot, "packages", name, "src")) + .join(delimiter); const cultcacheRsRoot = existsSync(resolve(cultLibRoot, "packages", "cultcache-rs")) ? resolve(cultLibRoot, "packages", "cultcache-rs") : resolve(cultCacheTsRoot, "..", "cultcache-rs"); @@ -161,8 +164,8 @@ test("CultCache can register name and index lookups after entries already exist" body: "world", }); - cache.registerNameLookup(noteDocument, "title"); - cache.registerIndex(noteDocument, "author", "author"); + await cache.registerNameLookup(noteDocument, "title"); + await cache.registerIndex(noteDocument, "author", "author"); assert.equal(cache.getIdByName(noteDocument, "Hello"), "note:hello"); assert.equal(cache.getIdByIndex(noteDocument, "author", "ari"), "note:hello"); @@ -531,6 +534,536 @@ test("SingleFileMessagePackBackingStore heals legacy envelopes whose payload was assert.ok(records[0]?.[3] instanceof Uint8Array); }); +test("CultCache rejects a second generic backing store", async () => { + const cache = new CultCache(); + await cache.addGenericBackingStore(new SingleFileMessagePackBackingStore(join(tmpdir(), "unused-a.cc"))); + await assert.rejects( + cache.addGenericBackingStore(new SingleFileMessagePackBackingStore(join(tmpdir(), "unused-b.cc"))), + /second generic store/u, + ); +}); + +test("CultCache rejects a type registered to two stores", async () => { + const cache = new CultCache(); + await cache.addBackingStore(new SingleFileMessagePackBackingStore(join(tmpdir(), "unused-a.cc")), "item"); + await assert.rejects( + cache.addBackingStore(new SingleFileMessagePackBackingStore(join(tmpdir(), "unused-b.cc")), "settings", "item"), + /"item" is already routed/u, + ); + // A refused registration claims nothing, so "settings" is still free. + await cache.addBackingStore(new SingleFileMessagePackBackingStore(join(tmpdir(), "unused-c.cc")), "settings"); +}); + +test("CultCache routes each type to its home store", async () => { + const itemDocument = defineDocumentType({ + type: "item", + schema: z.object({ name: z.string() }), + }); + const settingsDocument = defineDocumentType({ + type: "settings", + schema: z.object({ theme: z.string() }), + }); + const tempDir = await mkdtemp(join(tmpdir(), "cultcache-routes-")); + const genericPath = join(tempDir, "generic.cc"); + const settingsPath = join(tempDir, "settings.cc"); + const build = () => CultCache.builder() + .withRegistry(defineDocumentRegistry(itemDocument, settingsDocument)) + .withBackingStore(new SingleFileMessagePackBackingStore(settingsPath), settingsDocument) + .withGenericStore(new SingleFileMessagePackBackingStore(genericPath)) + .build(); + const keysIn = async (path: string) => + (await new SingleFileMessagePackBackingStore(path).pullAll()).map((entry) => `${entry.type}:${entry.key}`); + + const cache = build(); + await cache.put(itemDocument, "potion", { name: "Potion" }); + await cache.put(settingsDocument, "app", { theme: "ash" }); + assert.deepEqual(await keysIn(genericPath), ["item:potion"]); + assert.deepEqual(await keysIn(settingsPath), ["settings:app"]); + + await cache.delete(settingsDocument, "app"); + assert.deepEqual(await keysIn(settingsPath), []); + assert.deepEqual(await keysIn(genericPath), ["item:potion"]); + + const reloaded = build(); + await reloaded.pullAllBackingStores(); + assert.deepEqual(reloaded.getRequired(itemDocument, "potion"), { name: "Potion" }); + assert.equal(reloaded.get(settingsDocument, "app"), undefined); +}); + +test("CultCache refuses an attach that would move a held record's home", async () => { + const settingsDocument = defineDocumentType({ type: "settings", schema: z.object({ theme: z.string() }) }); + const tempDir = await mkdtemp(join(tmpdir(), "cultcache-home-move-")); + const genericPath = join(tempDir, "generic.cc"); + const settingsPath = join(tempDir, "settings.cc"); + const cache = CultCache.builder() + .withRegistry(defineDocumentRegistry(settingsDocument)) + .withGenericStore(new SingleFileMessagePackBackingStore(genericPath)) + .build(); + await cache.put(settingsDocument, "app", { theme: "v1-generic" }); + + await assert.rejects( + async () => cache.addBackingStore(new SingleFileMessagePackBackingStore(settingsPath), settingsDocument), + /would move "settings" from the generic store to the store routed to settings/u, + ); + // Nothing attached: the next write still lands in the generic store, and the refused store stays untouched. + await cache.put(settingsDocument, "app", { theme: "v2-generic" }); + const keysIn = async (path: string) => + (await new SingleFileMessagePackBackingStore(path).pullAll()).map((entry) => `${entry.type}:${entry.key}`); + assert.deepEqual(await keysIn(genericPath), ["settings:app"]); + assert.equal(existsSync(settingsPath), false); +}); + +test("CultCache refuses at load a record delivered by a store that is not its home", async () => { + const settingsDocument = defineDocumentType({ type: "settings", schema: z.object({ theme: z.string() }) }); + const tempDir = await mkdtemp(join(tmpdir(), "cultcache-load-home-")); + const genericPath = join(tempDir, "generic.cc"); + const settingsPath = join(tempDir, "settings.cc"); + const writer = CultCache.builder() + .withRegistry(defineDocumentRegistry(settingsDocument)) + .withGenericStore(new SingleFileMessagePackBackingStore(genericPath)) + .build(); + await writer.put(settingsDocument, "app", { theme: "stray" }); + const genericBytes = await readFile(genericPath); + + const cache = CultCache.builder() + .withRegistry(defineDocumentRegistry(settingsDocument)) + .withBackingStore(new SingleFileMessagePackBackingStore(settingsPath), settingsDocument) + .withGenericStore(new SingleFileMessagePackBackingStore(genericPath)) + .build(); + // The cache already holds a record, in its home store, before the refused load. + await cache.put(settingsDocument, "held", { theme: "home" }); + const before = cache.snapshot(); + const settingsBytes = await readFile(settingsPath); + await assert.rejects( + cache.pullAllBackingStores(), + /"settings" record "app" was loaded from the generic store, but its home is the store routed to settings/u, + ); + assert.equal(cache.get(settingsDocument, "app"), undefined); + assert.deepEqual(cache.snapshot(), before); + assert.deepEqual(cache.getRequired(settingsDocument, "held"), { theme: "home" }); + assert.deepEqual(await readFile(genericPath), genericBytes); + assert.deepEqual(await readFile(settingsPath), settingsBytes); +}); + +async function storeState(path: string): Promise { + if (!existsSync(path)) { + return []; + } + return (await new SingleFileMessagePackBackingStore(path).pullAll()) + .map((entry) => `${entry.type}:${entry.key}:${Buffer.from(entry.payload).toString("hex")}`) + .sort(); +} + +test("putWithThrowingAccessorChangesNeitherStoreNorCache", async () => { + const itemDocument = defineDocumentType({ + type: "item", + schema: z.object({ n: z.string() }), + name: (value: { n: string }) => { + if (value.n === "bad") { + throw new Error("accessor refuses bad"); + } + return value.n; + }, + }); + const storePath = join(await mkdtemp(join(tmpdir(), "cultcache-put-accessor-")), "generic.cc"); + const cache = CultCache.builder() + .withRegistry(defineDocumentRegistry(itemDocument)) + .withGenericStore(new SingleFileMessagePackBackingStore(storePath)) + .build(); + await cache.put(itemDocument, "k", { n: "good" }); + const storeBefore = await storeState(storePath); + const cacheBefore = cache.snapshot(); + + await assert.rejects(cache.put(itemDocument, "k", { n: "bad" }), /accessor refuses bad/u); + assert.deepEqual(await storeState(storePath), storeBefore); + assert.deepEqual(cache.snapshot(), cacheBefore); + assert.deepEqual(cache.getRequired(itemDocument, "k"), { n: "good" }); + assert.equal(cache.getKeyByName(itemDocument, "good"), "k"); +}); + +test("overwriteUnderRegisteredAccessorThatNowThrowsChangesNeitherStoreNorCache", async () => { + const itemDocument = defineDocumentType({ type: "item", schema: z.object({ c: z.string() }) }); + const storePath = join(await mkdtemp(join(tmpdir(), "cultcache-overwrite-accessor-")), "generic.cc"); + const cache = CultCache.builder() + .withRegistry(defineDocumentRegistry(itemDocument)) + .withGenericStore(new SingleFileMessagePackBackingStore(storePath)) + .build(); + await cache.put(itemDocument, "k", { c: "old" }); + let refusing = false; + await cache.registerIndex(itemDocument, "category", (value: { c: string }) => { + if (refusing) { + throw new Error("accessor refuses now"); + } + return value.c; + }); + refusing = true; + const storeBefore = await storeState(storePath); + const cacheBefore = cache.snapshot(); + + await assert.rejects(cache.put(itemDocument, "k", { c: "new" }), /accessor refuses now/u); + assert.deepEqual(await storeState(storePath), storeBefore); + assert.deepEqual(cache.snapshot(), cacheBefore); + assert.equal(cache.getKeyByIndex(itemDocument, "category", "old"), "k"); + // Deleting runs no accessor: it uses the lookup values stored when the record was admitted. + assert.equal(await cache.delete(itemDocument, "k"), true); + assert.equal(cache.getKeyByIndex(itemDocument, "category", "old"), undefined); +}); + +test("registeringAccessorThatThrowsOnHeldRecordInstallsNothing", async () => { + const itemDocument = defineDocumentType({ type: "item", schema: z.object({ n: z.string() }) }); + const storePath = join(await mkdtemp(join(tmpdir(), "cultcache-register-accessor-")), "generic.cc"); + const cache = CultCache.builder() + .withRegistry(defineDocumentRegistry(itemDocument)) + .withGenericStore(new SingleFileMessagePackBackingStore(storePath)) + .build(); + await cache.put(itemDocument, "old", { n: "bad" }); + await assert.rejects( + async () => cache.registerIndex(itemDocument, "i", (value: { n: string }) => { + if (value.n === "bad") { + throw new Error("index refuses bad"); + } + return value.n; + }), + /index refuses bad/u, + ); + await cache.put(itemDocument, "old", { n: "fine" }); + assert.deepEqual(cache.getRequired(itemDocument, "old"), { n: "fine" }); + assert.equal(cache.getKeyByIndex(itemDocument, "i", "fine"), undefined); +}); + +test("globalReplaceWithFailingStorePushKeepsOldGlobal", async () => { + const settingsDocument = defineDocumentType({ + type: "settings", + schema: z.object({ t: z.string() }), + global: true, + }); + const storePath = join(await mkdtemp(join(tmpdir(), "cultcache-global-push-")), "generic.cc"); + const inner = new SingleFileMessagePackBackingStore(storePath); + let failPush = false; + const store: CacheBackingStore = { + pullAll: () => inner.pullAll(), + delete: (entry) => inner.delete(entry), + pushAll: (entries, options) => inner.pushAll(entries, options), + push: async (entry) => { + if (failPush) { + throw new Error("disk full"); + } + return inner.push(entry); + }, + }; + const cache = CultCache.builder() + .withRegistry(defineDocumentRegistry(settingsDocument)) + .withGenericStore(store) + .build(); + await cache.putGlobal(settingsDocument, { t: "base" }); + const storeBefore = await storeState(storePath); + const cacheBefore = cache.snapshot(); + failPush = true; + + await assert.rejects(cache.putGlobal(settingsDocument, { t: "new" }), /disk full/u); + await assert.rejects(cache.put(settingsDocument, "other", { t: "new" }), /must use key "__global__"/u); + assert.deepEqual(await storeState(storePath), storeBefore); + assert.deepEqual(cache.snapshot(), cacheBefore); + assert.deepEqual(cache.getRequiredGlobal(settingsDocument), { t: "base" }); +}); + +test("concurrentGlobalPutsCannotBothReachDisk", async () => { + const settingsDocument = defineDocumentType({ + type: "settings", + schema: z.object({ t: z.string() }), + global: true, + }); + const storePath = join(await mkdtemp(join(tmpdir(), "cultcache-global-race-")), "generic.cc"); + const inner = new SingleFileMessagePackBackingStore(storePath); + // The first push after arming writes, then pauses until released. A second write that reaches the + // store during that pause is recorded: serialized writes cannot. + let armed = false; + let paused = false; + let reachedStoreWhilePaused = false; + let release!: () => void; + let reportPaused!: () => void; + const pausedSignal = new Promise((resolve) => { + reportPaused = resolve; + }); + const store: CacheBackingStore = { + pullAll: () => inner.pullAll(), + delete: (entry) => inner.delete(entry), + push: async (entry) => { + reachedStoreWhilePaused ||= paused; + await inner.push(entry); + if (armed) { + armed = false; + paused = true; + await new Promise((resolve) => { + release = resolve; + reportPaused(); + }); + paused = false; + } + }, + }; + const build = (backing: CacheBackingStore) => CultCache.builder() + .withRegistry(defineDocumentRegistry(settingsDocument)) + .withGenericStore(backing) + .build(); + const cache = build(store); + await cache.putGlobal(settingsDocument, { t: "base" }); + + armed = true; + const first = cache.putGlobal(settingsDocument, { t: "first" }); + await pausedSignal; + const second = cache.putGlobal(settingsDocument, { t: "second" }); + await new Promise((resolve) => setTimeout(resolve, 50)); + release(); + await Promise.all([first, second]); + + assert.equal(reachedStoreWhilePaused, false); + const onDisk = await inner.pullAll(); + assert.deepEqual(onDisk.map((entry) => entry.key), [CultCache.GLOBAL_KEY]); + assert.deepEqual( + Buffer.from(onDisk[0].payload), + Buffer.from(cache.getRequiredEnvelope(settingsDocument, CultCache.GLOBAL_KEY).payload), + ); + assert.deepEqual(cache.getRequiredGlobal(settingsDocument), { t: "second" }); + const reloaded = build(new SingleFileMessagePackBackingStore(storePath)); + await reloaded.pullAllBackingStores(); + assert.deepEqual(reloaded.getRequiredGlobal(settingsDocument), { t: "second" }); +}); + +test("unawaitedFailingPutIsAnUnhandledRejection", async () => { + const itemDocument = defineDocumentType({ type: "item", schema: z.object({ n: z.string() }) }); + const store: CacheBackingStore = { + pullAll: async () => [], + push: async () => { + throw new Error("disk full"); + }, + delete: async () => undefined, + }; + const cache = CultCache.builder() + .withRegistry(defineDocumentRegistry(itemDocument)) + .withGenericStore(store) + .build(); + + const runnerListeners = process.listeners("unhandledRejection"); + process.removeAllListeners("unhandledRejection"); + const unhandled: unknown[] = []; + const listener = (reason: unknown) => { + unhandled.push(reason); + }; + process.on("unhandledRejection", listener); + try { + void cache.put(itemDocument, "k", { n: "x" }); + await new Promise((resolve) => setTimeout(resolve, 50)); + } finally { + process.off("unhandledRejection", listener); + for (const runnerListener of runnerListeners) { + process.on("unhandledRejection", runnerListener); + } + } + + assert.equal(unhandled.length, 1); + assert.match(String((unhandled[0] as Error).message), /disk full/u); + // The chain survived the rejection: the next operation still runs. + await assert.rejects(cache.put(itemDocument, "k", { n: "y" }), /disk full/u); +}); + +test("storeChangeHandlerWritingBackQueuesAfterTheOperation", async () => { + const itemDocument = defineDocumentType({ type: "item", schema: z.object({ n: z.string() }) }); + const storePath = join(await mkdtemp(join(tmpdir(), "cultcache-writeback-")), "generic.cc"); + const inner = new SingleFileMessagePackBackingStore(storePath); + const changes = new EventEmitter(); + const pushed: string[] = []; + const store: CacheBackingStore = { + pullAll: () => inner.pullAll(), + delete: (entry) => inner.delete(entry), + push: async (entry) => { + await inner.push(entry); + pushed.push(entry.key); + changes.emit("pushed", entry.key); + }, + }; + const cache = CultCache.builder() + .withRegistry(defineDocumentRegistry(itemDocument)) + .withGenericStore(store) + .build(); + // The handler writes back and the store does not wait for it: that write queues behind the put. + const writeBacks: Promise[] = []; + changes.on("pushed", (key: string) => { + if (key === "k") { + writeBacks.push(cache.put(itemDocument, "side", { n: "side" })); + } + }); + + await cache.put(itemDocument, "k", { n: "x" }); + await Promise.all(writeBacks); + + assert.equal(writeBacks.length, 1); + assert.deepEqual(pushed, ["k", "side"]); + assert.deepEqual(cache.get(itemDocument, "k"), { n: "x" }); + assert.deepEqual(cache.get(itemDocument, "side"), { n: "side" }); + assert.deepEqual( + (await inner.pullAll()).map((entry) => entry.key).sort(), + cache.snapshot().map((entry) => entry.key).sort(), + ); +}); + +test("legacyKeyGlobalLoadsWithoutWritingAndFirstWriteLeavesOneGlobal", async () => { + const settingsDocument = defineDocumentType({ + type: "settings", + schema: z.object({ t: z.string() }), + global: true, + }); + const storePath = join(await mkdtemp(join(tmpdir(), "cultcache-legacy-global-")), "generic.cc"); + const build = () => CultCache.builder() + .withRegistry(defineDocumentRegistry(settingsDocument)) + .withGenericStore(new SingleFileMessagePackBackingStore(storePath)) + .build(); + const seed = CultCache.builder().withRegistry(defineDocumentRegistry(settingsDocument)).build(); + await seed.putGlobal(settingsDocument, { t: "old" }); + const legacy = { ...seed.getRequiredEnvelope(settingsDocument, CultCache.GLOBAL_KEY), key: "legacy" }; + await new SingleFileMessagePackBackingStore(storePath).push(legacy); + const bytes = await readFile(storePath); + + const cache = build(); + await cache.pullAllBackingStores(); + assert.deepEqual(cache.getRequiredGlobal(settingsDocument), { t: "old" }); + assert.deepEqual(cache.get(settingsDocument, CultCache.GLOBAL_KEY), { t: "old" }); + assert.deepEqual(await readFile(storePath), bytes); + + await cache.putGlobal(settingsDocument, { t: "new" }); + assert.deepEqual( + (await new SingleFileMessagePackBackingStore(storePath).pullAll()).map((entry) => entry.key), + [CultCache.GLOBAL_KEY], + ); + const reloaded = build(); + await reloaded.pullAllBackingStores(); + assert.deepEqual(reloaded.getRequiredGlobal(settingsDocument), { t: "new" }); + + // Deleting an adopted global removes the legacy record. + await new SingleFileMessagePackBackingStore(storePath).pushAll([legacy]); + const deleting = build(); + await deleting.pullAllBackingStores(); + assert.equal(await deleting.deleteGlobal(settingsDocument), true); + assert.deepEqual(await new SingleFileMessagePackBackingStore(storePath).pullAll(), []); + + // Two globals of one type on disk, under any keys, are refused. + await new SingleFileMessagePackBackingStore(storePath).pushAll([legacy, { ...legacy, key: CultCache.GLOBAL_KEY }]); + await assert.rejects(build().pullAllBackingStores(), /has multiple persisted entries/u); +}); + +test("concurrentPutAndAttachCannotLandRecordInNonHomeStore", async () => { + const settingsDocument = defineDocumentType({ type: "settings", schema: z.object({ t: z.string() }) }); + const tempDir = await mkdtemp(join(tmpdir(), "cultcache-put-attach-race-")); + const genericPath = join(tempDir, "generic.cc"); + const settingsPath = join(tempDir, "settings.cc"); + const cache = CultCache.builder() + .withRegistry(defineDocumentRegistry(settingsDocument)) + .withGenericStore(new SingleFileMessagePackBackingStore(genericPath)) + .build(); + + const pending = cache.put(settingsDocument, "app", { t: "x" }); + const attach = Promise.resolve().then(() => + cache.addBackingStore(new SingleFileMessagePackBackingStore(settingsPath), settingsDocument), + ); + await pending; + await assert.rejects(attach, /would move "settings" from the generic store/u); + + assert.deepEqual( + (await new SingleFileMessagePackBackingStore(genericPath).pullAll()).map((entry) => entry.key), + ["app"], + ); + assert.equal(existsSync(settingsPath), false); + const reloaded = CultCache.builder() + .withRegistry(defineDocumentRegistry(settingsDocument)) + .withGenericStore(new SingleFileMessagePackBackingStore(genericPath)) + .build(); + await reloaded.pullAllBackingStores(); + assert.deepEqual(reloaded.getRequired(settingsDocument, "app"), { t: "x" }); +}); + +test("pullWithDuplicateGlobalLeavesCacheUnchanged", async () => { + const itemDocument = defineDocumentType({ type: "item", schema: z.object({ name: z.string() }) }); + const settingsDocument = defineDocumentType({ + type: "settings", + schema: z.object({ theme: z.string() }), + global: true, + }); + const tempDir = await mkdtemp(join(tmpdir(), "cultcache-duplicate-global-")); + const storePath = join(tempDir, "generic.cc"); + const cache = CultCache.builder() + .withRegistry(defineDocumentRegistry(itemDocument, settingsDocument)) + .withGenericStore(new SingleFileMessagePackBackingStore(storePath)) + .build(); + await cache.put(itemDocument, "potion", { name: "Potion" }); + await cache.putGlobal(settingsDocument, { theme: "ash" }); + const before = cache.snapshot(); + + // A second record of the global type lands in the store file behind the cache's back. + const stray = cache.getRequiredEnvelope(settingsDocument, CultCache.GLOBAL_KEY); + await new SingleFileMessagePackBackingStore(storePath).push({ ...stray, key: "stray" }); + + await assert.rejects(cache.pullAllBackingStores(), /has multiple persisted entries/u); + assert.deepEqual(cache.snapshot(), before); + assert.deepEqual(cache.getRequired(itemDocument, "potion"), { name: "Potion" }); + assert.deepEqual(cache.getRequiredGlobal(settingsDocument), { theme: "ash" }); +}); + +test("pullWithThrowingIndexAccessorLeavesCacheUnchanged", async () => { + const itemDocument = defineDocumentType({ + type: "item", + schema: z.object({ name: z.string(), category: z.string() }), + name: "name", + indexes: { + category: (item: { name: string; category: string }) => { + if (item.name === "Bomb") { + throw new Error("category accessor refuses Bomb"); + } + return item.category; + }, + }, + }); + const settingsDocument = defineDocumentType({ + type: "settings", + schema: z.object({ theme: z.string() }), + global: true, + }); + const tempDir = await mkdtemp(join(tmpdir(), "cultcache-throwing-accessor-")); + const storePath = join(tempDir, "generic.cc"); + const cache = CultCache.builder() + .withRegistry(defineDocumentRegistry(itemDocument, settingsDocument)) + .withGenericStore(new SingleFileMessagePackBackingStore(storePath)) + .build(); + await cache.put(itemDocument, "potion", { name: "Potion", category: "consumable" }); + await cache.putGlobal(settingsDocument, { theme: "ash" }); + const before = cache.snapshot(); + + // A record whose index accessor throws lands in the store file behind the cache's back. + const potion = cache.getRequiredEnvelope(itemDocument, "potion"); + await new SingleFileMessagePackBackingStore(storePath).push({ + ...potion, + key: "bomb", + payload: encode({ name: "Bomb", category: "explosive" }), + }); + + await assert.rejects(cache.pullAllBackingStores(), /category accessor refuses Bomb/u); + assert.deepEqual(cache.snapshot(), before); + assert.deepEqual(cache.getRequired(itemDocument, "potion"), { name: "Potion", category: "consumable" }); + assert.equal(cache.get(itemDocument, "bomb"), undefined); + assert.equal(cache.getKeyByName(itemDocument, "Potion"), "potion"); + assert.equal(cache.getKeyByName(itemDocument, "Bomb"), undefined); + assert.equal(cache.getKeyByIndex(itemDocument, "category", "consumable"), "potion"); + assert.deepEqual(cache.getRequiredGlobal(settingsDocument), { theme: "ash" }); +}); + +test("CultCache with zero stores writes and deletes in memory", async () => { + const settingsDocument = defineDocumentType({ type: "settings", schema: z.object({ theme: z.string() }) }); + const cache = CultCache.builder().withRegistry(defineDocumentRegistry(settingsDocument)).build(); + await cache.put(settingsDocument, "x", { theme: "mem" }); + assert.deepEqual(cache.getRequired(settingsDocument, "x"), { theme: "mem" }); + assert.equal(await cache.delete(settingsDocument, "x"), true); + assert.equal(cache.get(settingsDocument, "x"), undefined); +}); + test("CultCache v1 MessagePack stores are readable across TS, Rust, C#, and Python", async () => { await buildInteropPeers(); const tempDir = await mkdtemp(join(tmpdir(), "cultcache-interop-")); @@ -612,6 +1145,32 @@ test("CultCache v1 MessagePack stores are readable across TS, Rust, C#, and Pyth assert.ok(read.tags.includes("interop")); } } + + // C# write-routed attaches one home store per type: each file is a complete v1 snapshot holding only its own records. + const catalogFile = join(tempDir, "catalog.cc"); + const runFile = join(tempDir, "run.cc"); + const routed = await runJsonCommand("csharp-write-routed", dotnetCommand, [ + csharpInteropDll, + "write-routed", + catalogFile, + runFile, + ], cultLibRoot); + const recordsIn = async (file: string) => { + const decoded = decode(await readFile(file)) as unknown[]; + assert.equal(decoded[0], "cultcache.store.v1"); + assert.ok(Array.isArray(decoded[1]), `${file} has no schema catalog`); + return (decoded[2] as unknown[][]).map((record) => ({ schemaId: record[1], key: record[0] })); + }; + const catalogRecords = await recordsIn(catalogFile); + const runRecords = await recordsIn(runFile); + assert.deepEqual(catalogRecords.map((record) => record.key), ["note:csharp-routed"]); + assert.deepEqual(runRecords.map((record) => record.key), ["run-note:csharp-routed"]); + assert.notEqual(catalogRecords[0]?.schemaId, runRecords[0]?.schemaId); + for (const reader of readers) { + const read = await reader.read(catalogFile); + assert.equal(read.documentId, routed.documentId, `${reader.name} failed to read the routed catalog store`); + assert.equal(read.body, "One home store per document type."); + } }); test("CultCache interop reader accepts missing compatible trailing slots and rejects mismatched slots", async () => { diff --git a/packages/cultmath/.gitignore b/packages/cultmath/.gitignore new file mode 100644 index 00000000..c9b141d0 --- /dev/null +++ b/packages/cultmath/.gitignore @@ -0,0 +1,9 @@ +bin/ +obj/ +**/bin/ +**/obj/ +target/ +**/target/ +/native/cultmath-core/target.meta +.tools/ +/artifacts/ diff --git a/packages/cultmath/README.md b/packages/cultmath/README.md new file mode 100644 index 00000000..0be46738 --- /dev/null +++ b/packages/cultmath/README.md @@ -0,0 +1,136 @@ +# CultMath + +CultMath is the project name. Its clean .NET package identity is +`GameCult.Math`, owned and released from the CultLib monorepo. + +Clean-room math shaped for people who think in HLSL. + +CultMath provides lowercase vector types, shader-style intrinsics, and native +hot-path kernels so math-heavy renderer code can move between C#, Rust, and HLSL +without turning every line into a translation exercise. It is not a +Unity.Mathematics clone. It is a small, Aquarium-born library built from public +shader semantics. + +## Current Surface + +- `float2`, `float3`, `float4` +- component-wise arithmetic +- scalar-to-vector splats +- every HLSL swizzle (`xyzw` and `rgba`, repeats readable, non-repeating + masks writable) and every HLSL mixed constructor, generated by + `tools/generate-swizzles.ps1` +- conversions to and from `System.Numerics` +- `Random`, a compact deterministic XORShift stream with scalar and vector draws +- `BatchMath` SoA kernels over `Span` / `ReadOnlySpan` using + `System.Numerics.Vector` when available, including radial falloff + acceleration and semi-implicit Euler integration +- `math` intrinsics: `radians`, `degrees`, `abs`, `floor`, `ceil`, `frac`, + `min`, `max`, `clamp`, `saturate`, `lerp`, `step`, `smoothstep`, `dot`, + `cross`, `length`, `distance`, `normalize`, `reflect`, `csum`, exponential + `decay`/`damp`, intercept/segment helpers, Catmull-Rom, Bezier curves, and + deterministic value-noise primitives +- `shaders/CultMath.hlsl`, a canonical HLSL mirror include for shader-side + parity. HLSL already owns `float2`, `float3`, and `float4`; the include + exposes `cultmath_*` functions for shared semantics, including CultMath's + safe `normalize` contract. +- `Voronoi.SampleTones`, a C# batch surface that calls the Rust + `cultmath-core` native kernel when `cultmath_core` is available and falls back + to the managed parity path otherwise. + +## Native Core + +The canonical hot-path body lives in `native/cultmath-core`. It exports a C ABI +for batch kernels such as `cultmath_apollonian_voronoi_tones`. + +Runtime wrappers own syntax and integration. Rust owns optimized primitive +kernel behavior and parity fixtures. + +## Example + +```csharp +using CultMath; + +float3 normal = math.normalize(new float3(0.25f, 1.0f, -0.1f)); +float rim = math.saturate(1.0f - math.dot(normal, new float3(0.0f, 0.0f, -1.0f))); +float glow = math.smoothstep(0.2f, 1.0f, rim); +float grain = math.value_noise_bicubic(math.float2(12.0f, 3.5f)); +``` + +```hlsl +#include "CultMath.hlsl" + +float3 normal = cultmath_normalize(float3(0.25, 1.0, -0.1)); +float rim = cultmath_saturate(1.0 - dot(normal, float3(0.0, 0.0, -1.0))); +float glow = cultmath_smoothstep(0.2, 1.0, rim); +float grain = cultmath_value_noise_bicubic(float2(12.0, 3.5)); +``` + +## Provenance + +The small spline, damping, and pursuit helpers were promoted from Aetheria's +local `AetheriaMath` surface. The value-noise family follows public shader +literature from Inigo Quilez's noise and mini-spline articles, expressed in +CultMath names with tests instead of copied project-local helper drift. + +## Build + +```powershell +dotnet build packages/cultmath/src/CultMath/CultMath.csproj +dotnet test packages/cultmath/tests/CultMath.Tests/CultMath.Tests.csproj +cargo test --manifest-path packages/cultmath/native/cultmath-core/Cargo.toml +cargo build --release --manifest-path packages/cultmath/native/cultmath-core/Cargo.toml +.\packages\cultmath\scripts\build-unity-package.ps1 +``` + +## Unity package + +CultMath's Unity 2021.3 surface is the tracked precompiled package at +`packages/cultmath/unity/org.gamecult.cultmath`. Unity consumes `CultMath.dll` and the numeric +`CultMath.hlsl`; it does not compile the repository's current-language C# source +tree and does not require a consumer `csc.rsp` workaround. + +```json +"org.gamecult.cultmath": "https://github.com/GameCult/CultLib.git?path=/packages/cultmath/unity/org.gamecult.cultmath" +``` + +The repository root is not a Unity package. Build the tracked package with +`packages/cultmath/scripts/build-unity-package.ps1`; the script fresh-builds the netstandard2.1 +assembly and rejects stale binaries, missing metadata, unexpected assemblies, +or leaked C# source. + +## Shader Tooling + +CultMath keeps portable DXC outside git under `packages/cultmath/.tools/`: + +```powershell +.\packages\cultmath\tools\get-dxc.ps1 +.\packages\cultmath\tools\get-spirv-cross.ps1 +.\packages\cultmath\tools\compile-hlsl-spirv.ps1 ` + -ShaderPath E:\Projects\Odin\crates\muninn-move-tracker\shaders\MoveSphereCandidate.comp.hlsl ` + -OutputPath E:\Projects\Odin\crates\muninn-move-tracker\artifacts\shader\MoveSphereCandidate.comp.spv ` + -IncludePath E:\Projects\CultLib\packages\cultmath\shaders +.\packages\cultmath\tools\compile-spirv-metal.ps1 ` + -SpirvPath E:\Projects\Odin\crates\muninn-move-tracker\artifacts\shader\MoveSphereCandidate.comp.spv ` + -OutputPath E:\Projects\Odin\crates\muninn-move-tracker\artifacts\shader\MoveSphereCandidate.comp.metal +``` + +DXC emits Vulkan SPIR-V with `-spirv`. `-TargetEnv vulkan1.2` is exposed by the +script, but the current Windows DXC release compiled this shader successfully +with DXC's default Vulkan target while rejecting the dotted target-env value. +The SPIR-V script applies default Vulkan register shifts for cbuffers, textures, +UAVs, and samplers so cross-compiled Metal buffers do not alias by resource +class. SPIRV-Cross emits Metal Shading Language from SPIR-V on Windows; building +an iPad-ready `.metallib` still belongs to Apple's Metal compiler on an Apple +toolchain. + +## Doctrine + +The goal is mechanical sympathy between CPU-side authoring, native kernels, and +GPU-side shader math. If an intrinsic exists here, it should behave like the +shader concept unless a runtime's numeric rules make that impossible. Weird +deviations get tests and documentation, not folklore. + +CultMath is still not a shader generator. Projects should include +`shaders/CultMath.hlsl` directly or add that directory to their shader include +path. If a shared operation needs CPU/GPU parity, put the primitive here and +test it here before copying local math helpers into a renderer or daemon. diff --git a/packages/cultmath/README.md.meta b/packages/cultmath/README.md.meta new file mode 100644 index 00000000..4411da72 --- /dev/null +++ b/packages/cultmath/README.md.meta @@ -0,0 +1,7 @@ +fileFormatVersion: 2 +guid: 867daebafe0ce48479cf043461ee6bdf +TextScriptImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/packages/cultmath/THIRD-PARTY-NOTICES.md b/packages/cultmath/THIRD-PARTY-NOTICES.md new file mode 100644 index 00000000..06ebb6c3 --- /dev/null +++ b/packages/cultmath/THIRD-PARTY-NOTICES.md @@ -0,0 +1,11 @@ +# Third-Party Notices + +## 2D and 3D Simplex Noise + +`math.snoise` and `cultmath_snoise` port the textureless 2D and 3D +simplex-noise functions by Ian McEwan and Ashima Arts, as distributed by Stefan +Gustavson's `webgl-noise` project and Unity Mathematics. + +- Copyright 2011 Ashima Arts. +- Upstream: +- License: MIT. diff --git a/packages/cultmath/THIRD-PARTY-NOTICES.md.meta b/packages/cultmath/THIRD-PARTY-NOTICES.md.meta new file mode 100644 index 00000000..4b43ddaa --- /dev/null +++ b/packages/cultmath/THIRD-PARTY-NOTICES.md.meta @@ -0,0 +1,7 @@ +fileFormatVersion: 2 +guid: 7c28c8631c7d4054eb751d7ce952df33 +TextScriptImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/packages/cultmath/docs.meta b/packages/cultmath/docs.meta new file mode 100644 index 00000000..ee5fab29 --- /dev/null +++ b/packages/cultmath/docs.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: de535b89253410b41b5989f712b80af4 +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/packages/cultmath/docs/design.md b/packages/cultmath/docs/design.md new file mode 100644 index 00000000..de09ee26 --- /dev/null +++ b/packages/cultmath/docs/design.md @@ -0,0 +1,143 @@ +# Design + +CultMath exists to make renderer math readable in every place it needs to live: +Rust hot kernels, C# host code, and HLSL shader code. + +## Parity Target: HLSL + +HLSL is the semantic authority for every type and intrinsic HLSL defines. The +goal is source compatibility: HLSL-shaped math compiles as C# under +`using static CultMath.math;`. C# forces float literal suffixes (`0.5f`); the +rest of the spelling should survive the move. + +That makes these HLSL rules, not options: + +- Every swizzle exists. `float2/3/4`, `int2/3/4`, and `bool2/3/4` expose every + read swizzle of length 2 to 4, repeats included (`v.wwxy`), in both `xyzw` and + `rgba` naming (never mixed), plus single-component `r/g/b/a`. `double2/3` get + lengths 2 and 3 only, because there is no `double4`. Components and swizzles are + writable: `v.xz = f2;`, `c.rgb *= 0.5f;`, and `v[1] = s;` work. Setters exist + only for non-repeating masks, because HLSL rejects `v.xx = ...`. +- Every HLSL mixed constructor exists, as a struct constructor and as a `math` + function: `float4(float2, float2)`, `float4(float, float2, float)`, + `float3(float, float2)`, the copy form `float3(float3)`, and the rest, for the + float, int, and bool families (double gets constructors only; a `math.double3` + function would hide the `double3` type name). +- Swizzles and mixed constructors are generated. `tools/generate-swizzles.ps1` + writes the checked-in `src/CultMath/Swizzles.g.cs` (partial structs plus a + partial `math`). Edit the generator, rerun it, and commit both; never edit the + output by hand. +- Comparison operators (`<`, `>`, `<=`, `>=`, `==`, `!=`) are component-wise and + return `bool2`/`bool3`/`bool4`. Reduce them with `any` and `all`. + `Equals`/`GetHashCode` stay value-based so vectors still work as dictionary + keys. +- `select(condition, whenTrue, whenFalse)` uses HLSL's argument order. +- Matrices are row-major: constructors take rows, `m[i]` returns a reference to + row i (so `m[1][2] = s;` writes one element), `_mRC` is row R column C, and + products go through `mul` (`mul(m, v)` treats `v` as a column, `mul(v, m)` as a + row). `*` on matrices would be component-wise in HLSL, so CultMath does not + define it. +- `m[r][c] = v` writes through locals, array elements, fields of classes, and + ref locals. Through a readonly field, an `in` parameter, a property + (auto-properties included), or a static readonly value such as + `float3x3.identity`, C# takes a defensive copy: the write compiles and is + silently lost. Copy the matrix to a local, write to it, and assign it back. +- Scalar/vector arithmetic and comparisons have explicit overloads on both + sides rather than leaning on implicit scalar splat. +- Intrinsics return HLSL's types and edge cases: `sign` returns `int`/`intN` + (NaN gives 0); `step(y, x)` is `x < y ? 0 : 1` (NaN gives 1), which is how dxc + lowers it, in C# and in the HLSL mirror; float and double `min` and `max`, and + float `clamp` and `saturate`, follow the DXIL `FMin`/`FMax`/`Saturate` operations dxc emits for + them: `min(a, b)` is `a < b ? a : b`, `max(a, b)` is `a >= b ? a : b`, a NaN + operand returns the other (so `saturate(NaN)` is 0), `clamp(x, a, b)` is + `min(max(x, a), b)`, and `saturate(x)` is `min(1, max(0, x))`, which maps -0 + to +0; `any`/`all` accept numeric vectors (a component is true when it is + not 0); `min`, `max`, and `abs` on int vectors return int vectors. + +dxc marks float arithmetic and compares `fast` (no NaNs) unless a value is +`precise`, so drivers may optimize NaN handling away and NaN results on real +GPUs are not guaranteed. CultMath defines the CPU result by dxc's lowering and +the DXIL operation spec (DirectXShaderCompiler `docs/DXIL.rst`). + +NaN does not reliably propagate through CultMath, just as it does not on the +GPU. Because `min`, `max`, and `saturate` return the non-NaN operand, some +functions turn NaN or infinity into numbers: `smoothstep(a, a, a)` is 0; +infinite inputs to `smoothstep` and the intercept helpers give numbers; the +bezier, `smoothstep01`, and `smootherstep` functions return finite values for +NaN input; and `normalize((NaN, 2.41))` is `(NaN, 2.41e20)`, because the NaN +length is clamped to the epsilon and the other component is divided by it. +Callers that must detect bad data should check `isnan` before calling these +functions. + +Deferred until a consumer needs them: `float4x4`, `transpose`, `determinant`, +and HLSL's one-based `_11`..`_33` element names. + +### HLSL Target: Source Transformations + +`HlslSourceCompatibilityTests` compiles the real `shaders/CultMath.hlsl` with +Roslyn against CultMath. These transformations are the complete list; if a body +needs anything else, the test fails: + +1. Include guards (`#ifndef`/`#define`/`#endif`) are dropped. +2. Functions taking `Texture2D`/`SamplerState` are dropped; GPU resources have + no CultMath analog. +3. File-scope `static const` becomes `const`. +4. Floating literals gain `f` (`0.5` becomes `0.5f`). +5. The file body is wrapped in a class, since C# has no free functions; HLSL + functions become private instance methods. + +HLSL constructs that stay outside the target and must not appear in the mirror: +`const` locals of vector type (C# `const` is primitives only; write a plain +local) and writes through a swizzle component (`v.xy.x = s`, which C# rejects as +modifying a temporary). + +The HLSL side is checked too: `tools/get-dxc.ps1` fetches Microsoft's +DirectXShaderCompiler, and +`dxc -T lib_6_3 -HV 2021 shaders/CultMath.hlsl` must compile cleanly. + +The mirror test compares bit patterns (any NaN equals any NaN; -0 differs from +0). It proves the text of `CultMath.hlsl` computes the same float32 results as +C# `math` on the CPU, not that a GPU agrees: driver `sin` precision alone makes +`cultmath_hash` and value noise differ bit for bit on hardware. The mirror is +compiled under `using static CultMath.math;`, so its intrinsics are C# `math` +itself: the test proves the composition in the text, not the intrinsic rules, +which only `HlslSemanticsTests` pins. + +Where HLSL is silent, CultMath keeps its own decisions and does not defer to +Unity.Mathematics: `normalize` is safe (divides by `max(length, 1e-20)`), +`hash` returns float, and `Random` is CultMath's own xorshift32. Engine-shaped +helpers that HLSL lacks (`float2x2.Rotate`, `float3x3.Euler`, `quaternion`, +`snoise`) enter only when a consumer needs them, with their semantics stated in +code. + +Known C# friction: under `using static CultMath.math;` the constructor +functions (`float3(...)`, `float3x3(...)`) hide the type names in member access, +so static members need a qualified type (`CultMath.float3x3.Euler(...)`). + +## Rules + +- Keep type names HLSL-shaped: `float2`, `float3`, `float4`, `float3x3`, and `math`. +- Keep Rust as the owner of native hot-path kernels and parity fixtures. +- Keep C#, shader, and scripting surfaces as runtime wrappers, not competing + math authorities. +- Keep `shaders/CultMath.hlsl` as the canonical shader mirror for shared + CultMath functions HLSL does not already provide (HLSL intrinsics such as + `mul`, `any`, and `log` need no mirror). Project-local shader helpers should + move here once more than one organ needs them or once CPU/GPU parity matters. +- Implement public shader semantics directly; do not copy Unity.Mathematics. +- Prefer plain mutable value types with public component fields, as HLSL does. +- Prefer component-wise overloads over clever generic machinery until the API + earns that complexity. +- Keep the surface boring and exact. Fill what a consumer uses, guided by HLSL + semantics; noise primitives, matrices, quaternions, and packing helpers need an + owner before they enter. +- Keep the core assembly engine-free. Unity conversions live in the Unity + package's `CultMath.UnityBridge` assembly. + +## Non-Goals + +- Compiling C# to shaders. +- Generating HLSL from Rust or Rust from HLSL. +- Replacing `System.Numerics` for general .NET work. +- Recreating every Unity.Mathematics type because a spreadsheet somewhere says + “coverage.” That way lies decorative obesity. diff --git a/packages/cultmath/docs/design.md.meta b/packages/cultmath/docs/design.md.meta new file mode 100644 index 00000000..0d31ca6a --- /dev/null +++ b/packages/cultmath/docs/design.md.meta @@ -0,0 +1,7 @@ +fileFormatVersion: 2 +guid: 07021a8be816e3448a9490c10845b0aa +TextScriptImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/packages/cultmath/native.meta b/packages/cultmath/native.meta new file mode 100644 index 00000000..f4135336 --- /dev/null +++ b/packages/cultmath/native.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: 9d1cb2ac88c530b4d9da456523115e12 +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/packages/cultmath/native/cultmath-core.meta b/packages/cultmath/native/cultmath-core.meta new file mode 100644 index 00000000..48a2ef76 --- /dev/null +++ b/packages/cultmath/native/cultmath-core.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: a42d65478346c354c8b4f353c5581b49 +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/packages/cultmath/native/cultmath-core/Cargo.lock b/packages/cultmath/native/cultmath-core/Cargo.lock new file mode 100644 index 00000000..09598b97 --- /dev/null +++ b/packages/cultmath/native/cultmath-core/Cargo.lock @@ -0,0 +1,7 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "cultmath-core" +version = "0.1.0" diff --git a/packages/cultmath/native/cultmath-core/Cargo.lock.meta b/packages/cultmath/native/cultmath-core/Cargo.lock.meta new file mode 100644 index 00000000..9ca3ec60 --- /dev/null +++ b/packages/cultmath/native/cultmath-core/Cargo.lock.meta @@ -0,0 +1,7 @@ +fileFormatVersion: 2 +guid: f1c365f7eead2684ba621124e653bdab +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/packages/cultmath/native/cultmath-core/Cargo.toml b/packages/cultmath/native/cultmath-core/Cargo.toml new file mode 100644 index 00000000..cf7819a0 --- /dev/null +++ b/packages/cultmath/native/cultmath-core/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "cultmath-core" +version = "0.1.0" +edition = "2021" + +[lib] +name = "cultmath_core" +crate-type = ["cdylib", "staticlib"] + +[profile.release] +lto = "thin" +codegen-units = 1 +panic = "abort" diff --git a/packages/cultmath/native/cultmath-core/Cargo.toml.meta b/packages/cultmath/native/cultmath-core/Cargo.toml.meta new file mode 100644 index 00000000..8875de09 --- /dev/null +++ b/packages/cultmath/native/cultmath-core/Cargo.toml.meta @@ -0,0 +1,7 @@ +fileFormatVersion: 2 +guid: cdd2d4eab053b764b9a06d36b080e43e +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/packages/cultmath/native/cultmath-core/src.meta b/packages/cultmath/native/cultmath-core/src.meta new file mode 100644 index 00000000..8e5232db --- /dev/null +++ b/packages/cultmath/native/cultmath-core/src.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: 7de448b7e258d2c448bb7391051cc628 +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/packages/cultmath/native/cultmath-core/src/lib.rs b/packages/cultmath/native/cultmath-core/src/lib.rs new file mode 100644 index 00000000..46e7ced6 --- /dev/null +++ b/packages/cultmath/native/cultmath-core/src/lib.rs @@ -0,0 +1,213 @@ +#[repr(C)] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct CultMathColor32 { + pub r: u8, + pub g: u8, + pub b: u8, + pub a: u8, +} + +#[no_mangle] +pub unsafe extern "C" fn cultmath_apollonian_voronoi_tones( + xs: *const f32, + ys: *const f32, + tones: *const u8, + spans: *const f32, + count: usize, + resolution_y: f32, + frame_index: i32, + out_colors: *mut CultMathColor32, +) -> i32 { + if xs.is_null() || ys.is_null() || tones.is_null() || spans.is_null() || out_colors.is_null() { + return -1; + } + + let xs = unsafe { std::slice::from_raw_parts(xs, count) }; + let ys = unsafe { std::slice::from_raw_parts(ys, count) }; + let tones = unsafe { std::slice::from_raw_parts(tones, count) }; + let spans = unsafe { std::slice::from_raw_parts(spans, count) }; + let out_colors = unsafe { std::slice::from_raw_parts_mut(out_colors, count) }; + for index in 0..count { + out_colors[index] = tone_color( + xs[index], + ys[index], + resolution_y, + frame_index, + tones[index], + spans[index], + ); + } + + 0 +} + +fn tone_color(px: f32, py: f32, resolution_y: f32, frame_index: i32, tone: u8, span: f32) -> CultMathColor32 { + let sample = stable_sample(px, py, resolution_y, frame_index, tone, span); + let luminance = saturate(sample.0 * 0.299 + sample.1 * 0.587 + sample.2 * 0.114); + match tone { + 3 => { + let glow = 0.62 + luminance * 0.38; + color(255.0 * glow, 96.0 * glow, 14.0 * glow) + } + 4 => { + let glow = 0.68 + luminance * 0.32; + color(245.0 * glow, 248.0 * glow, 252.0 * glow) + } + _ => { + let (gain, lift) = match tone { + 1 => (0.42, 14.0), + 2 => (1.05, 24.0), + _ => (0.24, 4.0), + }; + color( + lift + sample.0 * 255.0 * gain, + lift + sample.1 * 255.0 * gain, + lift + sample.2 * 255.0 * gain, + ) + } + } +} + +fn stable_sample(px: f32, py: f32, resolution_y: f32, frame_index: i32, tone: u8, span: f32) -> (f32, f32, f32) { + let seed = px * 12.9898 + py * 78.233 + hash1((tone as f32 + 1.0) * 19.19); + let jitter_x = (hash1(seed) - 0.5) * span; + let jitter_y = (hash1(seed + 37.17) - 0.5) * span; + voronoi(px + jitter_x, py + jitter_y, resolution_y, frame_index) +} + +fn voronoi(px: f32, py: f32, resolution_y: f32, frame_index: i32) -> (f32, f32, f32) { + let time = frame_index as f32 / 120.0; + let scale = 6.0; + let x = scale * px / resolution_y.max(1.0); + let y = scale * py / resolution_y.max(1.0); + let nx = x.floor(); + let ny = y.floor(); + let fx = frac(x); + let fy = frac(y); + let mut distance = 8.0; + let mut red = 0.0; + let mut green = 0.0; + let mut blue = 0.0; + let smoothness = 0.005; + + for j in -2..=2 { + for i in -2..=2 { + let gx = i as f32; + let gy = j as f32; + let (mut ox, mut oy) = hash2(nx + gx, ny + gy); + let weight = ox * 0.5 + 0.5; + ox = 0.5 + 0.5 * (time + 6.2831 * ox).sin(); + oy = 0.5 + 0.5 * (time + 6.2831 * oy).sin(); + let dx = (gx - fx + ox).abs(); + let dy = (gy - fy + oy).abs(); + let d = dx.max(dy) * weight; + let seed = hash1((nx + gx) * 7.0 + (ny + gy) * 113.0); + let hue = frac((nx + gx) * 0.173 + (ny + gy) * 0.379 + seed * 0.431); + let (candidate_red, candidate_green, candidate_blue) = pastel_spectrum(hue); + let h = smoothstep(0.0, 1.0, 0.5 + 0.5 * (distance - d) / smoothness); + let correction = h * (1.0 - h) * smoothness / (1.0 + 3.0 * smoothness); + distance = lerp(distance, d, h) - correction; + red = lerp(red, candidate_red, h) - correction; + green = lerp(green, candidate_green, h) - correction; + blue = lerp(blue, candidate_blue, h) - correction; + } + } + + let edge_dimming = 1.0 - 0.1 * smoothstep(0.04, 0.05, distance); + ( + (red * edge_dimming).max(0.0), + (green * edge_dimming).max(0.0), + (blue * edge_dimming).max(0.0), + ) +} + +fn hash1(value: f32) -> f32 { + frac(value.sin() * 43758.5453) +} + +fn hash2(x: f32, y: f32) -> (f32, f32) { + ( + frac((x * 127.1 + y * 311.7).sin() * 43758.5453), + frac((x * 269.5 + y * 183.3).sin() * 43758.5453), + ) +} + +fn frac(value: f32) -> f32 { + value - value.floor() +} + +fn saturate(value: f32) -> f32 { + value.clamp(0.0, 1.0) +} + +fn smoothstep(edge0: f32, edge1: f32, value: f32) -> f32 { + let t = saturate((value - edge0) / (edge1 - edge0)); + t * t * (3.0 - 2.0 * t) +} + +fn lerp(start: f32, end: f32, amount: f32) -> f32 { + start + (end - start) * amount +} + +fn pastel_spectrum(hue: f32) -> (f32, f32, f32) { + hsv_to_rgb(hue, 0.52, 0.98) +} + +fn hsv_to_rgb(hue: f32, saturation: f32, value: f32) -> (f32, f32, f32) { + let h = frac(hue) * 6.0; + let c = value * saturation; + let x = c * (1.0 - ((h % 2.0) - 1.0).abs()); + let m = value - c; + let (r, g, b) = if h < 1.0 { + (c, x, 0.0) + } else if h < 2.0 { + (x, c, 0.0) + } else if h < 3.0 { + (0.0, c, x) + } else if h < 4.0 { + (0.0, x, c) + } else if h < 5.0 { + (x, 0.0, c) + } else { + (c, 0.0, x) + }; + + (r + m, g + m, b + m) +} + +fn color(r: f32, g: f32, b: f32) -> CultMathColor32 { + CultMathColor32 { + r: r.clamp(0.0, 255.0) as u8, + g: g.clamp(0.0, 255.0) as u8, + b: b.clamp(0.0, 255.0) as u8, + a: 255, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn batch_sampler_writes_colors() { + let xs = [0.0, 20.0, 100.0]; + let ys = [0.0, 30.0, 140.0]; + let tones = [0, 3, 4]; + let spans = [1920.0, 8.0, 8.0]; + let mut out = [CultMathColor32::default(); 3]; + let rc = unsafe { + cultmath_apollonian_voronoi_tones( + xs.as_ptr(), + ys.as_ptr(), + tones.as_ptr(), + spans.as_ptr(), + xs.len(), + 1080.0, + 12, + out.as_mut_ptr(), + ) + }; + assert_eq!(rc, 0); + assert!(out.iter().any(|color| color.r != 0 || color.g != 0 || color.b != 0)); + } +} diff --git a/packages/cultmath/native/cultmath-core/src/lib.rs.meta b/packages/cultmath/native/cultmath-core/src/lib.rs.meta new file mode 100644 index 00000000..2fa8aee8 --- /dev/null +++ b/packages/cultmath/native/cultmath-core/src/lib.rs.meta @@ -0,0 +1,7 @@ +fileFormatVersion: 2 +guid: b06c69c432a633f4584830d6b28d4473 +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/packages/cultmath/scripts.meta b/packages/cultmath/scripts.meta new file mode 100644 index 00000000..6f03c566 --- /dev/null +++ b/packages/cultmath/scripts.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: 0ddff5b782cf14e43ae5cda048e040d8 +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/packages/cultmath/scripts/build-unity-package.ps1 b/packages/cultmath/scripts/build-unity-package.ps1 new file mode 100644 index 00000000..b64e9446 --- /dev/null +++ b/packages/cultmath/scripts/build-unity-package.ps1 @@ -0,0 +1,159 @@ +param( + [string] $Configuration = "Release", + [string] $OutputDirectory = "artifacts\unity\org.gamecult.cultmath" +) + +$ErrorActionPreference = "Stop" + +$repoRoot = Split-Path -Parent $PSScriptRoot +$projectPath = Join-Path $repoRoot "src\CultMath\CultMath.csproj" +$templateRoot = Join-Path $repoRoot "unity\org.gamecult.cultmath" +$buildRoot = Join-Path $repoRoot "artifacts\cultmath-unity-build" +$outputRoot = if ([System.IO.Path]::IsPathRooted($OutputDirectory)) { + $OutputDirectory +} else { + Join-Path $repoRoot $OutputDirectory +} +$freshAssembly = Join-Path $buildRoot "CultMath.dll" +$freshSymbols = Join-Path $buildRoot "CultMath.pdb" +$trackedPluginRoot = Join-Path $templateRoot "Runtime\Plugins" +$trackedAssembly = Join-Path $trackedPluginRoot "CultMath.dll" +$trackedSymbols = Join-Path $trackedPluginRoot "CultMath.pdb" + +if (Test-Path -LiteralPath $buildRoot) { + Remove-Item -LiteralPath $buildRoot -Recurse -Force +} +if (Test-Path -LiteralPath $outputRoot) { + Remove-Item -LiteralPath $outputRoot -Recurse -Force +} + +# The tracked DLL and pdb are committed beside their source, so neither may name a commit: Source Link +# writes the commit SHA into the pdb, and the DLL carries that pdb's content id. +dotnet build $projectPath -c $Configuration -f netstandard2.1 -o $buildRoot --no-incremental ` + --disable-build-servers -p:UseSharedCompilation=false -p:ContinuousIntegrationBuild=true -p:EnableSourceLink=false -m:1 +if ($LASTEXITCODE -ne 0) { + throw "CultMath Unity assembly build failed with exit code $LASTEXITCODE" +} + +foreach ($path in @($freshAssembly, $freshSymbols)) { + if (-not (Test-Path -LiteralPath $path -PathType Leaf)) { + throw "CultMath Unity build is missing $path" + } +} + +$requiredTemplateFiles = @( + "LICENSE.md", + "LICENSE.md.meta", + "README.md", + "README.md.meta", + "THIRD-PARTY-NOTICES.md", + "THIRD-PARTY-NOTICES.md.meta", + "package.json", + "package.json.meta", + "Runtime\Plugins\CultMath.dll", + "Runtime\Plugins\CultMath.dll.meta", + "Runtime\Plugins\CultMath.pdb", + "Runtime\Plugins\CultMath.pdb.meta", + "Runtime\UnityBridge\CultMath.UnityBridge.asmdef", + "Runtime\UnityBridge\CultMath.UnityBridge.asmdef.meta", + "Runtime\UnityBridge\UnityConversions.cs", + "Runtime\UnityBridge\UnityConversions.cs.meta", + "Shaders\CultMath.hlsl", + "Shaders\CultMath.hlsl.meta" +) +foreach ($relativePath in $requiredTemplateFiles) { + if (-not (Test-Path -LiteralPath (Join-Path $templateRoot $relativePath) -PathType Leaf)) { + throw "Tracked CultMath Unity package is incomplete: missing $relativePath" + } +} + +$unityAssetFiles = @(Get-ChildItem -LiteralPath $templateRoot -Recurse -File | Where-Object { + $_.Extension -in @(".asmdef", ".cs", ".dll", ".pdb", ".hlsl") +}) +$missingAssetMetas = @($unityAssetFiles | Where-Object { -not (Test-Path -LiteralPath ($_.FullName + ".meta")) }) +$missingDirectoryMetas = @(Get-ChildItem -LiteralPath $templateRoot -Recurse -Directory | Where-Object { + -not (Test-Path -LiteralPath ($_.FullName + ".meta")) +}) +if ($missingAssetMetas.Count -ne 0 -or $missingDirectoryMetas.Count -ne 0) { + $missing = @($missingAssetMetas.FullName) + @($missingDirectoryMetas.FullName) + throw "Tracked CultMath Unity package has assets without stable .meta files: $($missing -join ', ')" +} + +$manifest = Get-Content -LiteralPath (Join-Path $templateRoot "package.json") -Raw | ConvertFrom-Json +$pluginMeta = Get-Content -LiteralPath ($trackedAssembly + ".meta") -Raw +if ($pluginMeta -notmatch '(?m)^\s*isExplicitlyReferenced:\s*0\s*$') { + throw "CultMath.dll must be auto-referenced; explicit-only plugins do not reach dependent Unity asmdefs" +} +# The precompiled DLL is the only authority for CultMath's numeric types. The one +# compiled assembly allowed beside it is the UnityEngine conversion bridge, which +# references that DLL and must not re-declare core sources. +$bridgeRoot = Join-Path $templateRoot "Runtime\UnityBridge" +$bridgeDefinition = Join-Path $bridgeRoot "CultMath.UnityBridge.asmdef" +$definitions = @(Get-ChildItem -LiteralPath $templateRoot -Recurse -File -Filter "*.asmdef") +if ($definitions.Count -ne 1 -or $definitions[0].FullName -ne $bridgeDefinition) { + throw "CultMath's Unity package may define only the Runtime\UnityBridge assembly; found: $($definitions.FullName -join ', ')" +} +$bridge = Get-Content -LiteralPath $bridgeDefinition -Raw | ConvertFrom-Json +if ($bridge.noEngineReferences -or -not $bridge.overrideReferences -or @($bridge.precompiledReferences) -notcontains "CultMath.dll") { + throw "CultMath.UnityBridge must reference UnityEngine and the precompiled CultMath.dll" +} +$expectedFileVersion = "$($manifest.version).0" +$trackedFileVersion = [Diagnostics.FileVersionInfo]::GetVersionInfo($trackedAssembly).FileVersion +$freshFileVersion = [Diagnostics.FileVersionInfo]::GetVersionInfo($freshAssembly).FileVersion +if ($trackedFileVersion -ne $expectedFileVersion -or $freshFileVersion -ne $expectedFileVersion) { + throw "CultMath Unity assemblies must match package version $expectedFileVersion; tracked=$trackedFileVersion fresh=$freshFileVersion" +} + +foreach ($pair in @(@($freshAssembly, $trackedAssembly), @($freshSymbols, $trackedSymbols))) { + $freshHash = (Get-FileHash -LiteralPath $pair[0] -Algorithm SHA256).Hash + $trackedHash = (Get-FileHash -LiteralPath $pair[1] -Algorithm SHA256).Hash + if ($freshHash -ne $trackedHash) { + throw "Tracked Unity artifact is stale: $($pair[1]) differs from a fresh deterministic build" + } +} + +$trackedSourceFiles = @(Get-ChildItem -LiteralPath $templateRoot -Recurse -File -Filter "*.cs") +$straySources = @($trackedSourceFiles | Where-Object { $_.DirectoryName -ne $bridgeRoot }) +if ($trackedSourceFiles.Count -eq 0 -or $straySources.Count -ne 0) { + throw "CultMath Unity package may compile only bridge sources under Runtime\UnityBridge: $($straySources.FullName -join ', ')" +} + +# Check what the bridge declares, not what its files are called: it may declare types only in +# CultMath.UnityBridge and never a type name the core CultMath sources declare. +$declarationPattern = '\b(?:struct|class|enum|interface|record|delegate\s+[\w<>.,\s]+?)\s+(\w+)' +function Get-CodeWithoutComments([string]$path) { + $text = Get-Content -LiteralPath $path -Raw + return [regex]::Replace($text, '(?s)/\*.*?\*/|//[^\r\n]*', '') +} +$coreTypeNames = @(Get-ChildItem -LiteralPath (Join-Path $repoRoot "src\CultMath") -File -Filter "*.cs" | ForEach-Object { + [regex]::Matches((Get-CodeWithoutComments $_.FullName), $declarationPattern) | ForEach-Object { $_.Groups[1].Value } +} | Sort-Object -Unique) +if ($coreTypeNames -notcontains "float3" -or $coreTypeNames -notcontains "math") { + throw "Could not read CultMath core type declarations; the bridge check would be vacuous" +} +foreach ($source in $trackedSourceFiles) { + $code = Get-CodeWithoutComments $source.FullName + $redeclared = @([regex]::Matches($code, $declarationPattern) | ForEach-Object { $_.Groups[1].Value } | Where-Object { $coreTypeNames -contains $_ }) + $foreignNamespaces = @([regex]::Matches($code, '\bnamespace\s+([\w.]+)') | ForEach-Object { $_.Groups[1].Value } | Where-Object { $_ -ne "CultMath.UnityBridge" }) + if ($redeclared.Count -ne 0 -or $foreignNamespaces.Count -ne 0) { + throw "CultMath.UnityBridge source $($source.Name) re-declares core types [$($redeclared -join ', ')] or uses namespaces [$($foreignNamespaces -join ', ')]; the precompiled DLL owns CultMath's types" + } +} + +New-Item -ItemType Directory -Force -Path $outputRoot | Out-Null +Copy-Item -Path (Join-Path $templateRoot "*") -Destination $outputRoot -Recurse -Force + +$stagedAssemblies = @(Get-ChildItem -LiteralPath (Join-Path $outputRoot "Runtime\Plugins") -Filter "*.dll") +$stagedBridgeRoot = Join-Path $outputRoot "Runtime\UnityBridge" +$stagedSources = @(Get-ChildItem -LiteralPath $outputRoot -Recurse -File -Filter "*.cs") +$stagedDefinitions = @(Get-ChildItem -LiteralPath $outputRoot -Recurse -File -Filter "*.asmdef") +$stagedStraySources = @($stagedSources | Where-Object { $_.DirectoryName -ne $stagedBridgeRoot }) +if ($stagedAssemblies.Count -ne 1 -or $stagedAssemblies[0].Name -ne "CultMath.dll" -or $stagedStraySources.Count -ne 0 -or $stagedDefinitions.Count -ne 1) { + throw "CultMath package must contain one owned auto-referenced DLL and only the UnityBridge assembly as source" +} + +Write-Host "CultMath Unity package: $outputRoot" +Write-Host "Package: $($manifest.name)@$($manifest.version)" +Write-Host "Managed assemblies: $($stagedAssemblies.Count)" +Write-Host "Bridge C# source files: $($stagedSources.Count)" +Write-Host "Asmdef definitions: $($stagedDefinitions.Count)" diff --git a/packages/cultmath/shaders.meta b/packages/cultmath/shaders.meta new file mode 100644 index 00000000..46f9d445 --- /dev/null +++ b/packages/cultmath/shaders.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: 3bed74ebcba445543800218dbe185034 +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/packages/cultmath/shaders/CultMath.hlsl b/packages/cultmath/shaders/CultMath.hlsl new file mode 100644 index 00000000..365705f5 --- /dev/null +++ b/packages/cultmath/shaders/CultMath.hlsl @@ -0,0 +1,356 @@ +#ifndef CULTMATH_HLSL +#define CULTMATH_HLSL + +// CultMath HLSL mirror. +// +// HLSL already owns the float2/float3/float4 type names and most primitive +// intrinsics. This include names the CultMath semantic surface explicitly so +// shader code, C# wrappers, and Rust/native parity fixtures can share the same +// small vocabulary without local helper drift. + +static const float CULTMATH_PI = 3.14159265358979323846; +static const float CULTMATH_TAU = 6.28318530717958647692; +static const float CULTMATH_HALF_PI = 1.57079632679489661923; +static const float CULTMATH_NORMALIZE_EPSILON = 1.0e-20; + +float cultmath_radians(float degrees_value) { return degrees_value * (CULTMATH_PI / 180.0); } +float cultmath_degrees(float radians_value) { return radians_value * (180.0 / CULTMATH_PI); } + +float cultmath_frac(float value) { return value - floor(value); } +float2 cultmath_frac(float2 value) { return value - floor(value); } +float3 cultmath_frac(float3 value) { return value - floor(value); } +float4 cultmath_frac(float4 value) { return value - floor(value); } + +float cultmath_clamp(float value, float minimum, float maximum) { return min(max(value, minimum), maximum); } +float2 cultmath_clamp(float2 value, float2 minimum, float2 maximum) { return min(max(value, minimum), maximum); } +float3 cultmath_clamp(float3 value, float3 minimum, float3 maximum) { return min(max(value, minimum), maximum); } +float4 cultmath_clamp(float4 value, float4 minimum, float4 maximum) { return min(max(value, minimum), maximum); } + +// DXIL Saturate is FMin(1, FMax(0, x)); dxc lowers min/max to the same FMin/FMax, NaN rules included. +float cultmath_saturate(float value) { return min(1.0, max(0.0, value)); } +float2 cultmath_saturate(float2 value) { return min(float2(1.0, 1.0), max(float2(0.0, 0.0), value)); } +float3 cultmath_saturate(float3 value) { return min(float3(1.0, 1.0, 1.0), max(float3(0.0, 0.0, 0.0), value)); } +float4 cultmath_saturate(float4 value) { return min(float4(1.0, 1.0, 1.0, 1.0), max(float4(0.0, 0.0, 0.0, 0.0), value)); } + +float cultmath_lerp(float start, float end, float amount) { return start + (end - start) * amount; } +float2 cultmath_lerp(float2 start, float2 end, float2 amount) { return start + (end - start) * amount; } +float3 cultmath_lerp(float3 start, float3 end, float3 amount) { return start + (end - start) * amount; } +float4 cultmath_lerp(float4 start, float4 end, float4 amount) { return start + (end - start) * amount; } + +float cultmath_step(float edge, float value) { return value < edge ? 0.0 : 1.0; } +float2 cultmath_step(float2 edge, float2 value) { return float2(cultmath_step(edge.x, value.x), cultmath_step(edge.y, value.y)); } +float3 cultmath_step(float3 edge, float3 value) { return float3(cultmath_step(edge.x, value.x), cultmath_step(edge.y, value.y), cultmath_step(edge.z, value.z)); } +float4 cultmath_step(float4 edge, float4 value) { return float4(cultmath_step(edge.x, value.x), cultmath_step(edge.y, value.y), cultmath_step(edge.z, value.z), cultmath_step(edge.w, value.w)); } + +float cultmath_smoothstep(float minimum, float maximum, float value) +{ + float t = cultmath_saturate((value - minimum) / (maximum - minimum)); + return t * t * (3.0 - 2.0 * t); +} + +float2 cultmath_smoothstep(float2 minimum, float2 maximum, float2 value) +{ + float2 t = cultmath_saturate((value - minimum) / (maximum - minimum)); + return t * t * (float2(3.0, 3.0) - float2(2.0, 2.0) * t); +} + +float3 cultmath_smoothstep(float3 minimum, float3 maximum, float3 value) +{ + float3 t = cultmath_saturate((value - minimum) / (maximum - minimum)); + return t * t * (float3(3.0, 3.0, 3.0) - float3(2.0, 2.0, 2.0) * t); +} + +float4 cultmath_smoothstep(float4 minimum, float4 maximum, float4 value) +{ + float4 t = cultmath_saturate((value - minimum) / (maximum - minimum)); + return t * t * (float4(3.0, 3.0, 3.0, 3.0) - float4(2.0, 2.0, 2.0, 2.0) * t); +} + +float cultmath_smoothstep01(float value) { return cultmath_smoothstep(0.0, 1.0, value); } + +float cultmath_smootherstep(float value) +{ + value = cultmath_saturate(value); + return value * value * value * (value * (value * 6.0 - 15.0) + 10.0); +} + +float2 cultmath_snoise_mod289(float2 value) { return value - floor(value * (1.0 / 289.0)) * 289.0; } +float3 cultmath_snoise_mod289(float3 value) { return value - floor(value * (1.0 / 289.0)) * 289.0; } +float4 cultmath_snoise_mod289(float4 value) { return value - floor(value * (1.0 / 289.0)) * 289.0; } +float3 cultmath_snoise_permute(float3 value) { return cultmath_snoise_mod289(((value * 34.0) + 1.0) * value); } +float4 cultmath_snoise_permute(float4 value) { return cultmath_snoise_mod289(((value * 34.0) + 1.0) * value); } + +// Ashima Arts / Ian McEwan 3D simplex noise (MIT), same float32 evaluation +// order as the C# math.snoise(float3) mirror. +float cultmath_snoise(float3 value) +{ + float2 c = float2(1.0 / 6.0, 1.0 / 3.0); + float3 i = floor(value + dot(value, c.yyy)); + float3 x0 = value - i + dot(i, c.xxx); + float3 g = step(x0.yzx, x0.xyz); + float3 l = 1.0 - g; + float3 i1 = min(g.xyz, l.zxy); + float3 i2 = max(g.xyz, l.zxy); + float3 x1 = x0 - i1 + c.xxx; + float3 x2 = x0 - i2 + c.yyy; + float3 x3 = x0 - 0.5; + + i = cultmath_snoise_mod289(i); + float4 p = cultmath_snoise_permute(cultmath_snoise_permute(cultmath_snoise_permute( + i.z + float4(0.0, i1.z, i2.z, 1.0)) + i.y + float4(0.0, i1.y, i2.y, 1.0)) + i.x + float4(0.0, i1.x, i2.x, 1.0)); + + const float n_ = 0.142857142857; + float3 ns = float3(2.0 * n_, 0.5 * n_ - 1.0, n_); + float4 j = p - 49.0 * floor(p * ns.z * ns.z); + float4 x_ = floor(j * ns.z); + float4 y_ = floor(j - 7.0 * x_); + float4 x = x_ * ns.x + ns.yyyy; + float4 y = y_ * ns.x + ns.yyyy; + float4 h = 1.0 - abs(x) - abs(y); + + float4 b0 = float4(x.xy, y.xy); + float4 b1 = float4(x.zw, y.zw); + float4 s0 = floor(b0) * 2.0 + 1.0; + float4 s1 = floor(b1) * 2.0 + 1.0; + float4 sh = -step(h, 0.0); + float4 a0 = b0.xzyw + s0.xzyw * sh.xxyy; + float4 a1 = b1.xzyw + s1.xzyw * sh.zzww; + + float3 p0 = float3(a0.xy, h.x); + float3 p1 = float3(a0.zw, h.y); + float3 p2 = float3(a1.xy, h.z); + float3 p3 = float3(a1.zw, h.w); + float4 norm = 1.79284291400159 - 0.85373472095314 * float4(dot(p0, p0), dot(p1, p1), dot(p2, p2), dot(p3, p3)); + p0 *= norm.x; + p1 *= norm.y; + p2 *= norm.z; + p3 *= norm.w; + + float4 m = max(0.6 - float4(dot(x0, x0), dot(x1, x1), dot(x2, x2), dot(x3, x3)), 0.0); + m = m * m; + return 42.0 * dot(m * m, float4(dot(p0, x0), dot(p1, x1), dot(p2, x2), dot(p3, x3))); +} + +float cultmath_snoise(float2 value) +{ + float4 c = float4( + 0.211324865405187, + 0.366025403784439, + -0.577350269189626, + 0.024390243902439); + float2 i = floor(value + dot(value, c.yy)); + float2 x0 = value - i + dot(i, c.xx); + float2 i1 = x0.x > x0.y ? float2(1.0, 0.0) : float2(0.0, 1.0); + float4 x12 = float4(x0.x + c.x - i1.x, x0.y + c.x - i1.y, x0.x + c.z, x0.y + c.z); + + i = cultmath_snoise_mod289(i); + float3 p = cultmath_snoise_permute( + cultmath_snoise_permute(i.y + float3(0.0, i1.y, 1.0)) + i.x + float3(0.0, i1.x, 1.0)); + float3 m = max(0.5 - float3(dot(x0, x0), dot(x12.xy, x12.xy), dot(x12.zw, x12.zw)), 0.0); + m *= m; + m *= m; + + float3 x = 2.0 * frac(p * c.w) - 1.0; + float3 h = abs(x) - 0.5; + float3 ox = floor(x + 0.5); + float3 a0 = x - ox; + m *= 1.79284291400159 - 0.85373472095314 * (a0 * a0 + h * h); + + float3 g = float3( + a0.x * x0.x + h.x * x0.y, + a0.y * x12.x + h.y * x12.y, + a0.z * x12.z + h.z * x12.w); + return 130.0 * dot(m, g); +} + +float cultmath_lengthsq(float2 value) { return dot(value, value); } +float cultmath_lengthsq(float3 value) { return dot(value, value); } +float cultmath_lengthsq(float4 value) { return dot(value, value); } + +float cultmath_distance(float2 left, float2 right) { return length(left - right); } +float cultmath_distance(float3 left, float3 right) { return length(left - right); } +float cultmath_distance(float4 left, float4 right) { return length(left - right); } + +float2 cultmath_normalize(float2 value) { return value / max(length(value), CULTMATH_NORMALIZE_EPSILON); } +float3 cultmath_normalize(float3 value) { return value / max(length(value), CULTMATH_NORMALIZE_EPSILON); } +float4 cultmath_normalize(float4 value) { return value / max(length(value), CULTMATH_NORMALIZE_EPSILON); } + +float2 cultmath_reflect(float2 incident, float2 normal_value) +{ + return incident - 2.0 * dot(normal_value, incident) * normal_value; +} + +float3 cultmath_reflect(float3 incident, float3 normal_value) +{ + return incident - 2.0 * dot(normal_value, incident) * normal_value; +} + +float2 cultmath_rotate(float2 value, float radians_value) +{ + float s = sin(radians_value); + float c = cos(radians_value); + return float2(value.x * c - value.y * s, value.x * s + value.y * c); +} + +float2 cultmath_rotate_degrees(float2 value, float degrees_value) +{ + return cultmath_rotate(value, cultmath_radians(degrees_value)); +} + +float cultmath_csum(float2 value) { return value.x + value.y; } +float cultmath_csum(float3 value) { return value.x + value.y + value.z; } +float cultmath_csum(float4 value) { return value.x + value.y + value.z + value.w; } + +float cultmath_decay(float source, float lambda_value, float dt) { return source * exp(-lambda_value * dt); } +float2 cultmath_decay(float2 source, float lambda_value, float dt) { return source * exp(-lambda_value * dt); } +float3 cultmath_decay(float3 source, float lambda_value, float dt) { return source * exp(-lambda_value * dt); } +float4 cultmath_decay(float4 source, float lambda_value, float dt) { return source * exp(-lambda_value * dt); } + +float cultmath_damp(float start, float end, float lambda_value, float dt) +{ + return cultmath_lerp(start, end, 1.0 - exp(-lambda_value * dt)); +} + +float2 cultmath_damp(float2 start, float2 end, float lambda_value, float dt) +{ + return cultmath_lerp(start, end, 1.0 - exp(-lambda_value * dt)); +} + +float3 cultmath_damp(float3 start, float3 end, float lambda_value, float dt) +{ + return cultmath_lerp(start, end, 1.0 - exp(-lambda_value * dt)); +} + +float4 cultmath_damp(float4 start, float4 end, float lambda_value, float dt) +{ + return cultmath_lerp(start, end, 1.0 - exp(-lambda_value * dt)); +} + +float cultmath_catmullrom(float p0, float p1, float p2, float p3, float t) +{ + float t2 = t * t; + float t3 = t2 * t; + return 0.5 * ((2.0 * p1) + (p2 - p0) * t + (2.0 * p0 - 5.0 * p1 + 4.0 * p2 - p3) * t2 + (-p0 + 3.0 * p1 - 3.0 * p2 + p3) * t3); +} + +float2 cultmath_catmullrom(float2 p0, float2 p1, float2 p2, float2 p3, float t) +{ + float t2 = t * t; + float t3 = t2 * t; + return 0.5 * ((2.0 * p1) + (p2 - p0) * t + (2.0 * p0 - 5.0 * p1 + 4.0 * p2 - p3) * t2 + (-p0 + 3.0 * p1 - 3.0 * p2 + p3) * t3); +} + +float3 cultmath_catmullrom(float3 p0, float3 p1, float3 p2, float3 p3, float t) +{ + float t2 = t * t; + float t3 = t2 * t; + return 0.5 * ((2.0 * p1) + (p2 - p0) * t + (2.0 * p0 - 5.0 * p1 + 4.0 * p2 - p3) * t2 + (-p0 + 3.0 * p1 - 3.0 * p2 + p3) * t3); +} + +float4 cultmath_catmullrom(float4 p0, float4 p1, float4 p2, float4 p3, float t) +{ + float t2 = t * t; + float t3 = t2 * t; + return 0.5 * ((2.0 * p1) + (p2 - p0) * t + (2.0 * p0 - 5.0 * p1 + 4.0 * p2 - p3) * t2 + (-p0 + 3.0 * p1 - 3.0 * p2 + p3) * t3); +} + +float cultmath_quadratic_bezier(float p0, float p1, float p2, float t) +{ + t = cultmath_saturate(t); + float inv = 1.0 - t; + return inv * inv * p0 + 2.0 * inv * t * p1 + t * t * p2; +} + +float3 cultmath_quadratic_bezier(float3 p0, float3 p1, float3 p2, float t) +{ + t = cultmath_saturate(t); + float inv = 1.0 - t; + return inv * inv * p0 + 2.0 * inv * t * p1 + t * t * p2; +} + +float cultmath_cubic_bezier(float p0, float p1, float p2, float p3, float t) +{ + t = cultmath_saturate(t); + float inv = 1.0 - t; + return inv * inv * inv * p0 + 3.0 * inv * inv * t * p1 + 3.0 * inv * t * t * p2 + t * t * t * p3; +} + +float3 cultmath_cubic_bezier(float3 p0, float3 p1, float3 p2, float3 p3, float t) +{ + t = cultmath_saturate(t); + float inv = 1.0 - t; + return inv * inv * inv * p0 + 3.0 * inv * inv * t * p1 + 3.0 * inv * t * t * p2 + t * t * t * p3; +} + +float cultmath_hash(float value) { return cultmath_frac(sin(value) * 43758.5453); } +float cultmath_hash(float2 value) { return cultmath_hash(dot(value, float2(127.1, 311.7))); } +float cultmath_hash(float3 value) { return cultmath_hash(dot(value, float3(127.1, 311.7, 74.7))); } + +float cultmath_value_noise(float2 position) +{ + float2 cell = floor(position); + float2 local = cultmath_frac(position); + float2 u = local * local * (float2(3.0, 3.0) - float2(2.0, 2.0) * local); + + float a = cultmath_hash(cell); + float b = cultmath_hash(cell + float2(1.0, 0.0)); + float c = cultmath_hash(cell + float2(0.0, 1.0)); + float d = cultmath_hash(cell + float2(1.0, 1.0)); + + return cultmath_lerp(cultmath_lerp(a, b, u.x), cultmath_lerp(c, d, u.x), u.y); +} + +float cultmath_value_noise_bicubic(float2 position) +{ + float2 cell = floor(position); + float2 local = cultmath_frac(position); + + float y0 = cultmath_catmullrom(cultmath_hash(cell + float2(-1.0, -1.0)), cultmath_hash(cell + float2(0.0, -1.0)), cultmath_hash(cell + float2(1.0, -1.0)), cultmath_hash(cell + float2(2.0, -1.0)), local.x); + float y1 = cultmath_catmullrom(cultmath_hash(cell + float2(-1.0, 0.0)), cultmath_hash(cell + float2(0.0, 0.0)), cultmath_hash(cell + float2(1.0, 0.0)), cultmath_hash(cell + float2(2.0, 0.0)), local.x); + float y2 = cultmath_catmullrom(cultmath_hash(cell + float2(-1.0, 1.0)), cultmath_hash(cell + float2(0.0, 1.0)), cultmath_hash(cell + float2(1.0, 1.0)), cultmath_hash(cell + float2(2.0, 1.0)), local.x); + float y3 = cultmath_catmullrom(cultmath_hash(cell + float2(-1.0, 2.0)), cultmath_hash(cell + float2(0.0, 2.0)), cultmath_hash(cell + float2(1.0, 2.0)), cultmath_hash(cell + float2(2.0, 2.0)), local.x); + + return cultmath_catmullrom(y0, y1, y2, y3, local.y); +} + +float cultmath_value_noise_texture(Texture2D noise_texture, SamplerState noise_sampler, float2 uv, float scale) +{ + return noise_texture.SampleLevel(noise_sampler, uv * scale, 0.0).r; +} + +float cultmath_value_noise_texture_bicubic(Texture2D noise_texture, SamplerState noise_sampler, float2 uv, float scale, float2 texel_size) +{ + float2 position = uv * scale; + float2 cell = floor(position); + float2 local = cultmath_frac(position); + float2 base_uv = cell / scale; + + float row0 = cultmath_catmullrom( + noise_texture.SampleLevel(noise_sampler, base_uv + texel_size * float2(-1.0, -1.0), 0.0).r, + noise_texture.SampleLevel(noise_sampler, base_uv + texel_size * float2(0.0, -1.0), 0.0).r, + noise_texture.SampleLevel(noise_sampler, base_uv + texel_size * float2(1.0, -1.0), 0.0).r, + noise_texture.SampleLevel(noise_sampler, base_uv + texel_size * float2(2.0, -1.0), 0.0).r, + local.x); + float row1 = cultmath_catmullrom( + noise_texture.SampleLevel(noise_sampler, base_uv + texel_size * float2(-1.0, 0.0), 0.0).r, + noise_texture.SampleLevel(noise_sampler, base_uv + texel_size * float2(0.0, 0.0), 0.0).r, + noise_texture.SampleLevel(noise_sampler, base_uv + texel_size * float2(1.0, 0.0), 0.0).r, + noise_texture.SampleLevel(noise_sampler, base_uv + texel_size * float2(2.0, 0.0), 0.0).r, + local.x); + float row2 = cultmath_catmullrom( + noise_texture.SampleLevel(noise_sampler, base_uv + texel_size * float2(-1.0, 1.0), 0.0).r, + noise_texture.SampleLevel(noise_sampler, base_uv + texel_size * float2(0.0, 1.0), 0.0).r, + noise_texture.SampleLevel(noise_sampler, base_uv + texel_size * float2(1.0, 1.0), 0.0).r, + noise_texture.SampleLevel(noise_sampler, base_uv + texel_size * float2(2.0, 1.0), 0.0).r, + local.x); + float row3 = cultmath_catmullrom( + noise_texture.SampleLevel(noise_sampler, base_uv + texel_size * float2(-1.0, 2.0), 0.0).r, + noise_texture.SampleLevel(noise_sampler, base_uv + texel_size * float2(0.0, 2.0), 0.0).r, + noise_texture.SampleLevel(noise_sampler, base_uv + texel_size * float2(1.0, 2.0), 0.0).r, + noise_texture.SampleLevel(noise_sampler, base_uv + texel_size * float2(2.0, 2.0), 0.0).r, + local.x); + + return cultmath_catmullrom(row0, row1, row2, row3, local.y); +} + +#endif diff --git a/packages/cultmath/shaders/CultMath.hlsl.meta b/packages/cultmath/shaders/CultMath.hlsl.meta new file mode 100644 index 00000000..d6b8542e --- /dev/null +++ b/packages/cultmath/shaders/CultMath.hlsl.meta @@ -0,0 +1,7 @@ +fileFormatVersion: 2 +guid: 33174674795143c4a905408718f956f8 +ShaderIncludeImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/packages/cultmath/src.meta b/packages/cultmath/src.meta new file mode 100644 index 00000000..3e88fcb9 --- /dev/null +++ b/packages/cultmath/src.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: 64f48d0cae0810e4ba9903e9094fe4c6 +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/packages/cultmath/src/CultMath.meta b/packages/cultmath/src/CultMath.meta new file mode 100644 index 00000000..381b726d --- /dev/null +++ b/packages/cultmath/src/CultMath.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: e857b457b93397d4fb4ad8481cb72e4c +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/packages/cultmath/src/CultMath/BatchMath.cs b/packages/cultmath/src/CultMath/BatchMath.cs new file mode 100644 index 00000000..04c042d9 --- /dev/null +++ b/packages/cultmath/src/CultMath/BatchMath.cs @@ -0,0 +1,245 @@ +using System; +using System.Numerics; + +namespace CultMath; + +public static class BatchMath +{ + private const float MinimumDistanceSquared = 1.0e-8f; + + public static int LaneCount => Vector.Count; + + public static bool IsHardwareAccelerated => Vector.IsHardwareAccelerated; + + public static void Clear(Span values) => values.Clear(); + + public static void AddRadialFalloffAcceleration2D( + ReadOnlySpan positionX, + ReadOnlySpan positionY, + float centerX, + float centerY, + float strength, + float radius, + Span accelerationX, + Span accelerationY) + { + ValidateEqualLengths(positionX.Length, positionY.Length, accelerationX.Length, accelerationY.Length); + + if (radius <= 0.0f || strength == 0.0f || positionX.Length == 0) + { + return; + } + + var radiusVector = new Vector(radius); + var centerXVector = new Vector(centerX); + var centerYVector = new Vector(centerY); + var strengthVector = new Vector(strength); + var one = Vector.One; + var zero = Vector.Zero; + var minimumDistanceSquared = new Vector(MinimumDistanceSquared); + + var i = 0; + var lanes = Vector.Count; + var pxBuffer = new float[lanes]; + var pyBuffer = new float[lanes]; + var axBuffer = new float[lanes]; + var ayBuffer = new float[lanes]; + for (; i <= positionX.Length - lanes; i += lanes) + { + CopySpanToArray(positionX.Slice(i, lanes), pxBuffer); + CopySpanToArray(positionY.Slice(i, lanes), pyBuffer); + CopySpanToArray(accelerationX.Slice(i, lanes), axBuffer); + CopySpanToArray(accelerationY.Slice(i, lanes), ayBuffer); + + var px = new Vector(pxBuffer); + var py = new Vector(pyBuffer); + var dx = centerXVector - px; + var dy = centerYVector - py; + var distanceSquared = Vector.Max(dx * dx + dy * dy, minimumDistanceSquared); + var distance = Vector.SquareRoot(distanceSquared); + var inRange = Vector.LessThanOrEqual(distance, radiusVector); + var falloff = Vector.Max(zero, one - distance / radiusVector); + var magnitude = strengthVector * falloff; + var ax = Vector.ConditionalSelect(inRange, dx / distance * magnitude, zero); + var ay = Vector.ConditionalSelect(inRange, dy / distance * magnitude, zero); + + (new Vector(axBuffer) + ax).CopyTo(axBuffer); + (new Vector(ayBuffer) + ay).CopyTo(ayBuffer); + CopyArrayToSpan(axBuffer, accelerationX.Slice(i, lanes)); + CopyArrayToSpan(ayBuffer, accelerationY.Slice(i, lanes)); + } + + for (; i < positionX.Length; i++) + { + var dx = centerX - positionX[i]; + var dy = centerY - positionY[i]; + var distanceSquared = MathF.Max(dx * dx + dy * dy, MinimumDistanceSquared); + var distance = MathF.Sqrt(distanceSquared); + if (distance > radius) + { + continue; + } + + var falloff = MathF.Max(0.0f, 1.0f - distance / radius); + var magnitude = strength * falloff; + accelerationX[i] += dx / distance * magnitude; + accelerationY[i] += dy / distance * magnitude; + } + } + + public static void AddRadialFalloffAcceleration2D( + ReadOnlySpan positions, + float2 center, + float strength, + float radius, + Span acceleration) + { + ValidateEqualLengths(positions.Length, acceleration.Length); + + if (radius <= 0.0f || strength == 0.0f || positions.Length == 0) + { + return; + } + + for (var i = 0; i < positions.Length; i++) + { + var delta = center - positions[i]; + var distanceSquared = MathF.Max(delta.x * delta.x + delta.y * delta.y, MinimumDistanceSquared); + var distance = MathF.Sqrt(distanceSquared); + if (distance > radius) + { + continue; + } + + var falloff = MathF.Max(0.0f, 1.0f - distance / radius); + acceleration[i] += delta / distance * (strength * falloff); + } + } + + public static void IntegrateSemiImplicitEuler2D( + float deltaTime, + ReadOnlySpan dynamicMask, + Span positionX, + Span positionY, + Span velocityX, + Span velocityY, + ReadOnlySpan accelerationX, + ReadOnlySpan accelerationY) + { + if (deltaTime <= 0.0f) + { + throw new ArgumentOutOfRangeException(nameof(deltaTime), "Delta time must be positive."); + } + + ValidateEqualLengths( + dynamicMask.Length, + positionX.Length, + positionY.Length, + velocityX.Length, + velocityY.Length, + accelerationX.Length, + accelerationY.Length); + + var delta = new Vector(deltaTime); + var i = 0; + var lanes = Vector.Count; + var maskBuffer = new float[lanes]; + var pxBuffer = new float[lanes]; + var pyBuffer = new float[lanes]; + var vxBuffer = new float[lanes]; + var vyBuffer = new float[lanes]; + var axBuffer = new float[lanes]; + var ayBuffer = new float[lanes]; + for (; i <= positionX.Length - lanes; i += lanes) + { + CopySpanToArray(dynamicMask.Slice(i, lanes), maskBuffer); + CopySpanToArray(velocityX.Slice(i, lanes), vxBuffer); + CopySpanToArray(velocityY.Slice(i, lanes), vyBuffer); + CopySpanToArray(positionX.Slice(i, lanes), pxBuffer); + CopySpanToArray(positionY.Slice(i, lanes), pyBuffer); + CopySpanToArray(accelerationX.Slice(i, lanes), axBuffer); + CopySpanToArray(accelerationY.Slice(i, lanes), ayBuffer); + + var mask = new Vector(maskBuffer); + var vx = new Vector(vxBuffer); + var vy = new Vector(vyBuffer); + var px = new Vector(pxBuffer); + var py = new Vector(pyBuffer); + var ax = new Vector(axBuffer); + var ay = new Vector(ayBuffer); + + vx += ax * delta * mask; + vy += ay * delta * mask; + px += vx * delta * mask; + py += vy * delta * mask; + + vx.CopyTo(vxBuffer); + vy.CopyTo(vyBuffer); + px.CopyTo(pxBuffer); + py.CopyTo(pyBuffer); + CopyArrayToSpan(vxBuffer, velocityX.Slice(i, lanes)); + CopyArrayToSpan(vyBuffer, velocityY.Slice(i, lanes)); + CopyArrayToSpan(pxBuffer, positionX.Slice(i, lanes)); + CopyArrayToSpan(pyBuffer, positionY.Slice(i, lanes)); + } + + for (; i < positionX.Length; i++) + { + var mask = dynamicMask[i]; + velocityX[i] += accelerationX[i] * deltaTime * mask; + velocityY[i] += accelerationY[i] * deltaTime * mask; + positionX[i] += velocityX[i] * deltaTime * mask; + positionY[i] += velocityY[i] * deltaTime * mask; + } + } + + public static void IntegrateSemiImplicitEuler2D( + float deltaTime, + ReadOnlySpan dynamicMask, + Span position, + Span velocity, + ReadOnlySpan acceleration) + { + if (deltaTime <= 0.0f) + { + throw new ArgumentOutOfRangeException(nameof(deltaTime), "Delta time must be positive."); + } + + ValidateEqualLengths(dynamicMask.Length, position.Length, velocity.Length, acceleration.Length); + + for (var i = 0; i < position.Length; i++) + { + var mask = dynamicMask[i]; + velocity[i] += acceleration[i] * deltaTime * mask; + position[i] += velocity[i] * deltaTime * mask; + } + } + + private static void ValidateEqualLengths(params int[] lengths) + { + var length = lengths[0]; + for (var i = 1; i < lengths.Length; i++) + { + if (lengths[i] != length) + { + throw new ArgumentException("CultMath batch spans must have equal length."); + } + } + } + + private static void CopySpanToArray(ReadOnlySpan source, float[] destination) + { + for (var i = 0; i < source.Length; i++) + { + destination[i] = source[i]; + } + } + + private static void CopyArrayToSpan(float[] source, Span destination) + { + for (var i = 0; i < destination.Length; i++) + { + destination[i] = source[i]; + } + } +} diff --git a/packages/cultmath/src/CultMath/BatchMath.cs.meta b/packages/cultmath/src/CultMath/BatchMath.cs.meta new file mode 100644 index 00000000..fe7c6033 --- /dev/null +++ b/packages/cultmath/src/CultMath/BatchMath.cs.meta @@ -0,0 +1,2 @@ +fileFormatVersion: 2 +guid: d4771320e190bdd408dd296c293a9d0e \ No newline at end of file diff --git a/packages/cultmath/src/CultMath/Color32.cs b/packages/cultmath/src/CultMath/Color32.cs new file mode 100644 index 00000000..e9a2b481 --- /dev/null +++ b/packages/cultmath/src/CultMath/Color32.cs @@ -0,0 +1,4 @@ +namespace CultMath; + +[System.Runtime.InteropServices.StructLayout(System.Runtime.InteropServices.LayoutKind.Sequential)] +public readonly record struct Color32(byte r, byte g, byte b, byte a = 255); diff --git a/packages/cultmath/src/CultMath/Color32.cs.meta b/packages/cultmath/src/CultMath/Color32.cs.meta new file mode 100644 index 00000000..71e1ba55 --- /dev/null +++ b/packages/cultmath/src/CultMath/Color32.cs.meta @@ -0,0 +1,2 @@ +fileFormatVersion: 2 +guid: bfa14a9aacb8ac4448761568643f2bc6 \ No newline at end of file diff --git a/packages/cultmath/src/CultMath/CultMath.asmdef b/packages/cultmath/src/CultMath/CultMath.asmdef new file mode 100644 index 00000000..449549f6 --- /dev/null +++ b/packages/cultmath/src/CultMath/CultMath.asmdef @@ -0,0 +1,14 @@ +{ + "name": "CultMath", + "rootNamespace": "CultMath", + "references": [], + "includePlatforms": [], + "excludePlatforms": [], + "allowUnsafeCode": true, + "overrideReferences": false, + "precompiledReferences": [], + "autoReferenced": true, + "defineConstraints": [], + "versionDefines": [], + "noEngineReferences": true +} diff --git a/packages/cultmath/src/CultMath/CultMath.asmdef.meta b/packages/cultmath/src/CultMath/CultMath.asmdef.meta new file mode 100644 index 00000000..5a26d2c5 --- /dev/null +++ b/packages/cultmath/src/CultMath/CultMath.asmdef.meta @@ -0,0 +1,7 @@ +fileFormatVersion: 2 +guid: f0b66b909d13e18468f7aa56221531e9 +AssemblyDefinitionImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/packages/cultmath/src/CultMath/CultMath.csproj b/packages/cultmath/src/CultMath/CultMath.csproj new file mode 100644 index 00000000..7c567105 --- /dev/null +++ b/packages/cultmath/src/CultMath/CultMath.csproj @@ -0,0 +1,26 @@ + + + + net8.0;netstandard2.1 + enable + enable + latest + CultMath + CultMath + GameCult.Math + 0.2.0 + Clean-room C# math shaped for HLSL-style renderer code. + GameCult + README.md + $(NoWarn);CS8981 + true + + false + + + + + + + + diff --git a/packages/cultmath/src/CultMath/CultMath.csproj.meta b/packages/cultmath/src/CultMath/CultMath.csproj.meta new file mode 100644 index 00000000..78fc304d --- /dev/null +++ b/packages/cultmath/src/CultMath/CultMath.csproj.meta @@ -0,0 +1,7 @@ +fileFormatVersion: 2 +guid: 77e1a0a2512910f41885650a7d0d6f60 +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/packages/cultmath/src/CultMath/CultMathTone.cs b/packages/cultmath/src/CultMath/CultMathTone.cs new file mode 100644 index 00000000..25dfffeb --- /dev/null +++ b/packages/cultmath/src/CultMath/CultMathTone.cs @@ -0,0 +1,10 @@ +namespace CultMath; + +public enum CultMathTone : byte +{ + Background = 0, + Panel = 1, + Edge = 2, + Header = 3, + Body = 4, +} diff --git a/packages/cultmath/src/CultMath/CultMathTone.cs.meta b/packages/cultmath/src/CultMath/CultMathTone.cs.meta new file mode 100644 index 00000000..f41d1475 --- /dev/null +++ b/packages/cultmath/src/CultMath/CultMathTone.cs.meta @@ -0,0 +1,2 @@ +fileFormatVersion: 2 +guid: d4bfc66f9f7e992448f884c2c338d50a \ No newline at end of file diff --git a/src/GameCult.Geometry/IsExternalInit.cs b/packages/cultmath/src/CultMath/IsExternalInit.cs similarity index 76% rename from src/GameCult.Geometry/IsExternalInit.cs rename to packages/cultmath/src/CultMath/IsExternalInit.cs index b33e9e1d..b05eb4a1 100644 --- a/src/GameCult.Geometry/IsExternalInit.cs +++ b/packages/cultmath/src/CultMath/IsExternalInit.cs @@ -1,7 +1,5 @@ -#if NETSTANDARD2_1 namespace System.Runtime.CompilerServices; internal static class IsExternalInit { } -#endif diff --git a/packages/cultmath/src/CultMath/IsExternalInit.cs.meta b/packages/cultmath/src/CultMath/IsExternalInit.cs.meta new file mode 100644 index 00000000..ca761bcb --- /dev/null +++ b/packages/cultmath/src/CultMath/IsExternalInit.cs.meta @@ -0,0 +1,2 @@ +fileFormatVersion: 2 +guid: de2b3c3e7f03c354aa2b84fbbc55b497 \ No newline at end of file diff --git a/packages/cultmath/src/CultMath/Random.cs b/packages/cultmath/src/CultMath/Random.cs new file mode 100644 index 00000000..6892e20c --- /dev/null +++ b/packages/cultmath/src/CultMath/Random.cs @@ -0,0 +1,66 @@ +namespace CultMath; + +public struct Random +{ + private const uint DefaultSeed = 0x6E624EB7u; + + public uint state; + + public Random(uint seed) + { + state = seed == 0 ? DefaultSeed : seed; + } + + public static Random CreateFromIndex(uint index) => new(index + DefaultSeed); + + public void InitState(uint seed) + { + state = seed == 0 ? DefaultSeed : seed; + } + + public uint NextUInt() + { + var x = state == 0 ? DefaultSeed : state; + x ^= x << 13; + x ^= x >> 17; + x ^= x << 5; + state = x == 0 ? DefaultSeed : x; + return state; + } + + public int NextInt(int max) => NextInt(0, max); + + public int NextInt(int min, int max) + { + if (max <= min) + { + return min; + } + + return min + (int)(NextUInt() % (uint)(max - min)); + } + + public float NextFloat() => (NextUInt() >> 8) * (1.0f / 16777216.0f); + + public float NextFloat(float max) => NextFloat() * max; + + public float NextFloat(float min, float max) => min + NextFloat() * (max - min); + + public float2 NextFloat2() => new(NextFloat(), NextFloat()); + + public float2 NextFloat2(float min, float max) => new(NextFloat(min, max), NextFloat(min, max)); + + public float2 NextFloat2(float2 min, float2 max) => min + NextFloat2() * (max - min); + + public float2 NextFloat2Direction() + { + var angle = NextFloat(0.0f, math.TAU); + return new float2(math.cos(angle), math.sin(angle)); + } + + public float3 NextFloat3() => new(NextFloat(), NextFloat(), NextFloat()); + + public float3 NextFloat3(float3 max) => NextFloat3() * max; + + public float3 NextFloat3(float3 min, float3 max) => min + NextFloat3() * (max - min); +} diff --git a/packages/cultmath/src/CultMath/Random.cs.meta b/packages/cultmath/src/CultMath/Random.cs.meta new file mode 100644 index 00000000..55032ded --- /dev/null +++ b/packages/cultmath/src/CultMath/Random.cs.meta @@ -0,0 +1,2 @@ +fileFormatVersion: 2 +guid: 7c1a557259026da4ca20b54264852c13 \ No newline at end of file diff --git a/packages/cultmath/src/CultMath/Swizzles.g.cs b/packages/cultmath/src/CultMath/Swizzles.g.cs new file mode 100644 index 00000000..ce84bbc4 --- /dev/null +++ b/packages/cultmath/src/CultMath/Swizzles.g.cs @@ -0,0 +1,3137 @@ +// +// Generated by tools/generate-swizzles.ps1. Do not edit by hand. +// + +namespace CultMath; + +public partial struct float2 +{ + public float2(float2 xy) : this(xy.x, xy.y) { } + public float r { readonly get => x; set => x = value; } + public float g { readonly get => y; set => y = value; } + public readonly float2 xx => new(x, x); + public float2 xy { readonly get => new(x, y); set { x = value.x; y = value.y; } } + public float2 yx { readonly get => new(y, x); set { y = value.x; x = value.y; } } + public readonly float2 yy => new(y, y); + public readonly float3 xxx => new(x, x, x); + public readonly float3 xxy => new(x, x, y); + public readonly float3 xyx => new(x, y, x); + public readonly float3 xyy => new(x, y, y); + public readonly float3 yxx => new(y, x, x); + public readonly float3 yxy => new(y, x, y); + public readonly float3 yyx => new(y, y, x); + public readonly float3 yyy => new(y, y, y); + public readonly float4 xxxx => new(x, x, x, x); + public readonly float4 xxxy => new(x, x, x, y); + public readonly float4 xxyx => new(x, x, y, x); + public readonly float4 xxyy => new(x, x, y, y); + public readonly float4 xyxx => new(x, y, x, x); + public readonly float4 xyxy => new(x, y, x, y); + public readonly float4 xyyx => new(x, y, y, x); + public readonly float4 xyyy => new(x, y, y, y); + public readonly float4 yxxx => new(y, x, x, x); + public readonly float4 yxxy => new(y, x, x, y); + public readonly float4 yxyx => new(y, x, y, x); + public readonly float4 yxyy => new(y, x, y, y); + public readonly float4 yyxx => new(y, y, x, x); + public readonly float4 yyxy => new(y, y, x, y); + public readonly float4 yyyx => new(y, y, y, x); + public readonly float4 yyyy => new(y, y, y, y); + public readonly float2 rr => new(x, x); + public float2 rg { readonly get => new(x, y); set { x = value.x; y = value.y; } } + public float2 gr { readonly get => new(y, x); set { y = value.x; x = value.y; } } + public readonly float2 gg => new(y, y); + public readonly float3 rrr => new(x, x, x); + public readonly float3 rrg => new(x, x, y); + public readonly float3 rgr => new(x, y, x); + public readonly float3 rgg => new(x, y, y); + public readonly float3 grr => new(y, x, x); + public readonly float3 grg => new(y, x, y); + public readonly float3 ggr => new(y, y, x); + public readonly float3 ggg => new(y, y, y); + public readonly float4 rrrr => new(x, x, x, x); + public readonly float4 rrrg => new(x, x, x, y); + public readonly float4 rrgr => new(x, x, y, x); + public readonly float4 rrgg => new(x, x, y, y); + public readonly float4 rgrr => new(x, y, x, x); + public readonly float4 rgrg => new(x, y, x, y); + public readonly float4 rggr => new(x, y, y, x); + public readonly float4 rggg => new(x, y, y, y); + public readonly float4 grrr => new(y, x, x, x); + public readonly float4 grrg => new(y, x, x, y); + public readonly float4 grgr => new(y, x, y, x); + public readonly float4 grgg => new(y, x, y, y); + public readonly float4 ggrr => new(y, y, x, x); + public readonly float4 ggrg => new(y, y, x, y); + public readonly float4 gggr => new(y, y, y, x); + public readonly float4 gggg => new(y, y, y, y); +} + +public partial struct float3 +{ + public float3(float x, float2 yz) : this(x, yz.x, yz.y) { } + public float3(float2 xy, float z) : this(xy.x, xy.y, z) { } + public float3(float3 xyz) : this(xyz.x, xyz.y, xyz.z) { } + public float r { readonly get => x; set => x = value; } + public float g { readonly get => y; set => y = value; } + public float b { readonly get => z; set => z = value; } + public readonly float2 xx => new(x, x); + public float2 xy { readonly get => new(x, y); set { x = value.x; y = value.y; } } + public float2 xz { readonly get => new(x, z); set { x = value.x; z = value.y; } } + public float2 yx { readonly get => new(y, x); set { y = value.x; x = value.y; } } + public readonly float2 yy => new(y, y); + public float2 yz { readonly get => new(y, z); set { y = value.x; z = value.y; } } + public float2 zx { readonly get => new(z, x); set { z = value.x; x = value.y; } } + public float2 zy { readonly get => new(z, y); set { z = value.x; y = value.y; } } + public readonly float2 zz => new(z, z); + public readonly float3 xxx => new(x, x, x); + public readonly float3 xxy => new(x, x, y); + public readonly float3 xxz => new(x, x, z); + public readonly float3 xyx => new(x, y, x); + public readonly float3 xyy => new(x, y, y); + public float3 xyz { readonly get => new(x, y, z); set { x = value.x; y = value.y; z = value.z; } } + public readonly float3 xzx => new(x, z, x); + public float3 xzy { readonly get => new(x, z, y); set { x = value.x; z = value.y; y = value.z; } } + public readonly float3 xzz => new(x, z, z); + public readonly float3 yxx => new(y, x, x); + public readonly float3 yxy => new(y, x, y); + public float3 yxz { readonly get => new(y, x, z); set { y = value.x; x = value.y; z = value.z; } } + public readonly float3 yyx => new(y, y, x); + public readonly float3 yyy => new(y, y, y); + public readonly float3 yyz => new(y, y, z); + public float3 yzx { readonly get => new(y, z, x); set { y = value.x; z = value.y; x = value.z; } } + public readonly float3 yzy => new(y, z, y); + public readonly float3 yzz => new(y, z, z); + public readonly float3 zxx => new(z, x, x); + public float3 zxy { readonly get => new(z, x, y); set { z = value.x; x = value.y; y = value.z; } } + public readonly float3 zxz => new(z, x, z); + public float3 zyx { readonly get => new(z, y, x); set { z = value.x; y = value.y; x = value.z; } } + public readonly float3 zyy => new(z, y, y); + public readonly float3 zyz => new(z, y, z); + public readonly float3 zzx => new(z, z, x); + public readonly float3 zzy => new(z, z, y); + public readonly float3 zzz => new(z, z, z); + public readonly float4 xxxx => new(x, x, x, x); + public readonly float4 xxxy => new(x, x, x, y); + public readonly float4 xxxz => new(x, x, x, z); + public readonly float4 xxyx => new(x, x, y, x); + public readonly float4 xxyy => new(x, x, y, y); + public readonly float4 xxyz => new(x, x, y, z); + public readonly float4 xxzx => new(x, x, z, x); + public readonly float4 xxzy => new(x, x, z, y); + public readonly float4 xxzz => new(x, x, z, z); + public readonly float4 xyxx => new(x, y, x, x); + public readonly float4 xyxy => new(x, y, x, y); + public readonly float4 xyxz => new(x, y, x, z); + public readonly float4 xyyx => new(x, y, y, x); + public readonly float4 xyyy => new(x, y, y, y); + public readonly float4 xyyz => new(x, y, y, z); + public readonly float4 xyzx => new(x, y, z, x); + public readonly float4 xyzy => new(x, y, z, y); + public readonly float4 xyzz => new(x, y, z, z); + public readonly float4 xzxx => new(x, z, x, x); + public readonly float4 xzxy => new(x, z, x, y); + public readonly float4 xzxz => new(x, z, x, z); + public readonly float4 xzyx => new(x, z, y, x); + public readonly float4 xzyy => new(x, z, y, y); + public readonly float4 xzyz => new(x, z, y, z); + public readonly float4 xzzx => new(x, z, z, x); + public readonly float4 xzzy => new(x, z, z, y); + public readonly float4 xzzz => new(x, z, z, z); + public readonly float4 yxxx => new(y, x, x, x); + public readonly float4 yxxy => new(y, x, x, y); + public readonly float4 yxxz => new(y, x, x, z); + public readonly float4 yxyx => new(y, x, y, x); + public readonly float4 yxyy => new(y, x, y, y); + public readonly float4 yxyz => new(y, x, y, z); + public readonly float4 yxzx => new(y, x, z, x); + public readonly float4 yxzy => new(y, x, z, y); + public readonly float4 yxzz => new(y, x, z, z); + public readonly float4 yyxx => new(y, y, x, x); + public readonly float4 yyxy => new(y, y, x, y); + public readonly float4 yyxz => new(y, y, x, z); + public readonly float4 yyyx => new(y, y, y, x); + public readonly float4 yyyy => new(y, y, y, y); + public readonly float4 yyyz => new(y, y, y, z); + public readonly float4 yyzx => new(y, y, z, x); + public readonly float4 yyzy => new(y, y, z, y); + public readonly float4 yyzz => new(y, y, z, z); + public readonly float4 yzxx => new(y, z, x, x); + public readonly float4 yzxy => new(y, z, x, y); + public readonly float4 yzxz => new(y, z, x, z); + public readonly float4 yzyx => new(y, z, y, x); + public readonly float4 yzyy => new(y, z, y, y); + public readonly float4 yzyz => new(y, z, y, z); + public readonly float4 yzzx => new(y, z, z, x); + public readonly float4 yzzy => new(y, z, z, y); + public readonly float4 yzzz => new(y, z, z, z); + public readonly float4 zxxx => new(z, x, x, x); + public readonly float4 zxxy => new(z, x, x, y); + public readonly float4 zxxz => new(z, x, x, z); + public readonly float4 zxyx => new(z, x, y, x); + public readonly float4 zxyy => new(z, x, y, y); + public readonly float4 zxyz => new(z, x, y, z); + public readonly float4 zxzx => new(z, x, z, x); + public readonly float4 zxzy => new(z, x, z, y); + public readonly float4 zxzz => new(z, x, z, z); + public readonly float4 zyxx => new(z, y, x, x); + public readonly float4 zyxy => new(z, y, x, y); + public readonly float4 zyxz => new(z, y, x, z); + public readonly float4 zyyx => new(z, y, y, x); + public readonly float4 zyyy => new(z, y, y, y); + public readonly float4 zyyz => new(z, y, y, z); + public readonly float4 zyzx => new(z, y, z, x); + public readonly float4 zyzy => new(z, y, z, y); + public readonly float4 zyzz => new(z, y, z, z); + public readonly float4 zzxx => new(z, z, x, x); + public readonly float4 zzxy => new(z, z, x, y); + public readonly float4 zzxz => new(z, z, x, z); + public readonly float4 zzyx => new(z, z, y, x); + public readonly float4 zzyy => new(z, z, y, y); + public readonly float4 zzyz => new(z, z, y, z); + public readonly float4 zzzx => new(z, z, z, x); + public readonly float4 zzzy => new(z, z, z, y); + public readonly float4 zzzz => new(z, z, z, z); + public readonly float2 rr => new(x, x); + public float2 rg { readonly get => new(x, y); set { x = value.x; y = value.y; } } + public float2 rb { readonly get => new(x, z); set { x = value.x; z = value.y; } } + public float2 gr { readonly get => new(y, x); set { y = value.x; x = value.y; } } + public readonly float2 gg => new(y, y); + public float2 gb { readonly get => new(y, z); set { y = value.x; z = value.y; } } + public float2 br { readonly get => new(z, x); set { z = value.x; x = value.y; } } + public float2 bg { readonly get => new(z, y); set { z = value.x; y = value.y; } } + public readonly float2 bb => new(z, z); + public readonly float3 rrr => new(x, x, x); + public readonly float3 rrg => new(x, x, y); + public readonly float3 rrb => new(x, x, z); + public readonly float3 rgr => new(x, y, x); + public readonly float3 rgg => new(x, y, y); + public float3 rgb { readonly get => new(x, y, z); set { x = value.x; y = value.y; z = value.z; } } + public readonly float3 rbr => new(x, z, x); + public float3 rbg { readonly get => new(x, z, y); set { x = value.x; z = value.y; y = value.z; } } + public readonly float3 rbb => new(x, z, z); + public readonly float3 grr => new(y, x, x); + public readonly float3 grg => new(y, x, y); + public float3 grb { readonly get => new(y, x, z); set { y = value.x; x = value.y; z = value.z; } } + public readonly float3 ggr => new(y, y, x); + public readonly float3 ggg => new(y, y, y); + public readonly float3 ggb => new(y, y, z); + public float3 gbr { readonly get => new(y, z, x); set { y = value.x; z = value.y; x = value.z; } } + public readonly float3 gbg => new(y, z, y); + public readonly float3 gbb => new(y, z, z); + public readonly float3 brr => new(z, x, x); + public float3 brg { readonly get => new(z, x, y); set { z = value.x; x = value.y; y = value.z; } } + public readonly float3 brb => new(z, x, z); + public float3 bgr { readonly get => new(z, y, x); set { z = value.x; y = value.y; x = value.z; } } + public readonly float3 bgg => new(z, y, y); + public readonly float3 bgb => new(z, y, z); + public readonly float3 bbr => new(z, z, x); + public readonly float3 bbg => new(z, z, y); + public readonly float3 bbb => new(z, z, z); + public readonly float4 rrrr => new(x, x, x, x); + public readonly float4 rrrg => new(x, x, x, y); + public readonly float4 rrrb => new(x, x, x, z); + public readonly float4 rrgr => new(x, x, y, x); + public readonly float4 rrgg => new(x, x, y, y); + public readonly float4 rrgb => new(x, x, y, z); + public readonly float4 rrbr => new(x, x, z, x); + public readonly float4 rrbg => new(x, x, z, y); + public readonly float4 rrbb => new(x, x, z, z); + public readonly float4 rgrr => new(x, y, x, x); + public readonly float4 rgrg => new(x, y, x, y); + public readonly float4 rgrb => new(x, y, x, z); + public readonly float4 rggr => new(x, y, y, x); + public readonly float4 rggg => new(x, y, y, y); + public readonly float4 rggb => new(x, y, y, z); + public readonly float4 rgbr => new(x, y, z, x); + public readonly float4 rgbg => new(x, y, z, y); + public readonly float4 rgbb => new(x, y, z, z); + public readonly float4 rbrr => new(x, z, x, x); + public readonly float4 rbrg => new(x, z, x, y); + public readonly float4 rbrb => new(x, z, x, z); + public readonly float4 rbgr => new(x, z, y, x); + public readonly float4 rbgg => new(x, z, y, y); + public readonly float4 rbgb => new(x, z, y, z); + public readonly float4 rbbr => new(x, z, z, x); + public readonly float4 rbbg => new(x, z, z, y); + public readonly float4 rbbb => new(x, z, z, z); + public readonly float4 grrr => new(y, x, x, x); + public readonly float4 grrg => new(y, x, x, y); + public readonly float4 grrb => new(y, x, x, z); + public readonly float4 grgr => new(y, x, y, x); + public readonly float4 grgg => new(y, x, y, y); + public readonly float4 grgb => new(y, x, y, z); + public readonly float4 grbr => new(y, x, z, x); + public readonly float4 grbg => new(y, x, z, y); + public readonly float4 grbb => new(y, x, z, z); + public readonly float4 ggrr => new(y, y, x, x); + public readonly float4 ggrg => new(y, y, x, y); + public readonly float4 ggrb => new(y, y, x, z); + public readonly float4 gggr => new(y, y, y, x); + public readonly float4 gggg => new(y, y, y, y); + public readonly float4 gggb => new(y, y, y, z); + public readonly float4 ggbr => new(y, y, z, x); + public readonly float4 ggbg => new(y, y, z, y); + public readonly float4 ggbb => new(y, y, z, z); + public readonly float4 gbrr => new(y, z, x, x); + public readonly float4 gbrg => new(y, z, x, y); + public readonly float4 gbrb => new(y, z, x, z); + public readonly float4 gbgr => new(y, z, y, x); + public readonly float4 gbgg => new(y, z, y, y); + public readonly float4 gbgb => new(y, z, y, z); + public readonly float4 gbbr => new(y, z, z, x); + public readonly float4 gbbg => new(y, z, z, y); + public readonly float4 gbbb => new(y, z, z, z); + public readonly float4 brrr => new(z, x, x, x); + public readonly float4 brrg => new(z, x, x, y); + public readonly float4 brrb => new(z, x, x, z); + public readonly float4 brgr => new(z, x, y, x); + public readonly float4 brgg => new(z, x, y, y); + public readonly float4 brgb => new(z, x, y, z); + public readonly float4 brbr => new(z, x, z, x); + public readonly float4 brbg => new(z, x, z, y); + public readonly float4 brbb => new(z, x, z, z); + public readonly float4 bgrr => new(z, y, x, x); + public readonly float4 bgrg => new(z, y, x, y); + public readonly float4 bgrb => new(z, y, x, z); + public readonly float4 bggr => new(z, y, y, x); + public readonly float4 bggg => new(z, y, y, y); + public readonly float4 bggb => new(z, y, y, z); + public readonly float4 bgbr => new(z, y, z, x); + public readonly float4 bgbg => new(z, y, z, y); + public readonly float4 bgbb => new(z, y, z, z); + public readonly float4 bbrr => new(z, z, x, x); + public readonly float4 bbrg => new(z, z, x, y); + public readonly float4 bbrb => new(z, z, x, z); + public readonly float4 bbgr => new(z, z, y, x); + public readonly float4 bbgg => new(z, z, y, y); + public readonly float4 bbgb => new(z, z, y, z); + public readonly float4 bbbr => new(z, z, z, x); + public readonly float4 bbbg => new(z, z, z, y); + public readonly float4 bbbb => new(z, z, z, z); +} + +public partial struct float4 +{ + public float4(float x, float y, float2 zw) : this(x, y, zw.x, zw.y) { } + public float4(float x, float2 yz, float w) : this(x, yz.x, yz.y, w) { } + public float4(float x, float3 yzw) : this(x, yzw.x, yzw.y, yzw.z) { } + public float4(float2 xy, float z, float w) : this(xy.x, xy.y, z, w) { } + public float4(float2 xy, float2 zw) : this(xy.x, xy.y, zw.x, zw.y) { } + public float4(float3 xyz, float w) : this(xyz.x, xyz.y, xyz.z, w) { } + public float4(float4 xyzw) : this(xyzw.x, xyzw.y, xyzw.z, xyzw.w) { } + public float r { readonly get => x; set => x = value; } + public float g { readonly get => y; set => y = value; } + public float b { readonly get => z; set => z = value; } + public float a { readonly get => w; set => w = value; } + public readonly float2 xx => new(x, x); + public float2 xy { readonly get => new(x, y); set { x = value.x; y = value.y; } } + public float2 xz { readonly get => new(x, z); set { x = value.x; z = value.y; } } + public float2 xw { readonly get => new(x, w); set { x = value.x; w = value.y; } } + public float2 yx { readonly get => new(y, x); set { y = value.x; x = value.y; } } + public readonly float2 yy => new(y, y); + public float2 yz { readonly get => new(y, z); set { y = value.x; z = value.y; } } + public float2 yw { readonly get => new(y, w); set { y = value.x; w = value.y; } } + public float2 zx { readonly get => new(z, x); set { z = value.x; x = value.y; } } + public float2 zy { readonly get => new(z, y); set { z = value.x; y = value.y; } } + public readonly float2 zz => new(z, z); + public float2 zw { readonly get => new(z, w); set { z = value.x; w = value.y; } } + public float2 wx { readonly get => new(w, x); set { w = value.x; x = value.y; } } + public float2 wy { readonly get => new(w, y); set { w = value.x; y = value.y; } } + public float2 wz { readonly get => new(w, z); set { w = value.x; z = value.y; } } + public readonly float2 ww => new(w, w); + public readonly float3 xxx => new(x, x, x); + public readonly float3 xxy => new(x, x, y); + public readonly float3 xxz => new(x, x, z); + public readonly float3 xxw => new(x, x, w); + public readonly float3 xyx => new(x, y, x); + public readonly float3 xyy => new(x, y, y); + public float3 xyz { readonly get => new(x, y, z); set { x = value.x; y = value.y; z = value.z; } } + public float3 xyw { readonly get => new(x, y, w); set { x = value.x; y = value.y; w = value.z; } } + public readonly float3 xzx => new(x, z, x); + public float3 xzy { readonly get => new(x, z, y); set { x = value.x; z = value.y; y = value.z; } } + public readonly float3 xzz => new(x, z, z); + public float3 xzw { readonly get => new(x, z, w); set { x = value.x; z = value.y; w = value.z; } } + public readonly float3 xwx => new(x, w, x); + public float3 xwy { readonly get => new(x, w, y); set { x = value.x; w = value.y; y = value.z; } } + public float3 xwz { readonly get => new(x, w, z); set { x = value.x; w = value.y; z = value.z; } } + public readonly float3 xww => new(x, w, w); + public readonly float3 yxx => new(y, x, x); + public readonly float3 yxy => new(y, x, y); + public float3 yxz { readonly get => new(y, x, z); set { y = value.x; x = value.y; z = value.z; } } + public float3 yxw { readonly get => new(y, x, w); set { y = value.x; x = value.y; w = value.z; } } + public readonly float3 yyx => new(y, y, x); + public readonly float3 yyy => new(y, y, y); + public readonly float3 yyz => new(y, y, z); + public readonly float3 yyw => new(y, y, w); + public float3 yzx { readonly get => new(y, z, x); set { y = value.x; z = value.y; x = value.z; } } + public readonly float3 yzy => new(y, z, y); + public readonly float3 yzz => new(y, z, z); + public float3 yzw { readonly get => new(y, z, w); set { y = value.x; z = value.y; w = value.z; } } + public float3 ywx { readonly get => new(y, w, x); set { y = value.x; w = value.y; x = value.z; } } + public readonly float3 ywy => new(y, w, y); + public float3 ywz { readonly get => new(y, w, z); set { y = value.x; w = value.y; z = value.z; } } + public readonly float3 yww => new(y, w, w); + public readonly float3 zxx => new(z, x, x); + public float3 zxy { readonly get => new(z, x, y); set { z = value.x; x = value.y; y = value.z; } } + public readonly float3 zxz => new(z, x, z); + public float3 zxw { readonly get => new(z, x, w); set { z = value.x; x = value.y; w = value.z; } } + public float3 zyx { readonly get => new(z, y, x); set { z = value.x; y = value.y; x = value.z; } } + public readonly float3 zyy => new(z, y, y); + public readonly float3 zyz => new(z, y, z); + public float3 zyw { readonly get => new(z, y, w); set { z = value.x; y = value.y; w = value.z; } } + public readonly float3 zzx => new(z, z, x); + public readonly float3 zzy => new(z, z, y); + public readonly float3 zzz => new(z, z, z); + public readonly float3 zzw => new(z, z, w); + public float3 zwx { readonly get => new(z, w, x); set { z = value.x; w = value.y; x = value.z; } } + public float3 zwy { readonly get => new(z, w, y); set { z = value.x; w = value.y; y = value.z; } } + public readonly float3 zwz => new(z, w, z); + public readonly float3 zww => new(z, w, w); + public readonly float3 wxx => new(w, x, x); + public float3 wxy { readonly get => new(w, x, y); set { w = value.x; x = value.y; y = value.z; } } + public float3 wxz { readonly get => new(w, x, z); set { w = value.x; x = value.y; z = value.z; } } + public readonly float3 wxw => new(w, x, w); + public float3 wyx { readonly get => new(w, y, x); set { w = value.x; y = value.y; x = value.z; } } + public readonly float3 wyy => new(w, y, y); + public float3 wyz { readonly get => new(w, y, z); set { w = value.x; y = value.y; z = value.z; } } + public readonly float3 wyw => new(w, y, w); + public float3 wzx { readonly get => new(w, z, x); set { w = value.x; z = value.y; x = value.z; } } + public float3 wzy { readonly get => new(w, z, y); set { w = value.x; z = value.y; y = value.z; } } + public readonly float3 wzz => new(w, z, z); + public readonly float3 wzw => new(w, z, w); + public readonly float3 wwx => new(w, w, x); + public readonly float3 wwy => new(w, w, y); + public readonly float3 wwz => new(w, w, z); + public readonly float3 www => new(w, w, w); + public readonly float4 xxxx => new(x, x, x, x); + public readonly float4 xxxy => new(x, x, x, y); + public readonly float4 xxxz => new(x, x, x, z); + public readonly float4 xxxw => new(x, x, x, w); + public readonly float4 xxyx => new(x, x, y, x); + public readonly float4 xxyy => new(x, x, y, y); + public readonly float4 xxyz => new(x, x, y, z); + public readonly float4 xxyw => new(x, x, y, w); + public readonly float4 xxzx => new(x, x, z, x); + public readonly float4 xxzy => new(x, x, z, y); + public readonly float4 xxzz => new(x, x, z, z); + public readonly float4 xxzw => new(x, x, z, w); + public readonly float4 xxwx => new(x, x, w, x); + public readonly float4 xxwy => new(x, x, w, y); + public readonly float4 xxwz => new(x, x, w, z); + public readonly float4 xxww => new(x, x, w, w); + public readonly float4 xyxx => new(x, y, x, x); + public readonly float4 xyxy => new(x, y, x, y); + public readonly float4 xyxz => new(x, y, x, z); + public readonly float4 xyxw => new(x, y, x, w); + public readonly float4 xyyx => new(x, y, y, x); + public readonly float4 xyyy => new(x, y, y, y); + public readonly float4 xyyz => new(x, y, y, z); + public readonly float4 xyyw => new(x, y, y, w); + public readonly float4 xyzx => new(x, y, z, x); + public readonly float4 xyzy => new(x, y, z, y); + public readonly float4 xyzz => new(x, y, z, z); + public float4 xyzw { readonly get => new(x, y, z, w); set { x = value.x; y = value.y; z = value.z; w = value.w; } } + public readonly float4 xywx => new(x, y, w, x); + public readonly float4 xywy => new(x, y, w, y); + public float4 xywz { readonly get => new(x, y, w, z); set { x = value.x; y = value.y; w = value.z; z = value.w; } } + public readonly float4 xyww => new(x, y, w, w); + public readonly float4 xzxx => new(x, z, x, x); + public readonly float4 xzxy => new(x, z, x, y); + public readonly float4 xzxz => new(x, z, x, z); + public readonly float4 xzxw => new(x, z, x, w); + public readonly float4 xzyx => new(x, z, y, x); + public readonly float4 xzyy => new(x, z, y, y); + public readonly float4 xzyz => new(x, z, y, z); + public float4 xzyw { readonly get => new(x, z, y, w); set { x = value.x; z = value.y; y = value.z; w = value.w; } } + public readonly float4 xzzx => new(x, z, z, x); + public readonly float4 xzzy => new(x, z, z, y); + public readonly float4 xzzz => new(x, z, z, z); + public readonly float4 xzzw => new(x, z, z, w); + public readonly float4 xzwx => new(x, z, w, x); + public float4 xzwy { readonly get => new(x, z, w, y); set { x = value.x; z = value.y; w = value.z; y = value.w; } } + public readonly float4 xzwz => new(x, z, w, z); + public readonly float4 xzww => new(x, z, w, w); + public readonly float4 xwxx => new(x, w, x, x); + public readonly float4 xwxy => new(x, w, x, y); + public readonly float4 xwxz => new(x, w, x, z); + public readonly float4 xwxw => new(x, w, x, w); + public readonly float4 xwyx => new(x, w, y, x); + public readonly float4 xwyy => new(x, w, y, y); + public float4 xwyz { readonly get => new(x, w, y, z); set { x = value.x; w = value.y; y = value.z; z = value.w; } } + public readonly float4 xwyw => new(x, w, y, w); + public readonly float4 xwzx => new(x, w, z, x); + public float4 xwzy { readonly get => new(x, w, z, y); set { x = value.x; w = value.y; z = value.z; y = value.w; } } + public readonly float4 xwzz => new(x, w, z, z); + public readonly float4 xwzw => new(x, w, z, w); + public readonly float4 xwwx => new(x, w, w, x); + public readonly float4 xwwy => new(x, w, w, y); + public readonly float4 xwwz => new(x, w, w, z); + public readonly float4 xwww => new(x, w, w, w); + public readonly float4 yxxx => new(y, x, x, x); + public readonly float4 yxxy => new(y, x, x, y); + public readonly float4 yxxz => new(y, x, x, z); + public readonly float4 yxxw => new(y, x, x, w); + public readonly float4 yxyx => new(y, x, y, x); + public readonly float4 yxyy => new(y, x, y, y); + public readonly float4 yxyz => new(y, x, y, z); + public readonly float4 yxyw => new(y, x, y, w); + public readonly float4 yxzx => new(y, x, z, x); + public readonly float4 yxzy => new(y, x, z, y); + public readonly float4 yxzz => new(y, x, z, z); + public float4 yxzw { readonly get => new(y, x, z, w); set { y = value.x; x = value.y; z = value.z; w = value.w; } } + public readonly float4 yxwx => new(y, x, w, x); + public readonly float4 yxwy => new(y, x, w, y); + public float4 yxwz { readonly get => new(y, x, w, z); set { y = value.x; x = value.y; w = value.z; z = value.w; } } + public readonly float4 yxww => new(y, x, w, w); + public readonly float4 yyxx => new(y, y, x, x); + public readonly float4 yyxy => new(y, y, x, y); + public readonly float4 yyxz => new(y, y, x, z); + public readonly float4 yyxw => new(y, y, x, w); + public readonly float4 yyyx => new(y, y, y, x); + public readonly float4 yyyy => new(y, y, y, y); + public readonly float4 yyyz => new(y, y, y, z); + public readonly float4 yyyw => new(y, y, y, w); + public readonly float4 yyzx => new(y, y, z, x); + public readonly float4 yyzy => new(y, y, z, y); + public readonly float4 yyzz => new(y, y, z, z); + public readonly float4 yyzw => new(y, y, z, w); + public readonly float4 yywx => new(y, y, w, x); + public readonly float4 yywy => new(y, y, w, y); + public readonly float4 yywz => new(y, y, w, z); + public readonly float4 yyww => new(y, y, w, w); + public readonly float4 yzxx => new(y, z, x, x); + public readonly float4 yzxy => new(y, z, x, y); + public readonly float4 yzxz => new(y, z, x, z); + public float4 yzxw { readonly get => new(y, z, x, w); set { y = value.x; z = value.y; x = value.z; w = value.w; } } + public readonly float4 yzyx => new(y, z, y, x); + public readonly float4 yzyy => new(y, z, y, y); + public readonly float4 yzyz => new(y, z, y, z); + public readonly float4 yzyw => new(y, z, y, w); + public readonly float4 yzzx => new(y, z, z, x); + public readonly float4 yzzy => new(y, z, z, y); + public readonly float4 yzzz => new(y, z, z, z); + public readonly float4 yzzw => new(y, z, z, w); + public float4 yzwx { readonly get => new(y, z, w, x); set { y = value.x; z = value.y; w = value.z; x = value.w; } } + public readonly float4 yzwy => new(y, z, w, y); + public readonly float4 yzwz => new(y, z, w, z); + public readonly float4 yzww => new(y, z, w, w); + public readonly float4 ywxx => new(y, w, x, x); + public readonly float4 ywxy => new(y, w, x, y); + public float4 ywxz { readonly get => new(y, w, x, z); set { y = value.x; w = value.y; x = value.z; z = value.w; } } + public readonly float4 ywxw => new(y, w, x, w); + public readonly float4 ywyx => new(y, w, y, x); + public readonly float4 ywyy => new(y, w, y, y); + public readonly float4 ywyz => new(y, w, y, z); + public readonly float4 ywyw => new(y, w, y, w); + public float4 ywzx { readonly get => new(y, w, z, x); set { y = value.x; w = value.y; z = value.z; x = value.w; } } + public readonly float4 ywzy => new(y, w, z, y); + public readonly float4 ywzz => new(y, w, z, z); + public readonly float4 ywzw => new(y, w, z, w); + public readonly float4 ywwx => new(y, w, w, x); + public readonly float4 ywwy => new(y, w, w, y); + public readonly float4 ywwz => new(y, w, w, z); + public readonly float4 ywww => new(y, w, w, w); + public readonly float4 zxxx => new(z, x, x, x); + public readonly float4 zxxy => new(z, x, x, y); + public readonly float4 zxxz => new(z, x, x, z); + public readonly float4 zxxw => new(z, x, x, w); + public readonly float4 zxyx => new(z, x, y, x); + public readonly float4 zxyy => new(z, x, y, y); + public readonly float4 zxyz => new(z, x, y, z); + public float4 zxyw { readonly get => new(z, x, y, w); set { z = value.x; x = value.y; y = value.z; w = value.w; } } + public readonly float4 zxzx => new(z, x, z, x); + public readonly float4 zxzy => new(z, x, z, y); + public readonly float4 zxzz => new(z, x, z, z); + public readonly float4 zxzw => new(z, x, z, w); + public readonly float4 zxwx => new(z, x, w, x); + public float4 zxwy { readonly get => new(z, x, w, y); set { z = value.x; x = value.y; w = value.z; y = value.w; } } + public readonly float4 zxwz => new(z, x, w, z); + public readonly float4 zxww => new(z, x, w, w); + public readonly float4 zyxx => new(z, y, x, x); + public readonly float4 zyxy => new(z, y, x, y); + public readonly float4 zyxz => new(z, y, x, z); + public float4 zyxw { readonly get => new(z, y, x, w); set { z = value.x; y = value.y; x = value.z; w = value.w; } } + public readonly float4 zyyx => new(z, y, y, x); + public readonly float4 zyyy => new(z, y, y, y); + public readonly float4 zyyz => new(z, y, y, z); + public readonly float4 zyyw => new(z, y, y, w); + public readonly float4 zyzx => new(z, y, z, x); + public readonly float4 zyzy => new(z, y, z, y); + public readonly float4 zyzz => new(z, y, z, z); + public readonly float4 zyzw => new(z, y, z, w); + public float4 zywx { readonly get => new(z, y, w, x); set { z = value.x; y = value.y; w = value.z; x = value.w; } } + public readonly float4 zywy => new(z, y, w, y); + public readonly float4 zywz => new(z, y, w, z); + public readonly float4 zyww => new(z, y, w, w); + public readonly float4 zzxx => new(z, z, x, x); + public readonly float4 zzxy => new(z, z, x, y); + public readonly float4 zzxz => new(z, z, x, z); + public readonly float4 zzxw => new(z, z, x, w); + public readonly float4 zzyx => new(z, z, y, x); + public readonly float4 zzyy => new(z, z, y, y); + public readonly float4 zzyz => new(z, z, y, z); + public readonly float4 zzyw => new(z, z, y, w); + public readonly float4 zzzx => new(z, z, z, x); + public readonly float4 zzzy => new(z, z, z, y); + public readonly float4 zzzz => new(z, z, z, z); + public readonly float4 zzzw => new(z, z, z, w); + public readonly float4 zzwx => new(z, z, w, x); + public readonly float4 zzwy => new(z, z, w, y); + public readonly float4 zzwz => new(z, z, w, z); + public readonly float4 zzww => new(z, z, w, w); + public readonly float4 zwxx => new(z, w, x, x); + public float4 zwxy { readonly get => new(z, w, x, y); set { z = value.x; w = value.y; x = value.z; y = value.w; } } + public readonly float4 zwxz => new(z, w, x, z); + public readonly float4 zwxw => new(z, w, x, w); + public float4 zwyx { readonly get => new(z, w, y, x); set { z = value.x; w = value.y; y = value.z; x = value.w; } } + public readonly float4 zwyy => new(z, w, y, y); + public readonly float4 zwyz => new(z, w, y, z); + public readonly float4 zwyw => new(z, w, y, w); + public readonly float4 zwzx => new(z, w, z, x); + public readonly float4 zwzy => new(z, w, z, y); + public readonly float4 zwzz => new(z, w, z, z); + public readonly float4 zwzw => new(z, w, z, w); + public readonly float4 zwwx => new(z, w, w, x); + public readonly float4 zwwy => new(z, w, w, y); + public readonly float4 zwwz => new(z, w, w, z); + public readonly float4 zwww => new(z, w, w, w); + public readonly float4 wxxx => new(w, x, x, x); + public readonly float4 wxxy => new(w, x, x, y); + public readonly float4 wxxz => new(w, x, x, z); + public readonly float4 wxxw => new(w, x, x, w); + public readonly float4 wxyx => new(w, x, y, x); + public readonly float4 wxyy => new(w, x, y, y); + public float4 wxyz { readonly get => new(w, x, y, z); set { w = value.x; x = value.y; y = value.z; z = value.w; } } + public readonly float4 wxyw => new(w, x, y, w); + public readonly float4 wxzx => new(w, x, z, x); + public float4 wxzy { readonly get => new(w, x, z, y); set { w = value.x; x = value.y; z = value.z; y = value.w; } } + public readonly float4 wxzz => new(w, x, z, z); + public readonly float4 wxzw => new(w, x, z, w); + public readonly float4 wxwx => new(w, x, w, x); + public readonly float4 wxwy => new(w, x, w, y); + public readonly float4 wxwz => new(w, x, w, z); + public readonly float4 wxww => new(w, x, w, w); + public readonly float4 wyxx => new(w, y, x, x); + public readonly float4 wyxy => new(w, y, x, y); + public float4 wyxz { readonly get => new(w, y, x, z); set { w = value.x; y = value.y; x = value.z; z = value.w; } } + public readonly float4 wyxw => new(w, y, x, w); + public readonly float4 wyyx => new(w, y, y, x); + public readonly float4 wyyy => new(w, y, y, y); + public readonly float4 wyyz => new(w, y, y, z); + public readonly float4 wyyw => new(w, y, y, w); + public float4 wyzx { readonly get => new(w, y, z, x); set { w = value.x; y = value.y; z = value.z; x = value.w; } } + public readonly float4 wyzy => new(w, y, z, y); + public readonly float4 wyzz => new(w, y, z, z); + public readonly float4 wyzw => new(w, y, z, w); + public readonly float4 wywx => new(w, y, w, x); + public readonly float4 wywy => new(w, y, w, y); + public readonly float4 wywz => new(w, y, w, z); + public readonly float4 wyww => new(w, y, w, w); + public readonly float4 wzxx => new(w, z, x, x); + public float4 wzxy { readonly get => new(w, z, x, y); set { w = value.x; z = value.y; x = value.z; y = value.w; } } + public readonly float4 wzxz => new(w, z, x, z); + public readonly float4 wzxw => new(w, z, x, w); + public float4 wzyx { readonly get => new(w, z, y, x); set { w = value.x; z = value.y; y = value.z; x = value.w; } } + public readonly float4 wzyy => new(w, z, y, y); + public readonly float4 wzyz => new(w, z, y, z); + public readonly float4 wzyw => new(w, z, y, w); + public readonly float4 wzzx => new(w, z, z, x); + public readonly float4 wzzy => new(w, z, z, y); + public readonly float4 wzzz => new(w, z, z, z); + public readonly float4 wzzw => new(w, z, z, w); + public readonly float4 wzwx => new(w, z, w, x); + public readonly float4 wzwy => new(w, z, w, y); + public readonly float4 wzwz => new(w, z, w, z); + public readonly float4 wzww => new(w, z, w, w); + public readonly float4 wwxx => new(w, w, x, x); + public readonly float4 wwxy => new(w, w, x, y); + public readonly float4 wwxz => new(w, w, x, z); + public readonly float4 wwxw => new(w, w, x, w); + public readonly float4 wwyx => new(w, w, y, x); + public readonly float4 wwyy => new(w, w, y, y); + public readonly float4 wwyz => new(w, w, y, z); + public readonly float4 wwyw => new(w, w, y, w); + public readonly float4 wwzx => new(w, w, z, x); + public readonly float4 wwzy => new(w, w, z, y); + public readonly float4 wwzz => new(w, w, z, z); + public readonly float4 wwzw => new(w, w, z, w); + public readonly float4 wwwx => new(w, w, w, x); + public readonly float4 wwwy => new(w, w, w, y); + public readonly float4 wwwz => new(w, w, w, z); + public readonly float4 wwww => new(w, w, w, w); + public readonly float2 rr => new(x, x); + public float2 rg { readonly get => new(x, y); set { x = value.x; y = value.y; } } + public float2 rb { readonly get => new(x, z); set { x = value.x; z = value.y; } } + public float2 ra { readonly get => new(x, w); set { x = value.x; w = value.y; } } + public float2 gr { readonly get => new(y, x); set { y = value.x; x = value.y; } } + public readonly float2 gg => new(y, y); + public float2 gb { readonly get => new(y, z); set { y = value.x; z = value.y; } } + public float2 ga { readonly get => new(y, w); set { y = value.x; w = value.y; } } + public float2 br { readonly get => new(z, x); set { z = value.x; x = value.y; } } + public float2 bg { readonly get => new(z, y); set { z = value.x; y = value.y; } } + public readonly float2 bb => new(z, z); + public float2 ba { readonly get => new(z, w); set { z = value.x; w = value.y; } } + public float2 ar { readonly get => new(w, x); set { w = value.x; x = value.y; } } + public float2 ag { readonly get => new(w, y); set { w = value.x; y = value.y; } } + public float2 ab { readonly get => new(w, z); set { w = value.x; z = value.y; } } + public readonly float2 aa => new(w, w); + public readonly float3 rrr => new(x, x, x); + public readonly float3 rrg => new(x, x, y); + public readonly float3 rrb => new(x, x, z); + public readonly float3 rra => new(x, x, w); + public readonly float3 rgr => new(x, y, x); + public readonly float3 rgg => new(x, y, y); + public float3 rgb { readonly get => new(x, y, z); set { x = value.x; y = value.y; z = value.z; } } + public float3 rga { readonly get => new(x, y, w); set { x = value.x; y = value.y; w = value.z; } } + public readonly float3 rbr => new(x, z, x); + public float3 rbg { readonly get => new(x, z, y); set { x = value.x; z = value.y; y = value.z; } } + public readonly float3 rbb => new(x, z, z); + public float3 rba { readonly get => new(x, z, w); set { x = value.x; z = value.y; w = value.z; } } + public readonly float3 rar => new(x, w, x); + public float3 rag { readonly get => new(x, w, y); set { x = value.x; w = value.y; y = value.z; } } + public float3 rab { readonly get => new(x, w, z); set { x = value.x; w = value.y; z = value.z; } } + public readonly float3 raa => new(x, w, w); + public readonly float3 grr => new(y, x, x); + public readonly float3 grg => new(y, x, y); + public float3 grb { readonly get => new(y, x, z); set { y = value.x; x = value.y; z = value.z; } } + public float3 gra { readonly get => new(y, x, w); set { y = value.x; x = value.y; w = value.z; } } + public readonly float3 ggr => new(y, y, x); + public readonly float3 ggg => new(y, y, y); + public readonly float3 ggb => new(y, y, z); + public readonly float3 gga => new(y, y, w); + public float3 gbr { readonly get => new(y, z, x); set { y = value.x; z = value.y; x = value.z; } } + public readonly float3 gbg => new(y, z, y); + public readonly float3 gbb => new(y, z, z); + public float3 gba { readonly get => new(y, z, w); set { y = value.x; z = value.y; w = value.z; } } + public float3 gar { readonly get => new(y, w, x); set { y = value.x; w = value.y; x = value.z; } } + public readonly float3 gag => new(y, w, y); + public float3 gab { readonly get => new(y, w, z); set { y = value.x; w = value.y; z = value.z; } } + public readonly float3 gaa => new(y, w, w); + public readonly float3 brr => new(z, x, x); + public float3 brg { readonly get => new(z, x, y); set { z = value.x; x = value.y; y = value.z; } } + public readonly float3 brb => new(z, x, z); + public float3 bra { readonly get => new(z, x, w); set { z = value.x; x = value.y; w = value.z; } } + public float3 bgr { readonly get => new(z, y, x); set { z = value.x; y = value.y; x = value.z; } } + public readonly float3 bgg => new(z, y, y); + public readonly float3 bgb => new(z, y, z); + public float3 bga { readonly get => new(z, y, w); set { z = value.x; y = value.y; w = value.z; } } + public readonly float3 bbr => new(z, z, x); + public readonly float3 bbg => new(z, z, y); + public readonly float3 bbb => new(z, z, z); + public readonly float3 bba => new(z, z, w); + public float3 bar { readonly get => new(z, w, x); set { z = value.x; w = value.y; x = value.z; } } + public float3 bag { readonly get => new(z, w, y); set { z = value.x; w = value.y; y = value.z; } } + public readonly float3 bab => new(z, w, z); + public readonly float3 baa => new(z, w, w); + public readonly float3 arr => new(w, x, x); + public float3 arg { readonly get => new(w, x, y); set { w = value.x; x = value.y; y = value.z; } } + public float3 arb { readonly get => new(w, x, z); set { w = value.x; x = value.y; z = value.z; } } + public readonly float3 ara => new(w, x, w); + public float3 agr { readonly get => new(w, y, x); set { w = value.x; y = value.y; x = value.z; } } + public readonly float3 agg => new(w, y, y); + public float3 agb { readonly get => new(w, y, z); set { w = value.x; y = value.y; z = value.z; } } + public readonly float3 aga => new(w, y, w); + public float3 abr { readonly get => new(w, z, x); set { w = value.x; z = value.y; x = value.z; } } + public float3 abg { readonly get => new(w, z, y); set { w = value.x; z = value.y; y = value.z; } } + public readonly float3 abb => new(w, z, z); + public readonly float3 aba => new(w, z, w); + public readonly float3 aar => new(w, w, x); + public readonly float3 aag => new(w, w, y); + public readonly float3 aab => new(w, w, z); + public readonly float3 aaa => new(w, w, w); + public readonly float4 rrrr => new(x, x, x, x); + public readonly float4 rrrg => new(x, x, x, y); + public readonly float4 rrrb => new(x, x, x, z); + public readonly float4 rrra => new(x, x, x, w); + public readonly float4 rrgr => new(x, x, y, x); + public readonly float4 rrgg => new(x, x, y, y); + public readonly float4 rrgb => new(x, x, y, z); + public readonly float4 rrga => new(x, x, y, w); + public readonly float4 rrbr => new(x, x, z, x); + public readonly float4 rrbg => new(x, x, z, y); + public readonly float4 rrbb => new(x, x, z, z); + public readonly float4 rrba => new(x, x, z, w); + public readonly float4 rrar => new(x, x, w, x); + public readonly float4 rrag => new(x, x, w, y); + public readonly float4 rrab => new(x, x, w, z); + public readonly float4 rraa => new(x, x, w, w); + public readonly float4 rgrr => new(x, y, x, x); + public readonly float4 rgrg => new(x, y, x, y); + public readonly float4 rgrb => new(x, y, x, z); + public readonly float4 rgra => new(x, y, x, w); + public readonly float4 rggr => new(x, y, y, x); + public readonly float4 rggg => new(x, y, y, y); + public readonly float4 rggb => new(x, y, y, z); + public readonly float4 rgga => new(x, y, y, w); + public readonly float4 rgbr => new(x, y, z, x); + public readonly float4 rgbg => new(x, y, z, y); + public readonly float4 rgbb => new(x, y, z, z); + public float4 rgba { readonly get => new(x, y, z, w); set { x = value.x; y = value.y; z = value.z; w = value.w; } } + public readonly float4 rgar => new(x, y, w, x); + public readonly float4 rgag => new(x, y, w, y); + public float4 rgab { readonly get => new(x, y, w, z); set { x = value.x; y = value.y; w = value.z; z = value.w; } } + public readonly float4 rgaa => new(x, y, w, w); + public readonly float4 rbrr => new(x, z, x, x); + public readonly float4 rbrg => new(x, z, x, y); + public readonly float4 rbrb => new(x, z, x, z); + public readonly float4 rbra => new(x, z, x, w); + public readonly float4 rbgr => new(x, z, y, x); + public readonly float4 rbgg => new(x, z, y, y); + public readonly float4 rbgb => new(x, z, y, z); + public float4 rbga { readonly get => new(x, z, y, w); set { x = value.x; z = value.y; y = value.z; w = value.w; } } + public readonly float4 rbbr => new(x, z, z, x); + public readonly float4 rbbg => new(x, z, z, y); + public readonly float4 rbbb => new(x, z, z, z); + public readonly float4 rbba => new(x, z, z, w); + public readonly float4 rbar => new(x, z, w, x); + public float4 rbag { readonly get => new(x, z, w, y); set { x = value.x; z = value.y; w = value.z; y = value.w; } } + public readonly float4 rbab => new(x, z, w, z); + public readonly float4 rbaa => new(x, z, w, w); + public readonly float4 rarr => new(x, w, x, x); + public readonly float4 rarg => new(x, w, x, y); + public readonly float4 rarb => new(x, w, x, z); + public readonly float4 rara => new(x, w, x, w); + public readonly float4 ragr => new(x, w, y, x); + public readonly float4 ragg => new(x, w, y, y); + public float4 ragb { readonly get => new(x, w, y, z); set { x = value.x; w = value.y; y = value.z; z = value.w; } } + public readonly float4 raga => new(x, w, y, w); + public readonly float4 rabr => new(x, w, z, x); + public float4 rabg { readonly get => new(x, w, z, y); set { x = value.x; w = value.y; z = value.z; y = value.w; } } + public readonly float4 rabb => new(x, w, z, z); + public readonly float4 raba => new(x, w, z, w); + public readonly float4 raar => new(x, w, w, x); + public readonly float4 raag => new(x, w, w, y); + public readonly float4 raab => new(x, w, w, z); + public readonly float4 raaa => new(x, w, w, w); + public readonly float4 grrr => new(y, x, x, x); + public readonly float4 grrg => new(y, x, x, y); + public readonly float4 grrb => new(y, x, x, z); + public readonly float4 grra => new(y, x, x, w); + public readonly float4 grgr => new(y, x, y, x); + public readonly float4 grgg => new(y, x, y, y); + public readonly float4 grgb => new(y, x, y, z); + public readonly float4 grga => new(y, x, y, w); + public readonly float4 grbr => new(y, x, z, x); + public readonly float4 grbg => new(y, x, z, y); + public readonly float4 grbb => new(y, x, z, z); + public float4 grba { readonly get => new(y, x, z, w); set { y = value.x; x = value.y; z = value.z; w = value.w; } } + public readonly float4 grar => new(y, x, w, x); + public readonly float4 grag => new(y, x, w, y); + public float4 grab { readonly get => new(y, x, w, z); set { y = value.x; x = value.y; w = value.z; z = value.w; } } + public readonly float4 graa => new(y, x, w, w); + public readonly float4 ggrr => new(y, y, x, x); + public readonly float4 ggrg => new(y, y, x, y); + public readonly float4 ggrb => new(y, y, x, z); + public readonly float4 ggra => new(y, y, x, w); + public readonly float4 gggr => new(y, y, y, x); + public readonly float4 gggg => new(y, y, y, y); + public readonly float4 gggb => new(y, y, y, z); + public readonly float4 ggga => new(y, y, y, w); + public readonly float4 ggbr => new(y, y, z, x); + public readonly float4 ggbg => new(y, y, z, y); + public readonly float4 ggbb => new(y, y, z, z); + public readonly float4 ggba => new(y, y, z, w); + public readonly float4 ggar => new(y, y, w, x); + public readonly float4 ggag => new(y, y, w, y); + public readonly float4 ggab => new(y, y, w, z); + public readonly float4 ggaa => new(y, y, w, w); + public readonly float4 gbrr => new(y, z, x, x); + public readonly float4 gbrg => new(y, z, x, y); + public readonly float4 gbrb => new(y, z, x, z); + public float4 gbra { readonly get => new(y, z, x, w); set { y = value.x; z = value.y; x = value.z; w = value.w; } } + public readonly float4 gbgr => new(y, z, y, x); + public readonly float4 gbgg => new(y, z, y, y); + public readonly float4 gbgb => new(y, z, y, z); + public readonly float4 gbga => new(y, z, y, w); + public readonly float4 gbbr => new(y, z, z, x); + public readonly float4 gbbg => new(y, z, z, y); + public readonly float4 gbbb => new(y, z, z, z); + public readonly float4 gbba => new(y, z, z, w); + public float4 gbar { readonly get => new(y, z, w, x); set { y = value.x; z = value.y; w = value.z; x = value.w; } } + public readonly float4 gbag => new(y, z, w, y); + public readonly float4 gbab => new(y, z, w, z); + public readonly float4 gbaa => new(y, z, w, w); + public readonly float4 garr => new(y, w, x, x); + public readonly float4 garg => new(y, w, x, y); + public float4 garb { readonly get => new(y, w, x, z); set { y = value.x; w = value.y; x = value.z; z = value.w; } } + public readonly float4 gara => new(y, w, x, w); + public readonly float4 gagr => new(y, w, y, x); + public readonly float4 gagg => new(y, w, y, y); + public readonly float4 gagb => new(y, w, y, z); + public readonly float4 gaga => new(y, w, y, w); + public float4 gabr { readonly get => new(y, w, z, x); set { y = value.x; w = value.y; z = value.z; x = value.w; } } + public readonly float4 gabg => new(y, w, z, y); + public readonly float4 gabb => new(y, w, z, z); + public readonly float4 gaba => new(y, w, z, w); + public readonly float4 gaar => new(y, w, w, x); + public readonly float4 gaag => new(y, w, w, y); + public readonly float4 gaab => new(y, w, w, z); + public readonly float4 gaaa => new(y, w, w, w); + public readonly float4 brrr => new(z, x, x, x); + public readonly float4 brrg => new(z, x, x, y); + public readonly float4 brrb => new(z, x, x, z); + public readonly float4 brra => new(z, x, x, w); + public readonly float4 brgr => new(z, x, y, x); + public readonly float4 brgg => new(z, x, y, y); + public readonly float4 brgb => new(z, x, y, z); + public float4 brga { readonly get => new(z, x, y, w); set { z = value.x; x = value.y; y = value.z; w = value.w; } } + public readonly float4 brbr => new(z, x, z, x); + public readonly float4 brbg => new(z, x, z, y); + public readonly float4 brbb => new(z, x, z, z); + public readonly float4 brba => new(z, x, z, w); + public readonly float4 brar => new(z, x, w, x); + public float4 brag { readonly get => new(z, x, w, y); set { z = value.x; x = value.y; w = value.z; y = value.w; } } + public readonly float4 brab => new(z, x, w, z); + public readonly float4 braa => new(z, x, w, w); + public readonly float4 bgrr => new(z, y, x, x); + public readonly float4 bgrg => new(z, y, x, y); + public readonly float4 bgrb => new(z, y, x, z); + public float4 bgra { readonly get => new(z, y, x, w); set { z = value.x; y = value.y; x = value.z; w = value.w; } } + public readonly float4 bggr => new(z, y, y, x); + public readonly float4 bggg => new(z, y, y, y); + public readonly float4 bggb => new(z, y, y, z); + public readonly float4 bgga => new(z, y, y, w); + public readonly float4 bgbr => new(z, y, z, x); + public readonly float4 bgbg => new(z, y, z, y); + public readonly float4 bgbb => new(z, y, z, z); + public readonly float4 bgba => new(z, y, z, w); + public float4 bgar { readonly get => new(z, y, w, x); set { z = value.x; y = value.y; w = value.z; x = value.w; } } + public readonly float4 bgag => new(z, y, w, y); + public readonly float4 bgab => new(z, y, w, z); + public readonly float4 bgaa => new(z, y, w, w); + public readonly float4 bbrr => new(z, z, x, x); + public readonly float4 bbrg => new(z, z, x, y); + public readonly float4 bbrb => new(z, z, x, z); + public readonly float4 bbra => new(z, z, x, w); + public readonly float4 bbgr => new(z, z, y, x); + public readonly float4 bbgg => new(z, z, y, y); + public readonly float4 bbgb => new(z, z, y, z); + public readonly float4 bbga => new(z, z, y, w); + public readonly float4 bbbr => new(z, z, z, x); + public readonly float4 bbbg => new(z, z, z, y); + public readonly float4 bbbb => new(z, z, z, z); + public readonly float4 bbba => new(z, z, z, w); + public readonly float4 bbar => new(z, z, w, x); + public readonly float4 bbag => new(z, z, w, y); + public readonly float4 bbab => new(z, z, w, z); + public readonly float4 bbaa => new(z, z, w, w); + public readonly float4 barr => new(z, w, x, x); + public float4 barg { readonly get => new(z, w, x, y); set { z = value.x; w = value.y; x = value.z; y = value.w; } } + public readonly float4 barb => new(z, w, x, z); + public readonly float4 bara => new(z, w, x, w); + public float4 bagr { readonly get => new(z, w, y, x); set { z = value.x; w = value.y; y = value.z; x = value.w; } } + public readonly float4 bagg => new(z, w, y, y); + public readonly float4 bagb => new(z, w, y, z); + public readonly float4 baga => new(z, w, y, w); + public readonly float4 babr => new(z, w, z, x); + public readonly float4 babg => new(z, w, z, y); + public readonly float4 babb => new(z, w, z, z); + public readonly float4 baba => new(z, w, z, w); + public readonly float4 baar => new(z, w, w, x); + public readonly float4 baag => new(z, w, w, y); + public readonly float4 baab => new(z, w, w, z); + public readonly float4 baaa => new(z, w, w, w); + public readonly float4 arrr => new(w, x, x, x); + public readonly float4 arrg => new(w, x, x, y); + public readonly float4 arrb => new(w, x, x, z); + public readonly float4 arra => new(w, x, x, w); + public readonly float4 argr => new(w, x, y, x); + public readonly float4 argg => new(w, x, y, y); + public float4 argb { readonly get => new(w, x, y, z); set { w = value.x; x = value.y; y = value.z; z = value.w; } } + public readonly float4 arga => new(w, x, y, w); + public readonly float4 arbr => new(w, x, z, x); + public float4 arbg { readonly get => new(w, x, z, y); set { w = value.x; x = value.y; z = value.z; y = value.w; } } + public readonly float4 arbb => new(w, x, z, z); + public readonly float4 arba => new(w, x, z, w); + public readonly float4 arar => new(w, x, w, x); + public readonly float4 arag => new(w, x, w, y); + public readonly float4 arab => new(w, x, w, z); + public readonly float4 araa => new(w, x, w, w); + public readonly float4 agrr => new(w, y, x, x); + public readonly float4 agrg => new(w, y, x, y); + public float4 agrb { readonly get => new(w, y, x, z); set { w = value.x; y = value.y; x = value.z; z = value.w; } } + public readonly float4 agra => new(w, y, x, w); + public readonly float4 aggr => new(w, y, y, x); + public readonly float4 aggg => new(w, y, y, y); + public readonly float4 aggb => new(w, y, y, z); + public readonly float4 agga => new(w, y, y, w); + public float4 agbr { readonly get => new(w, y, z, x); set { w = value.x; y = value.y; z = value.z; x = value.w; } } + public readonly float4 agbg => new(w, y, z, y); + public readonly float4 agbb => new(w, y, z, z); + public readonly float4 agba => new(w, y, z, w); + public readonly float4 agar => new(w, y, w, x); + public readonly float4 agag => new(w, y, w, y); + public readonly float4 agab => new(w, y, w, z); + public readonly float4 agaa => new(w, y, w, w); + public readonly float4 abrr => new(w, z, x, x); + public float4 abrg { readonly get => new(w, z, x, y); set { w = value.x; z = value.y; x = value.z; y = value.w; } } + public readonly float4 abrb => new(w, z, x, z); + public readonly float4 abra => new(w, z, x, w); + public float4 abgr { readonly get => new(w, z, y, x); set { w = value.x; z = value.y; y = value.z; x = value.w; } } + public readonly float4 abgg => new(w, z, y, y); + public readonly float4 abgb => new(w, z, y, z); + public readonly float4 abga => new(w, z, y, w); + public readonly float4 abbr => new(w, z, z, x); + public readonly float4 abbg => new(w, z, z, y); + public readonly float4 abbb => new(w, z, z, z); + public readonly float4 abba => new(w, z, z, w); + public readonly float4 abar => new(w, z, w, x); + public readonly float4 abag => new(w, z, w, y); + public readonly float4 abab => new(w, z, w, z); + public readonly float4 abaa => new(w, z, w, w); + public readonly float4 aarr => new(w, w, x, x); + public readonly float4 aarg => new(w, w, x, y); + public readonly float4 aarb => new(w, w, x, z); + public readonly float4 aara => new(w, w, x, w); + public readonly float4 aagr => new(w, w, y, x); + public readonly float4 aagg => new(w, w, y, y); + public readonly float4 aagb => new(w, w, y, z); + public readonly float4 aaga => new(w, w, y, w); + public readonly float4 aabr => new(w, w, z, x); + public readonly float4 aabg => new(w, w, z, y); + public readonly float4 aabb => new(w, w, z, z); + public readonly float4 aaba => new(w, w, z, w); + public readonly float4 aaar => new(w, w, w, x); + public readonly float4 aaag => new(w, w, w, y); + public readonly float4 aaab => new(w, w, w, z); + public readonly float4 aaaa => new(w, w, w, w); +} + +public partial struct int2 +{ + public int2(int2 xy) : this(xy.x, xy.y) { } + public int r { readonly get => x; set => x = value; } + public int g { readonly get => y; set => y = value; } + public readonly int2 xx => new(x, x); + public int2 xy { readonly get => new(x, y); set { x = value.x; y = value.y; } } + public int2 yx { readonly get => new(y, x); set { y = value.x; x = value.y; } } + public readonly int2 yy => new(y, y); + public readonly int3 xxx => new(x, x, x); + public readonly int3 xxy => new(x, x, y); + public readonly int3 xyx => new(x, y, x); + public readonly int3 xyy => new(x, y, y); + public readonly int3 yxx => new(y, x, x); + public readonly int3 yxy => new(y, x, y); + public readonly int3 yyx => new(y, y, x); + public readonly int3 yyy => new(y, y, y); + public readonly int4 xxxx => new(x, x, x, x); + public readonly int4 xxxy => new(x, x, x, y); + public readonly int4 xxyx => new(x, x, y, x); + public readonly int4 xxyy => new(x, x, y, y); + public readonly int4 xyxx => new(x, y, x, x); + public readonly int4 xyxy => new(x, y, x, y); + public readonly int4 xyyx => new(x, y, y, x); + public readonly int4 xyyy => new(x, y, y, y); + public readonly int4 yxxx => new(y, x, x, x); + public readonly int4 yxxy => new(y, x, x, y); + public readonly int4 yxyx => new(y, x, y, x); + public readonly int4 yxyy => new(y, x, y, y); + public readonly int4 yyxx => new(y, y, x, x); + public readonly int4 yyxy => new(y, y, x, y); + public readonly int4 yyyx => new(y, y, y, x); + public readonly int4 yyyy => new(y, y, y, y); + public readonly int2 rr => new(x, x); + public int2 rg { readonly get => new(x, y); set { x = value.x; y = value.y; } } + public int2 gr { readonly get => new(y, x); set { y = value.x; x = value.y; } } + public readonly int2 gg => new(y, y); + public readonly int3 rrr => new(x, x, x); + public readonly int3 rrg => new(x, x, y); + public readonly int3 rgr => new(x, y, x); + public readonly int3 rgg => new(x, y, y); + public readonly int3 grr => new(y, x, x); + public readonly int3 grg => new(y, x, y); + public readonly int3 ggr => new(y, y, x); + public readonly int3 ggg => new(y, y, y); + public readonly int4 rrrr => new(x, x, x, x); + public readonly int4 rrrg => new(x, x, x, y); + public readonly int4 rrgr => new(x, x, y, x); + public readonly int4 rrgg => new(x, x, y, y); + public readonly int4 rgrr => new(x, y, x, x); + public readonly int4 rgrg => new(x, y, x, y); + public readonly int4 rggr => new(x, y, y, x); + public readonly int4 rggg => new(x, y, y, y); + public readonly int4 grrr => new(y, x, x, x); + public readonly int4 grrg => new(y, x, x, y); + public readonly int4 grgr => new(y, x, y, x); + public readonly int4 grgg => new(y, x, y, y); + public readonly int4 ggrr => new(y, y, x, x); + public readonly int4 ggrg => new(y, y, x, y); + public readonly int4 gggr => new(y, y, y, x); + public readonly int4 gggg => new(y, y, y, y); +} + +public partial struct int3 +{ + public int3(int x, int2 yz) : this(x, yz.x, yz.y) { } + public int3(int2 xy, int z) : this(xy.x, xy.y, z) { } + public int3(int3 xyz) : this(xyz.x, xyz.y, xyz.z) { } + public int r { readonly get => x; set => x = value; } + public int g { readonly get => y; set => y = value; } + public int b { readonly get => z; set => z = value; } + public readonly int2 xx => new(x, x); + public int2 xy { readonly get => new(x, y); set { x = value.x; y = value.y; } } + public int2 xz { readonly get => new(x, z); set { x = value.x; z = value.y; } } + public int2 yx { readonly get => new(y, x); set { y = value.x; x = value.y; } } + public readonly int2 yy => new(y, y); + public int2 yz { readonly get => new(y, z); set { y = value.x; z = value.y; } } + public int2 zx { readonly get => new(z, x); set { z = value.x; x = value.y; } } + public int2 zy { readonly get => new(z, y); set { z = value.x; y = value.y; } } + public readonly int2 zz => new(z, z); + public readonly int3 xxx => new(x, x, x); + public readonly int3 xxy => new(x, x, y); + public readonly int3 xxz => new(x, x, z); + public readonly int3 xyx => new(x, y, x); + public readonly int3 xyy => new(x, y, y); + public int3 xyz { readonly get => new(x, y, z); set { x = value.x; y = value.y; z = value.z; } } + public readonly int3 xzx => new(x, z, x); + public int3 xzy { readonly get => new(x, z, y); set { x = value.x; z = value.y; y = value.z; } } + public readonly int3 xzz => new(x, z, z); + public readonly int3 yxx => new(y, x, x); + public readonly int3 yxy => new(y, x, y); + public int3 yxz { readonly get => new(y, x, z); set { y = value.x; x = value.y; z = value.z; } } + public readonly int3 yyx => new(y, y, x); + public readonly int3 yyy => new(y, y, y); + public readonly int3 yyz => new(y, y, z); + public int3 yzx { readonly get => new(y, z, x); set { y = value.x; z = value.y; x = value.z; } } + public readonly int3 yzy => new(y, z, y); + public readonly int3 yzz => new(y, z, z); + public readonly int3 zxx => new(z, x, x); + public int3 zxy { readonly get => new(z, x, y); set { z = value.x; x = value.y; y = value.z; } } + public readonly int3 zxz => new(z, x, z); + public int3 zyx { readonly get => new(z, y, x); set { z = value.x; y = value.y; x = value.z; } } + public readonly int3 zyy => new(z, y, y); + public readonly int3 zyz => new(z, y, z); + public readonly int3 zzx => new(z, z, x); + public readonly int3 zzy => new(z, z, y); + public readonly int3 zzz => new(z, z, z); + public readonly int4 xxxx => new(x, x, x, x); + public readonly int4 xxxy => new(x, x, x, y); + public readonly int4 xxxz => new(x, x, x, z); + public readonly int4 xxyx => new(x, x, y, x); + public readonly int4 xxyy => new(x, x, y, y); + public readonly int4 xxyz => new(x, x, y, z); + public readonly int4 xxzx => new(x, x, z, x); + public readonly int4 xxzy => new(x, x, z, y); + public readonly int4 xxzz => new(x, x, z, z); + public readonly int4 xyxx => new(x, y, x, x); + public readonly int4 xyxy => new(x, y, x, y); + public readonly int4 xyxz => new(x, y, x, z); + public readonly int4 xyyx => new(x, y, y, x); + public readonly int4 xyyy => new(x, y, y, y); + public readonly int4 xyyz => new(x, y, y, z); + public readonly int4 xyzx => new(x, y, z, x); + public readonly int4 xyzy => new(x, y, z, y); + public readonly int4 xyzz => new(x, y, z, z); + public readonly int4 xzxx => new(x, z, x, x); + public readonly int4 xzxy => new(x, z, x, y); + public readonly int4 xzxz => new(x, z, x, z); + public readonly int4 xzyx => new(x, z, y, x); + public readonly int4 xzyy => new(x, z, y, y); + public readonly int4 xzyz => new(x, z, y, z); + public readonly int4 xzzx => new(x, z, z, x); + public readonly int4 xzzy => new(x, z, z, y); + public readonly int4 xzzz => new(x, z, z, z); + public readonly int4 yxxx => new(y, x, x, x); + public readonly int4 yxxy => new(y, x, x, y); + public readonly int4 yxxz => new(y, x, x, z); + public readonly int4 yxyx => new(y, x, y, x); + public readonly int4 yxyy => new(y, x, y, y); + public readonly int4 yxyz => new(y, x, y, z); + public readonly int4 yxzx => new(y, x, z, x); + public readonly int4 yxzy => new(y, x, z, y); + public readonly int4 yxzz => new(y, x, z, z); + public readonly int4 yyxx => new(y, y, x, x); + public readonly int4 yyxy => new(y, y, x, y); + public readonly int4 yyxz => new(y, y, x, z); + public readonly int4 yyyx => new(y, y, y, x); + public readonly int4 yyyy => new(y, y, y, y); + public readonly int4 yyyz => new(y, y, y, z); + public readonly int4 yyzx => new(y, y, z, x); + public readonly int4 yyzy => new(y, y, z, y); + public readonly int4 yyzz => new(y, y, z, z); + public readonly int4 yzxx => new(y, z, x, x); + public readonly int4 yzxy => new(y, z, x, y); + public readonly int4 yzxz => new(y, z, x, z); + public readonly int4 yzyx => new(y, z, y, x); + public readonly int4 yzyy => new(y, z, y, y); + public readonly int4 yzyz => new(y, z, y, z); + public readonly int4 yzzx => new(y, z, z, x); + public readonly int4 yzzy => new(y, z, z, y); + public readonly int4 yzzz => new(y, z, z, z); + public readonly int4 zxxx => new(z, x, x, x); + public readonly int4 zxxy => new(z, x, x, y); + public readonly int4 zxxz => new(z, x, x, z); + public readonly int4 zxyx => new(z, x, y, x); + public readonly int4 zxyy => new(z, x, y, y); + public readonly int4 zxyz => new(z, x, y, z); + public readonly int4 zxzx => new(z, x, z, x); + public readonly int4 zxzy => new(z, x, z, y); + public readonly int4 zxzz => new(z, x, z, z); + public readonly int4 zyxx => new(z, y, x, x); + public readonly int4 zyxy => new(z, y, x, y); + public readonly int4 zyxz => new(z, y, x, z); + public readonly int4 zyyx => new(z, y, y, x); + public readonly int4 zyyy => new(z, y, y, y); + public readonly int4 zyyz => new(z, y, y, z); + public readonly int4 zyzx => new(z, y, z, x); + public readonly int4 zyzy => new(z, y, z, y); + public readonly int4 zyzz => new(z, y, z, z); + public readonly int4 zzxx => new(z, z, x, x); + public readonly int4 zzxy => new(z, z, x, y); + public readonly int4 zzxz => new(z, z, x, z); + public readonly int4 zzyx => new(z, z, y, x); + public readonly int4 zzyy => new(z, z, y, y); + public readonly int4 zzyz => new(z, z, y, z); + public readonly int4 zzzx => new(z, z, z, x); + public readonly int4 zzzy => new(z, z, z, y); + public readonly int4 zzzz => new(z, z, z, z); + public readonly int2 rr => new(x, x); + public int2 rg { readonly get => new(x, y); set { x = value.x; y = value.y; } } + public int2 rb { readonly get => new(x, z); set { x = value.x; z = value.y; } } + public int2 gr { readonly get => new(y, x); set { y = value.x; x = value.y; } } + public readonly int2 gg => new(y, y); + public int2 gb { readonly get => new(y, z); set { y = value.x; z = value.y; } } + public int2 br { readonly get => new(z, x); set { z = value.x; x = value.y; } } + public int2 bg { readonly get => new(z, y); set { z = value.x; y = value.y; } } + public readonly int2 bb => new(z, z); + public readonly int3 rrr => new(x, x, x); + public readonly int3 rrg => new(x, x, y); + public readonly int3 rrb => new(x, x, z); + public readonly int3 rgr => new(x, y, x); + public readonly int3 rgg => new(x, y, y); + public int3 rgb { readonly get => new(x, y, z); set { x = value.x; y = value.y; z = value.z; } } + public readonly int3 rbr => new(x, z, x); + public int3 rbg { readonly get => new(x, z, y); set { x = value.x; z = value.y; y = value.z; } } + public readonly int3 rbb => new(x, z, z); + public readonly int3 grr => new(y, x, x); + public readonly int3 grg => new(y, x, y); + public int3 grb { readonly get => new(y, x, z); set { y = value.x; x = value.y; z = value.z; } } + public readonly int3 ggr => new(y, y, x); + public readonly int3 ggg => new(y, y, y); + public readonly int3 ggb => new(y, y, z); + public int3 gbr { readonly get => new(y, z, x); set { y = value.x; z = value.y; x = value.z; } } + public readonly int3 gbg => new(y, z, y); + public readonly int3 gbb => new(y, z, z); + public readonly int3 brr => new(z, x, x); + public int3 brg { readonly get => new(z, x, y); set { z = value.x; x = value.y; y = value.z; } } + public readonly int3 brb => new(z, x, z); + public int3 bgr { readonly get => new(z, y, x); set { z = value.x; y = value.y; x = value.z; } } + public readonly int3 bgg => new(z, y, y); + public readonly int3 bgb => new(z, y, z); + public readonly int3 bbr => new(z, z, x); + public readonly int3 bbg => new(z, z, y); + public readonly int3 bbb => new(z, z, z); + public readonly int4 rrrr => new(x, x, x, x); + public readonly int4 rrrg => new(x, x, x, y); + public readonly int4 rrrb => new(x, x, x, z); + public readonly int4 rrgr => new(x, x, y, x); + public readonly int4 rrgg => new(x, x, y, y); + public readonly int4 rrgb => new(x, x, y, z); + public readonly int4 rrbr => new(x, x, z, x); + public readonly int4 rrbg => new(x, x, z, y); + public readonly int4 rrbb => new(x, x, z, z); + public readonly int4 rgrr => new(x, y, x, x); + public readonly int4 rgrg => new(x, y, x, y); + public readonly int4 rgrb => new(x, y, x, z); + public readonly int4 rggr => new(x, y, y, x); + public readonly int4 rggg => new(x, y, y, y); + public readonly int4 rggb => new(x, y, y, z); + public readonly int4 rgbr => new(x, y, z, x); + public readonly int4 rgbg => new(x, y, z, y); + public readonly int4 rgbb => new(x, y, z, z); + public readonly int4 rbrr => new(x, z, x, x); + public readonly int4 rbrg => new(x, z, x, y); + public readonly int4 rbrb => new(x, z, x, z); + public readonly int4 rbgr => new(x, z, y, x); + public readonly int4 rbgg => new(x, z, y, y); + public readonly int4 rbgb => new(x, z, y, z); + public readonly int4 rbbr => new(x, z, z, x); + public readonly int4 rbbg => new(x, z, z, y); + public readonly int4 rbbb => new(x, z, z, z); + public readonly int4 grrr => new(y, x, x, x); + public readonly int4 grrg => new(y, x, x, y); + public readonly int4 grrb => new(y, x, x, z); + public readonly int4 grgr => new(y, x, y, x); + public readonly int4 grgg => new(y, x, y, y); + public readonly int4 grgb => new(y, x, y, z); + public readonly int4 grbr => new(y, x, z, x); + public readonly int4 grbg => new(y, x, z, y); + public readonly int4 grbb => new(y, x, z, z); + public readonly int4 ggrr => new(y, y, x, x); + public readonly int4 ggrg => new(y, y, x, y); + public readonly int4 ggrb => new(y, y, x, z); + public readonly int4 gggr => new(y, y, y, x); + public readonly int4 gggg => new(y, y, y, y); + public readonly int4 gggb => new(y, y, y, z); + public readonly int4 ggbr => new(y, y, z, x); + public readonly int4 ggbg => new(y, y, z, y); + public readonly int4 ggbb => new(y, y, z, z); + public readonly int4 gbrr => new(y, z, x, x); + public readonly int4 gbrg => new(y, z, x, y); + public readonly int4 gbrb => new(y, z, x, z); + public readonly int4 gbgr => new(y, z, y, x); + public readonly int4 gbgg => new(y, z, y, y); + public readonly int4 gbgb => new(y, z, y, z); + public readonly int4 gbbr => new(y, z, z, x); + public readonly int4 gbbg => new(y, z, z, y); + public readonly int4 gbbb => new(y, z, z, z); + public readonly int4 brrr => new(z, x, x, x); + public readonly int4 brrg => new(z, x, x, y); + public readonly int4 brrb => new(z, x, x, z); + public readonly int4 brgr => new(z, x, y, x); + public readonly int4 brgg => new(z, x, y, y); + public readonly int4 brgb => new(z, x, y, z); + public readonly int4 brbr => new(z, x, z, x); + public readonly int4 brbg => new(z, x, z, y); + public readonly int4 brbb => new(z, x, z, z); + public readonly int4 bgrr => new(z, y, x, x); + public readonly int4 bgrg => new(z, y, x, y); + public readonly int4 bgrb => new(z, y, x, z); + public readonly int4 bggr => new(z, y, y, x); + public readonly int4 bggg => new(z, y, y, y); + public readonly int4 bggb => new(z, y, y, z); + public readonly int4 bgbr => new(z, y, z, x); + public readonly int4 bgbg => new(z, y, z, y); + public readonly int4 bgbb => new(z, y, z, z); + public readonly int4 bbrr => new(z, z, x, x); + public readonly int4 bbrg => new(z, z, x, y); + public readonly int4 bbrb => new(z, z, x, z); + public readonly int4 bbgr => new(z, z, y, x); + public readonly int4 bbgg => new(z, z, y, y); + public readonly int4 bbgb => new(z, z, y, z); + public readonly int4 bbbr => new(z, z, z, x); + public readonly int4 bbbg => new(z, z, z, y); + public readonly int4 bbbb => new(z, z, z, z); +} + +public partial struct int4 +{ + public int4(int x, int y, int2 zw) : this(x, y, zw.x, zw.y) { } + public int4(int x, int2 yz, int w) : this(x, yz.x, yz.y, w) { } + public int4(int x, int3 yzw) : this(x, yzw.x, yzw.y, yzw.z) { } + public int4(int2 xy, int z, int w) : this(xy.x, xy.y, z, w) { } + public int4(int2 xy, int2 zw) : this(xy.x, xy.y, zw.x, zw.y) { } + public int4(int3 xyz, int w) : this(xyz.x, xyz.y, xyz.z, w) { } + public int4(int4 xyzw) : this(xyzw.x, xyzw.y, xyzw.z, xyzw.w) { } + public int r { readonly get => x; set => x = value; } + public int g { readonly get => y; set => y = value; } + public int b { readonly get => z; set => z = value; } + public int a { readonly get => w; set => w = value; } + public readonly int2 xx => new(x, x); + public int2 xy { readonly get => new(x, y); set { x = value.x; y = value.y; } } + public int2 xz { readonly get => new(x, z); set { x = value.x; z = value.y; } } + public int2 xw { readonly get => new(x, w); set { x = value.x; w = value.y; } } + public int2 yx { readonly get => new(y, x); set { y = value.x; x = value.y; } } + public readonly int2 yy => new(y, y); + public int2 yz { readonly get => new(y, z); set { y = value.x; z = value.y; } } + public int2 yw { readonly get => new(y, w); set { y = value.x; w = value.y; } } + public int2 zx { readonly get => new(z, x); set { z = value.x; x = value.y; } } + public int2 zy { readonly get => new(z, y); set { z = value.x; y = value.y; } } + public readonly int2 zz => new(z, z); + public int2 zw { readonly get => new(z, w); set { z = value.x; w = value.y; } } + public int2 wx { readonly get => new(w, x); set { w = value.x; x = value.y; } } + public int2 wy { readonly get => new(w, y); set { w = value.x; y = value.y; } } + public int2 wz { readonly get => new(w, z); set { w = value.x; z = value.y; } } + public readonly int2 ww => new(w, w); + public readonly int3 xxx => new(x, x, x); + public readonly int3 xxy => new(x, x, y); + public readonly int3 xxz => new(x, x, z); + public readonly int3 xxw => new(x, x, w); + public readonly int3 xyx => new(x, y, x); + public readonly int3 xyy => new(x, y, y); + public int3 xyz { readonly get => new(x, y, z); set { x = value.x; y = value.y; z = value.z; } } + public int3 xyw { readonly get => new(x, y, w); set { x = value.x; y = value.y; w = value.z; } } + public readonly int3 xzx => new(x, z, x); + public int3 xzy { readonly get => new(x, z, y); set { x = value.x; z = value.y; y = value.z; } } + public readonly int3 xzz => new(x, z, z); + public int3 xzw { readonly get => new(x, z, w); set { x = value.x; z = value.y; w = value.z; } } + public readonly int3 xwx => new(x, w, x); + public int3 xwy { readonly get => new(x, w, y); set { x = value.x; w = value.y; y = value.z; } } + public int3 xwz { readonly get => new(x, w, z); set { x = value.x; w = value.y; z = value.z; } } + public readonly int3 xww => new(x, w, w); + public readonly int3 yxx => new(y, x, x); + public readonly int3 yxy => new(y, x, y); + public int3 yxz { readonly get => new(y, x, z); set { y = value.x; x = value.y; z = value.z; } } + public int3 yxw { readonly get => new(y, x, w); set { y = value.x; x = value.y; w = value.z; } } + public readonly int3 yyx => new(y, y, x); + public readonly int3 yyy => new(y, y, y); + public readonly int3 yyz => new(y, y, z); + public readonly int3 yyw => new(y, y, w); + public int3 yzx { readonly get => new(y, z, x); set { y = value.x; z = value.y; x = value.z; } } + public readonly int3 yzy => new(y, z, y); + public readonly int3 yzz => new(y, z, z); + public int3 yzw { readonly get => new(y, z, w); set { y = value.x; z = value.y; w = value.z; } } + public int3 ywx { readonly get => new(y, w, x); set { y = value.x; w = value.y; x = value.z; } } + public readonly int3 ywy => new(y, w, y); + public int3 ywz { readonly get => new(y, w, z); set { y = value.x; w = value.y; z = value.z; } } + public readonly int3 yww => new(y, w, w); + public readonly int3 zxx => new(z, x, x); + public int3 zxy { readonly get => new(z, x, y); set { z = value.x; x = value.y; y = value.z; } } + public readonly int3 zxz => new(z, x, z); + public int3 zxw { readonly get => new(z, x, w); set { z = value.x; x = value.y; w = value.z; } } + public int3 zyx { readonly get => new(z, y, x); set { z = value.x; y = value.y; x = value.z; } } + public readonly int3 zyy => new(z, y, y); + public readonly int3 zyz => new(z, y, z); + public int3 zyw { readonly get => new(z, y, w); set { z = value.x; y = value.y; w = value.z; } } + public readonly int3 zzx => new(z, z, x); + public readonly int3 zzy => new(z, z, y); + public readonly int3 zzz => new(z, z, z); + public readonly int3 zzw => new(z, z, w); + public int3 zwx { readonly get => new(z, w, x); set { z = value.x; w = value.y; x = value.z; } } + public int3 zwy { readonly get => new(z, w, y); set { z = value.x; w = value.y; y = value.z; } } + public readonly int3 zwz => new(z, w, z); + public readonly int3 zww => new(z, w, w); + public readonly int3 wxx => new(w, x, x); + public int3 wxy { readonly get => new(w, x, y); set { w = value.x; x = value.y; y = value.z; } } + public int3 wxz { readonly get => new(w, x, z); set { w = value.x; x = value.y; z = value.z; } } + public readonly int3 wxw => new(w, x, w); + public int3 wyx { readonly get => new(w, y, x); set { w = value.x; y = value.y; x = value.z; } } + public readonly int3 wyy => new(w, y, y); + public int3 wyz { readonly get => new(w, y, z); set { w = value.x; y = value.y; z = value.z; } } + public readonly int3 wyw => new(w, y, w); + public int3 wzx { readonly get => new(w, z, x); set { w = value.x; z = value.y; x = value.z; } } + public int3 wzy { readonly get => new(w, z, y); set { w = value.x; z = value.y; y = value.z; } } + public readonly int3 wzz => new(w, z, z); + public readonly int3 wzw => new(w, z, w); + public readonly int3 wwx => new(w, w, x); + public readonly int3 wwy => new(w, w, y); + public readonly int3 wwz => new(w, w, z); + public readonly int3 www => new(w, w, w); + public readonly int4 xxxx => new(x, x, x, x); + public readonly int4 xxxy => new(x, x, x, y); + public readonly int4 xxxz => new(x, x, x, z); + public readonly int4 xxxw => new(x, x, x, w); + public readonly int4 xxyx => new(x, x, y, x); + public readonly int4 xxyy => new(x, x, y, y); + public readonly int4 xxyz => new(x, x, y, z); + public readonly int4 xxyw => new(x, x, y, w); + public readonly int4 xxzx => new(x, x, z, x); + public readonly int4 xxzy => new(x, x, z, y); + public readonly int4 xxzz => new(x, x, z, z); + public readonly int4 xxzw => new(x, x, z, w); + public readonly int4 xxwx => new(x, x, w, x); + public readonly int4 xxwy => new(x, x, w, y); + public readonly int4 xxwz => new(x, x, w, z); + public readonly int4 xxww => new(x, x, w, w); + public readonly int4 xyxx => new(x, y, x, x); + public readonly int4 xyxy => new(x, y, x, y); + public readonly int4 xyxz => new(x, y, x, z); + public readonly int4 xyxw => new(x, y, x, w); + public readonly int4 xyyx => new(x, y, y, x); + public readonly int4 xyyy => new(x, y, y, y); + public readonly int4 xyyz => new(x, y, y, z); + public readonly int4 xyyw => new(x, y, y, w); + public readonly int4 xyzx => new(x, y, z, x); + public readonly int4 xyzy => new(x, y, z, y); + public readonly int4 xyzz => new(x, y, z, z); + public int4 xyzw { readonly get => new(x, y, z, w); set { x = value.x; y = value.y; z = value.z; w = value.w; } } + public readonly int4 xywx => new(x, y, w, x); + public readonly int4 xywy => new(x, y, w, y); + public int4 xywz { readonly get => new(x, y, w, z); set { x = value.x; y = value.y; w = value.z; z = value.w; } } + public readonly int4 xyww => new(x, y, w, w); + public readonly int4 xzxx => new(x, z, x, x); + public readonly int4 xzxy => new(x, z, x, y); + public readonly int4 xzxz => new(x, z, x, z); + public readonly int4 xzxw => new(x, z, x, w); + public readonly int4 xzyx => new(x, z, y, x); + public readonly int4 xzyy => new(x, z, y, y); + public readonly int4 xzyz => new(x, z, y, z); + public int4 xzyw { readonly get => new(x, z, y, w); set { x = value.x; z = value.y; y = value.z; w = value.w; } } + public readonly int4 xzzx => new(x, z, z, x); + public readonly int4 xzzy => new(x, z, z, y); + public readonly int4 xzzz => new(x, z, z, z); + public readonly int4 xzzw => new(x, z, z, w); + public readonly int4 xzwx => new(x, z, w, x); + public int4 xzwy { readonly get => new(x, z, w, y); set { x = value.x; z = value.y; w = value.z; y = value.w; } } + public readonly int4 xzwz => new(x, z, w, z); + public readonly int4 xzww => new(x, z, w, w); + public readonly int4 xwxx => new(x, w, x, x); + public readonly int4 xwxy => new(x, w, x, y); + public readonly int4 xwxz => new(x, w, x, z); + public readonly int4 xwxw => new(x, w, x, w); + public readonly int4 xwyx => new(x, w, y, x); + public readonly int4 xwyy => new(x, w, y, y); + public int4 xwyz { readonly get => new(x, w, y, z); set { x = value.x; w = value.y; y = value.z; z = value.w; } } + public readonly int4 xwyw => new(x, w, y, w); + public readonly int4 xwzx => new(x, w, z, x); + public int4 xwzy { readonly get => new(x, w, z, y); set { x = value.x; w = value.y; z = value.z; y = value.w; } } + public readonly int4 xwzz => new(x, w, z, z); + public readonly int4 xwzw => new(x, w, z, w); + public readonly int4 xwwx => new(x, w, w, x); + public readonly int4 xwwy => new(x, w, w, y); + public readonly int4 xwwz => new(x, w, w, z); + public readonly int4 xwww => new(x, w, w, w); + public readonly int4 yxxx => new(y, x, x, x); + public readonly int4 yxxy => new(y, x, x, y); + public readonly int4 yxxz => new(y, x, x, z); + public readonly int4 yxxw => new(y, x, x, w); + public readonly int4 yxyx => new(y, x, y, x); + public readonly int4 yxyy => new(y, x, y, y); + public readonly int4 yxyz => new(y, x, y, z); + public readonly int4 yxyw => new(y, x, y, w); + public readonly int4 yxzx => new(y, x, z, x); + public readonly int4 yxzy => new(y, x, z, y); + public readonly int4 yxzz => new(y, x, z, z); + public int4 yxzw { readonly get => new(y, x, z, w); set { y = value.x; x = value.y; z = value.z; w = value.w; } } + public readonly int4 yxwx => new(y, x, w, x); + public readonly int4 yxwy => new(y, x, w, y); + public int4 yxwz { readonly get => new(y, x, w, z); set { y = value.x; x = value.y; w = value.z; z = value.w; } } + public readonly int4 yxww => new(y, x, w, w); + public readonly int4 yyxx => new(y, y, x, x); + public readonly int4 yyxy => new(y, y, x, y); + public readonly int4 yyxz => new(y, y, x, z); + public readonly int4 yyxw => new(y, y, x, w); + public readonly int4 yyyx => new(y, y, y, x); + public readonly int4 yyyy => new(y, y, y, y); + public readonly int4 yyyz => new(y, y, y, z); + public readonly int4 yyyw => new(y, y, y, w); + public readonly int4 yyzx => new(y, y, z, x); + public readonly int4 yyzy => new(y, y, z, y); + public readonly int4 yyzz => new(y, y, z, z); + public readonly int4 yyzw => new(y, y, z, w); + public readonly int4 yywx => new(y, y, w, x); + public readonly int4 yywy => new(y, y, w, y); + public readonly int4 yywz => new(y, y, w, z); + public readonly int4 yyww => new(y, y, w, w); + public readonly int4 yzxx => new(y, z, x, x); + public readonly int4 yzxy => new(y, z, x, y); + public readonly int4 yzxz => new(y, z, x, z); + public int4 yzxw { readonly get => new(y, z, x, w); set { y = value.x; z = value.y; x = value.z; w = value.w; } } + public readonly int4 yzyx => new(y, z, y, x); + public readonly int4 yzyy => new(y, z, y, y); + public readonly int4 yzyz => new(y, z, y, z); + public readonly int4 yzyw => new(y, z, y, w); + public readonly int4 yzzx => new(y, z, z, x); + public readonly int4 yzzy => new(y, z, z, y); + public readonly int4 yzzz => new(y, z, z, z); + public readonly int4 yzzw => new(y, z, z, w); + public int4 yzwx { readonly get => new(y, z, w, x); set { y = value.x; z = value.y; w = value.z; x = value.w; } } + public readonly int4 yzwy => new(y, z, w, y); + public readonly int4 yzwz => new(y, z, w, z); + public readonly int4 yzww => new(y, z, w, w); + public readonly int4 ywxx => new(y, w, x, x); + public readonly int4 ywxy => new(y, w, x, y); + public int4 ywxz { readonly get => new(y, w, x, z); set { y = value.x; w = value.y; x = value.z; z = value.w; } } + public readonly int4 ywxw => new(y, w, x, w); + public readonly int4 ywyx => new(y, w, y, x); + public readonly int4 ywyy => new(y, w, y, y); + public readonly int4 ywyz => new(y, w, y, z); + public readonly int4 ywyw => new(y, w, y, w); + public int4 ywzx { readonly get => new(y, w, z, x); set { y = value.x; w = value.y; z = value.z; x = value.w; } } + public readonly int4 ywzy => new(y, w, z, y); + public readonly int4 ywzz => new(y, w, z, z); + public readonly int4 ywzw => new(y, w, z, w); + public readonly int4 ywwx => new(y, w, w, x); + public readonly int4 ywwy => new(y, w, w, y); + public readonly int4 ywwz => new(y, w, w, z); + public readonly int4 ywww => new(y, w, w, w); + public readonly int4 zxxx => new(z, x, x, x); + public readonly int4 zxxy => new(z, x, x, y); + public readonly int4 zxxz => new(z, x, x, z); + public readonly int4 zxxw => new(z, x, x, w); + public readonly int4 zxyx => new(z, x, y, x); + public readonly int4 zxyy => new(z, x, y, y); + public readonly int4 zxyz => new(z, x, y, z); + public int4 zxyw { readonly get => new(z, x, y, w); set { z = value.x; x = value.y; y = value.z; w = value.w; } } + public readonly int4 zxzx => new(z, x, z, x); + public readonly int4 zxzy => new(z, x, z, y); + public readonly int4 zxzz => new(z, x, z, z); + public readonly int4 zxzw => new(z, x, z, w); + public readonly int4 zxwx => new(z, x, w, x); + public int4 zxwy { readonly get => new(z, x, w, y); set { z = value.x; x = value.y; w = value.z; y = value.w; } } + public readonly int4 zxwz => new(z, x, w, z); + public readonly int4 zxww => new(z, x, w, w); + public readonly int4 zyxx => new(z, y, x, x); + public readonly int4 zyxy => new(z, y, x, y); + public readonly int4 zyxz => new(z, y, x, z); + public int4 zyxw { readonly get => new(z, y, x, w); set { z = value.x; y = value.y; x = value.z; w = value.w; } } + public readonly int4 zyyx => new(z, y, y, x); + public readonly int4 zyyy => new(z, y, y, y); + public readonly int4 zyyz => new(z, y, y, z); + public readonly int4 zyyw => new(z, y, y, w); + public readonly int4 zyzx => new(z, y, z, x); + public readonly int4 zyzy => new(z, y, z, y); + public readonly int4 zyzz => new(z, y, z, z); + public readonly int4 zyzw => new(z, y, z, w); + public int4 zywx { readonly get => new(z, y, w, x); set { z = value.x; y = value.y; w = value.z; x = value.w; } } + public readonly int4 zywy => new(z, y, w, y); + public readonly int4 zywz => new(z, y, w, z); + public readonly int4 zyww => new(z, y, w, w); + public readonly int4 zzxx => new(z, z, x, x); + public readonly int4 zzxy => new(z, z, x, y); + public readonly int4 zzxz => new(z, z, x, z); + public readonly int4 zzxw => new(z, z, x, w); + public readonly int4 zzyx => new(z, z, y, x); + public readonly int4 zzyy => new(z, z, y, y); + public readonly int4 zzyz => new(z, z, y, z); + public readonly int4 zzyw => new(z, z, y, w); + public readonly int4 zzzx => new(z, z, z, x); + public readonly int4 zzzy => new(z, z, z, y); + public readonly int4 zzzz => new(z, z, z, z); + public readonly int4 zzzw => new(z, z, z, w); + public readonly int4 zzwx => new(z, z, w, x); + public readonly int4 zzwy => new(z, z, w, y); + public readonly int4 zzwz => new(z, z, w, z); + public readonly int4 zzww => new(z, z, w, w); + public readonly int4 zwxx => new(z, w, x, x); + public int4 zwxy { readonly get => new(z, w, x, y); set { z = value.x; w = value.y; x = value.z; y = value.w; } } + public readonly int4 zwxz => new(z, w, x, z); + public readonly int4 zwxw => new(z, w, x, w); + public int4 zwyx { readonly get => new(z, w, y, x); set { z = value.x; w = value.y; y = value.z; x = value.w; } } + public readonly int4 zwyy => new(z, w, y, y); + public readonly int4 zwyz => new(z, w, y, z); + public readonly int4 zwyw => new(z, w, y, w); + public readonly int4 zwzx => new(z, w, z, x); + public readonly int4 zwzy => new(z, w, z, y); + public readonly int4 zwzz => new(z, w, z, z); + public readonly int4 zwzw => new(z, w, z, w); + public readonly int4 zwwx => new(z, w, w, x); + public readonly int4 zwwy => new(z, w, w, y); + public readonly int4 zwwz => new(z, w, w, z); + public readonly int4 zwww => new(z, w, w, w); + public readonly int4 wxxx => new(w, x, x, x); + public readonly int4 wxxy => new(w, x, x, y); + public readonly int4 wxxz => new(w, x, x, z); + public readonly int4 wxxw => new(w, x, x, w); + public readonly int4 wxyx => new(w, x, y, x); + public readonly int4 wxyy => new(w, x, y, y); + public int4 wxyz { readonly get => new(w, x, y, z); set { w = value.x; x = value.y; y = value.z; z = value.w; } } + public readonly int4 wxyw => new(w, x, y, w); + public readonly int4 wxzx => new(w, x, z, x); + public int4 wxzy { readonly get => new(w, x, z, y); set { w = value.x; x = value.y; z = value.z; y = value.w; } } + public readonly int4 wxzz => new(w, x, z, z); + public readonly int4 wxzw => new(w, x, z, w); + public readonly int4 wxwx => new(w, x, w, x); + public readonly int4 wxwy => new(w, x, w, y); + public readonly int4 wxwz => new(w, x, w, z); + public readonly int4 wxww => new(w, x, w, w); + public readonly int4 wyxx => new(w, y, x, x); + public readonly int4 wyxy => new(w, y, x, y); + public int4 wyxz { readonly get => new(w, y, x, z); set { w = value.x; y = value.y; x = value.z; z = value.w; } } + public readonly int4 wyxw => new(w, y, x, w); + public readonly int4 wyyx => new(w, y, y, x); + public readonly int4 wyyy => new(w, y, y, y); + public readonly int4 wyyz => new(w, y, y, z); + public readonly int4 wyyw => new(w, y, y, w); + public int4 wyzx { readonly get => new(w, y, z, x); set { w = value.x; y = value.y; z = value.z; x = value.w; } } + public readonly int4 wyzy => new(w, y, z, y); + public readonly int4 wyzz => new(w, y, z, z); + public readonly int4 wyzw => new(w, y, z, w); + public readonly int4 wywx => new(w, y, w, x); + public readonly int4 wywy => new(w, y, w, y); + public readonly int4 wywz => new(w, y, w, z); + public readonly int4 wyww => new(w, y, w, w); + public readonly int4 wzxx => new(w, z, x, x); + public int4 wzxy { readonly get => new(w, z, x, y); set { w = value.x; z = value.y; x = value.z; y = value.w; } } + public readonly int4 wzxz => new(w, z, x, z); + public readonly int4 wzxw => new(w, z, x, w); + public int4 wzyx { readonly get => new(w, z, y, x); set { w = value.x; z = value.y; y = value.z; x = value.w; } } + public readonly int4 wzyy => new(w, z, y, y); + public readonly int4 wzyz => new(w, z, y, z); + public readonly int4 wzyw => new(w, z, y, w); + public readonly int4 wzzx => new(w, z, z, x); + public readonly int4 wzzy => new(w, z, z, y); + public readonly int4 wzzz => new(w, z, z, z); + public readonly int4 wzzw => new(w, z, z, w); + public readonly int4 wzwx => new(w, z, w, x); + public readonly int4 wzwy => new(w, z, w, y); + public readonly int4 wzwz => new(w, z, w, z); + public readonly int4 wzww => new(w, z, w, w); + public readonly int4 wwxx => new(w, w, x, x); + public readonly int4 wwxy => new(w, w, x, y); + public readonly int4 wwxz => new(w, w, x, z); + public readonly int4 wwxw => new(w, w, x, w); + public readonly int4 wwyx => new(w, w, y, x); + public readonly int4 wwyy => new(w, w, y, y); + public readonly int4 wwyz => new(w, w, y, z); + public readonly int4 wwyw => new(w, w, y, w); + public readonly int4 wwzx => new(w, w, z, x); + public readonly int4 wwzy => new(w, w, z, y); + public readonly int4 wwzz => new(w, w, z, z); + public readonly int4 wwzw => new(w, w, z, w); + public readonly int4 wwwx => new(w, w, w, x); + public readonly int4 wwwy => new(w, w, w, y); + public readonly int4 wwwz => new(w, w, w, z); + public readonly int4 wwww => new(w, w, w, w); + public readonly int2 rr => new(x, x); + public int2 rg { readonly get => new(x, y); set { x = value.x; y = value.y; } } + public int2 rb { readonly get => new(x, z); set { x = value.x; z = value.y; } } + public int2 ra { readonly get => new(x, w); set { x = value.x; w = value.y; } } + public int2 gr { readonly get => new(y, x); set { y = value.x; x = value.y; } } + public readonly int2 gg => new(y, y); + public int2 gb { readonly get => new(y, z); set { y = value.x; z = value.y; } } + public int2 ga { readonly get => new(y, w); set { y = value.x; w = value.y; } } + public int2 br { readonly get => new(z, x); set { z = value.x; x = value.y; } } + public int2 bg { readonly get => new(z, y); set { z = value.x; y = value.y; } } + public readonly int2 bb => new(z, z); + public int2 ba { readonly get => new(z, w); set { z = value.x; w = value.y; } } + public int2 ar { readonly get => new(w, x); set { w = value.x; x = value.y; } } + public int2 ag { readonly get => new(w, y); set { w = value.x; y = value.y; } } + public int2 ab { readonly get => new(w, z); set { w = value.x; z = value.y; } } + public readonly int2 aa => new(w, w); + public readonly int3 rrr => new(x, x, x); + public readonly int3 rrg => new(x, x, y); + public readonly int3 rrb => new(x, x, z); + public readonly int3 rra => new(x, x, w); + public readonly int3 rgr => new(x, y, x); + public readonly int3 rgg => new(x, y, y); + public int3 rgb { readonly get => new(x, y, z); set { x = value.x; y = value.y; z = value.z; } } + public int3 rga { readonly get => new(x, y, w); set { x = value.x; y = value.y; w = value.z; } } + public readonly int3 rbr => new(x, z, x); + public int3 rbg { readonly get => new(x, z, y); set { x = value.x; z = value.y; y = value.z; } } + public readonly int3 rbb => new(x, z, z); + public int3 rba { readonly get => new(x, z, w); set { x = value.x; z = value.y; w = value.z; } } + public readonly int3 rar => new(x, w, x); + public int3 rag { readonly get => new(x, w, y); set { x = value.x; w = value.y; y = value.z; } } + public int3 rab { readonly get => new(x, w, z); set { x = value.x; w = value.y; z = value.z; } } + public readonly int3 raa => new(x, w, w); + public readonly int3 grr => new(y, x, x); + public readonly int3 grg => new(y, x, y); + public int3 grb { readonly get => new(y, x, z); set { y = value.x; x = value.y; z = value.z; } } + public int3 gra { readonly get => new(y, x, w); set { y = value.x; x = value.y; w = value.z; } } + public readonly int3 ggr => new(y, y, x); + public readonly int3 ggg => new(y, y, y); + public readonly int3 ggb => new(y, y, z); + public readonly int3 gga => new(y, y, w); + public int3 gbr { readonly get => new(y, z, x); set { y = value.x; z = value.y; x = value.z; } } + public readonly int3 gbg => new(y, z, y); + public readonly int3 gbb => new(y, z, z); + public int3 gba { readonly get => new(y, z, w); set { y = value.x; z = value.y; w = value.z; } } + public int3 gar { readonly get => new(y, w, x); set { y = value.x; w = value.y; x = value.z; } } + public readonly int3 gag => new(y, w, y); + public int3 gab { readonly get => new(y, w, z); set { y = value.x; w = value.y; z = value.z; } } + public readonly int3 gaa => new(y, w, w); + public readonly int3 brr => new(z, x, x); + public int3 brg { readonly get => new(z, x, y); set { z = value.x; x = value.y; y = value.z; } } + public readonly int3 brb => new(z, x, z); + public int3 bra { readonly get => new(z, x, w); set { z = value.x; x = value.y; w = value.z; } } + public int3 bgr { readonly get => new(z, y, x); set { z = value.x; y = value.y; x = value.z; } } + public readonly int3 bgg => new(z, y, y); + public readonly int3 bgb => new(z, y, z); + public int3 bga { readonly get => new(z, y, w); set { z = value.x; y = value.y; w = value.z; } } + public readonly int3 bbr => new(z, z, x); + public readonly int3 bbg => new(z, z, y); + public readonly int3 bbb => new(z, z, z); + public readonly int3 bba => new(z, z, w); + public int3 bar { readonly get => new(z, w, x); set { z = value.x; w = value.y; x = value.z; } } + public int3 bag { readonly get => new(z, w, y); set { z = value.x; w = value.y; y = value.z; } } + public readonly int3 bab => new(z, w, z); + public readonly int3 baa => new(z, w, w); + public readonly int3 arr => new(w, x, x); + public int3 arg { readonly get => new(w, x, y); set { w = value.x; x = value.y; y = value.z; } } + public int3 arb { readonly get => new(w, x, z); set { w = value.x; x = value.y; z = value.z; } } + public readonly int3 ara => new(w, x, w); + public int3 agr { readonly get => new(w, y, x); set { w = value.x; y = value.y; x = value.z; } } + public readonly int3 agg => new(w, y, y); + public int3 agb { readonly get => new(w, y, z); set { w = value.x; y = value.y; z = value.z; } } + public readonly int3 aga => new(w, y, w); + public int3 abr { readonly get => new(w, z, x); set { w = value.x; z = value.y; x = value.z; } } + public int3 abg { readonly get => new(w, z, y); set { w = value.x; z = value.y; y = value.z; } } + public readonly int3 abb => new(w, z, z); + public readonly int3 aba => new(w, z, w); + public readonly int3 aar => new(w, w, x); + public readonly int3 aag => new(w, w, y); + public readonly int3 aab => new(w, w, z); + public readonly int3 aaa => new(w, w, w); + public readonly int4 rrrr => new(x, x, x, x); + public readonly int4 rrrg => new(x, x, x, y); + public readonly int4 rrrb => new(x, x, x, z); + public readonly int4 rrra => new(x, x, x, w); + public readonly int4 rrgr => new(x, x, y, x); + public readonly int4 rrgg => new(x, x, y, y); + public readonly int4 rrgb => new(x, x, y, z); + public readonly int4 rrga => new(x, x, y, w); + public readonly int4 rrbr => new(x, x, z, x); + public readonly int4 rrbg => new(x, x, z, y); + public readonly int4 rrbb => new(x, x, z, z); + public readonly int4 rrba => new(x, x, z, w); + public readonly int4 rrar => new(x, x, w, x); + public readonly int4 rrag => new(x, x, w, y); + public readonly int4 rrab => new(x, x, w, z); + public readonly int4 rraa => new(x, x, w, w); + public readonly int4 rgrr => new(x, y, x, x); + public readonly int4 rgrg => new(x, y, x, y); + public readonly int4 rgrb => new(x, y, x, z); + public readonly int4 rgra => new(x, y, x, w); + public readonly int4 rggr => new(x, y, y, x); + public readonly int4 rggg => new(x, y, y, y); + public readonly int4 rggb => new(x, y, y, z); + public readonly int4 rgga => new(x, y, y, w); + public readonly int4 rgbr => new(x, y, z, x); + public readonly int4 rgbg => new(x, y, z, y); + public readonly int4 rgbb => new(x, y, z, z); + public int4 rgba { readonly get => new(x, y, z, w); set { x = value.x; y = value.y; z = value.z; w = value.w; } } + public readonly int4 rgar => new(x, y, w, x); + public readonly int4 rgag => new(x, y, w, y); + public int4 rgab { readonly get => new(x, y, w, z); set { x = value.x; y = value.y; w = value.z; z = value.w; } } + public readonly int4 rgaa => new(x, y, w, w); + public readonly int4 rbrr => new(x, z, x, x); + public readonly int4 rbrg => new(x, z, x, y); + public readonly int4 rbrb => new(x, z, x, z); + public readonly int4 rbra => new(x, z, x, w); + public readonly int4 rbgr => new(x, z, y, x); + public readonly int4 rbgg => new(x, z, y, y); + public readonly int4 rbgb => new(x, z, y, z); + public int4 rbga { readonly get => new(x, z, y, w); set { x = value.x; z = value.y; y = value.z; w = value.w; } } + public readonly int4 rbbr => new(x, z, z, x); + public readonly int4 rbbg => new(x, z, z, y); + public readonly int4 rbbb => new(x, z, z, z); + public readonly int4 rbba => new(x, z, z, w); + public readonly int4 rbar => new(x, z, w, x); + public int4 rbag { readonly get => new(x, z, w, y); set { x = value.x; z = value.y; w = value.z; y = value.w; } } + public readonly int4 rbab => new(x, z, w, z); + public readonly int4 rbaa => new(x, z, w, w); + public readonly int4 rarr => new(x, w, x, x); + public readonly int4 rarg => new(x, w, x, y); + public readonly int4 rarb => new(x, w, x, z); + public readonly int4 rara => new(x, w, x, w); + public readonly int4 ragr => new(x, w, y, x); + public readonly int4 ragg => new(x, w, y, y); + public int4 ragb { readonly get => new(x, w, y, z); set { x = value.x; w = value.y; y = value.z; z = value.w; } } + public readonly int4 raga => new(x, w, y, w); + public readonly int4 rabr => new(x, w, z, x); + public int4 rabg { readonly get => new(x, w, z, y); set { x = value.x; w = value.y; z = value.z; y = value.w; } } + public readonly int4 rabb => new(x, w, z, z); + public readonly int4 raba => new(x, w, z, w); + public readonly int4 raar => new(x, w, w, x); + public readonly int4 raag => new(x, w, w, y); + public readonly int4 raab => new(x, w, w, z); + public readonly int4 raaa => new(x, w, w, w); + public readonly int4 grrr => new(y, x, x, x); + public readonly int4 grrg => new(y, x, x, y); + public readonly int4 grrb => new(y, x, x, z); + public readonly int4 grra => new(y, x, x, w); + public readonly int4 grgr => new(y, x, y, x); + public readonly int4 grgg => new(y, x, y, y); + public readonly int4 grgb => new(y, x, y, z); + public readonly int4 grga => new(y, x, y, w); + public readonly int4 grbr => new(y, x, z, x); + public readonly int4 grbg => new(y, x, z, y); + public readonly int4 grbb => new(y, x, z, z); + public int4 grba { readonly get => new(y, x, z, w); set { y = value.x; x = value.y; z = value.z; w = value.w; } } + public readonly int4 grar => new(y, x, w, x); + public readonly int4 grag => new(y, x, w, y); + public int4 grab { readonly get => new(y, x, w, z); set { y = value.x; x = value.y; w = value.z; z = value.w; } } + public readonly int4 graa => new(y, x, w, w); + public readonly int4 ggrr => new(y, y, x, x); + public readonly int4 ggrg => new(y, y, x, y); + public readonly int4 ggrb => new(y, y, x, z); + public readonly int4 ggra => new(y, y, x, w); + public readonly int4 gggr => new(y, y, y, x); + public readonly int4 gggg => new(y, y, y, y); + public readonly int4 gggb => new(y, y, y, z); + public readonly int4 ggga => new(y, y, y, w); + public readonly int4 ggbr => new(y, y, z, x); + public readonly int4 ggbg => new(y, y, z, y); + public readonly int4 ggbb => new(y, y, z, z); + public readonly int4 ggba => new(y, y, z, w); + public readonly int4 ggar => new(y, y, w, x); + public readonly int4 ggag => new(y, y, w, y); + public readonly int4 ggab => new(y, y, w, z); + public readonly int4 ggaa => new(y, y, w, w); + public readonly int4 gbrr => new(y, z, x, x); + public readonly int4 gbrg => new(y, z, x, y); + public readonly int4 gbrb => new(y, z, x, z); + public int4 gbra { readonly get => new(y, z, x, w); set { y = value.x; z = value.y; x = value.z; w = value.w; } } + public readonly int4 gbgr => new(y, z, y, x); + public readonly int4 gbgg => new(y, z, y, y); + public readonly int4 gbgb => new(y, z, y, z); + public readonly int4 gbga => new(y, z, y, w); + public readonly int4 gbbr => new(y, z, z, x); + public readonly int4 gbbg => new(y, z, z, y); + public readonly int4 gbbb => new(y, z, z, z); + public readonly int4 gbba => new(y, z, z, w); + public int4 gbar { readonly get => new(y, z, w, x); set { y = value.x; z = value.y; w = value.z; x = value.w; } } + public readonly int4 gbag => new(y, z, w, y); + public readonly int4 gbab => new(y, z, w, z); + public readonly int4 gbaa => new(y, z, w, w); + public readonly int4 garr => new(y, w, x, x); + public readonly int4 garg => new(y, w, x, y); + public int4 garb { readonly get => new(y, w, x, z); set { y = value.x; w = value.y; x = value.z; z = value.w; } } + public readonly int4 gara => new(y, w, x, w); + public readonly int4 gagr => new(y, w, y, x); + public readonly int4 gagg => new(y, w, y, y); + public readonly int4 gagb => new(y, w, y, z); + public readonly int4 gaga => new(y, w, y, w); + public int4 gabr { readonly get => new(y, w, z, x); set { y = value.x; w = value.y; z = value.z; x = value.w; } } + public readonly int4 gabg => new(y, w, z, y); + public readonly int4 gabb => new(y, w, z, z); + public readonly int4 gaba => new(y, w, z, w); + public readonly int4 gaar => new(y, w, w, x); + public readonly int4 gaag => new(y, w, w, y); + public readonly int4 gaab => new(y, w, w, z); + public readonly int4 gaaa => new(y, w, w, w); + public readonly int4 brrr => new(z, x, x, x); + public readonly int4 brrg => new(z, x, x, y); + public readonly int4 brrb => new(z, x, x, z); + public readonly int4 brra => new(z, x, x, w); + public readonly int4 brgr => new(z, x, y, x); + public readonly int4 brgg => new(z, x, y, y); + public readonly int4 brgb => new(z, x, y, z); + public int4 brga { readonly get => new(z, x, y, w); set { z = value.x; x = value.y; y = value.z; w = value.w; } } + public readonly int4 brbr => new(z, x, z, x); + public readonly int4 brbg => new(z, x, z, y); + public readonly int4 brbb => new(z, x, z, z); + public readonly int4 brba => new(z, x, z, w); + public readonly int4 brar => new(z, x, w, x); + public int4 brag { readonly get => new(z, x, w, y); set { z = value.x; x = value.y; w = value.z; y = value.w; } } + public readonly int4 brab => new(z, x, w, z); + public readonly int4 braa => new(z, x, w, w); + public readonly int4 bgrr => new(z, y, x, x); + public readonly int4 bgrg => new(z, y, x, y); + public readonly int4 bgrb => new(z, y, x, z); + public int4 bgra { readonly get => new(z, y, x, w); set { z = value.x; y = value.y; x = value.z; w = value.w; } } + public readonly int4 bggr => new(z, y, y, x); + public readonly int4 bggg => new(z, y, y, y); + public readonly int4 bggb => new(z, y, y, z); + public readonly int4 bgga => new(z, y, y, w); + public readonly int4 bgbr => new(z, y, z, x); + public readonly int4 bgbg => new(z, y, z, y); + public readonly int4 bgbb => new(z, y, z, z); + public readonly int4 bgba => new(z, y, z, w); + public int4 bgar { readonly get => new(z, y, w, x); set { z = value.x; y = value.y; w = value.z; x = value.w; } } + public readonly int4 bgag => new(z, y, w, y); + public readonly int4 bgab => new(z, y, w, z); + public readonly int4 bgaa => new(z, y, w, w); + public readonly int4 bbrr => new(z, z, x, x); + public readonly int4 bbrg => new(z, z, x, y); + public readonly int4 bbrb => new(z, z, x, z); + public readonly int4 bbra => new(z, z, x, w); + public readonly int4 bbgr => new(z, z, y, x); + public readonly int4 bbgg => new(z, z, y, y); + public readonly int4 bbgb => new(z, z, y, z); + public readonly int4 bbga => new(z, z, y, w); + public readonly int4 bbbr => new(z, z, z, x); + public readonly int4 bbbg => new(z, z, z, y); + public readonly int4 bbbb => new(z, z, z, z); + public readonly int4 bbba => new(z, z, z, w); + public readonly int4 bbar => new(z, z, w, x); + public readonly int4 bbag => new(z, z, w, y); + public readonly int4 bbab => new(z, z, w, z); + public readonly int4 bbaa => new(z, z, w, w); + public readonly int4 barr => new(z, w, x, x); + public int4 barg { readonly get => new(z, w, x, y); set { z = value.x; w = value.y; x = value.z; y = value.w; } } + public readonly int4 barb => new(z, w, x, z); + public readonly int4 bara => new(z, w, x, w); + public int4 bagr { readonly get => new(z, w, y, x); set { z = value.x; w = value.y; y = value.z; x = value.w; } } + public readonly int4 bagg => new(z, w, y, y); + public readonly int4 bagb => new(z, w, y, z); + public readonly int4 baga => new(z, w, y, w); + public readonly int4 babr => new(z, w, z, x); + public readonly int4 babg => new(z, w, z, y); + public readonly int4 babb => new(z, w, z, z); + public readonly int4 baba => new(z, w, z, w); + public readonly int4 baar => new(z, w, w, x); + public readonly int4 baag => new(z, w, w, y); + public readonly int4 baab => new(z, w, w, z); + public readonly int4 baaa => new(z, w, w, w); + public readonly int4 arrr => new(w, x, x, x); + public readonly int4 arrg => new(w, x, x, y); + public readonly int4 arrb => new(w, x, x, z); + public readonly int4 arra => new(w, x, x, w); + public readonly int4 argr => new(w, x, y, x); + public readonly int4 argg => new(w, x, y, y); + public int4 argb { readonly get => new(w, x, y, z); set { w = value.x; x = value.y; y = value.z; z = value.w; } } + public readonly int4 arga => new(w, x, y, w); + public readonly int4 arbr => new(w, x, z, x); + public int4 arbg { readonly get => new(w, x, z, y); set { w = value.x; x = value.y; z = value.z; y = value.w; } } + public readonly int4 arbb => new(w, x, z, z); + public readonly int4 arba => new(w, x, z, w); + public readonly int4 arar => new(w, x, w, x); + public readonly int4 arag => new(w, x, w, y); + public readonly int4 arab => new(w, x, w, z); + public readonly int4 araa => new(w, x, w, w); + public readonly int4 agrr => new(w, y, x, x); + public readonly int4 agrg => new(w, y, x, y); + public int4 agrb { readonly get => new(w, y, x, z); set { w = value.x; y = value.y; x = value.z; z = value.w; } } + public readonly int4 agra => new(w, y, x, w); + public readonly int4 aggr => new(w, y, y, x); + public readonly int4 aggg => new(w, y, y, y); + public readonly int4 aggb => new(w, y, y, z); + public readonly int4 agga => new(w, y, y, w); + public int4 agbr { readonly get => new(w, y, z, x); set { w = value.x; y = value.y; z = value.z; x = value.w; } } + public readonly int4 agbg => new(w, y, z, y); + public readonly int4 agbb => new(w, y, z, z); + public readonly int4 agba => new(w, y, z, w); + public readonly int4 agar => new(w, y, w, x); + public readonly int4 agag => new(w, y, w, y); + public readonly int4 agab => new(w, y, w, z); + public readonly int4 agaa => new(w, y, w, w); + public readonly int4 abrr => new(w, z, x, x); + public int4 abrg { readonly get => new(w, z, x, y); set { w = value.x; z = value.y; x = value.z; y = value.w; } } + public readonly int4 abrb => new(w, z, x, z); + public readonly int4 abra => new(w, z, x, w); + public int4 abgr { readonly get => new(w, z, y, x); set { w = value.x; z = value.y; y = value.z; x = value.w; } } + public readonly int4 abgg => new(w, z, y, y); + public readonly int4 abgb => new(w, z, y, z); + public readonly int4 abga => new(w, z, y, w); + public readonly int4 abbr => new(w, z, z, x); + public readonly int4 abbg => new(w, z, z, y); + public readonly int4 abbb => new(w, z, z, z); + public readonly int4 abba => new(w, z, z, w); + public readonly int4 abar => new(w, z, w, x); + public readonly int4 abag => new(w, z, w, y); + public readonly int4 abab => new(w, z, w, z); + public readonly int4 abaa => new(w, z, w, w); + public readonly int4 aarr => new(w, w, x, x); + public readonly int4 aarg => new(w, w, x, y); + public readonly int4 aarb => new(w, w, x, z); + public readonly int4 aara => new(w, w, x, w); + public readonly int4 aagr => new(w, w, y, x); + public readonly int4 aagg => new(w, w, y, y); + public readonly int4 aagb => new(w, w, y, z); + public readonly int4 aaga => new(w, w, y, w); + public readonly int4 aabr => new(w, w, z, x); + public readonly int4 aabg => new(w, w, z, y); + public readonly int4 aabb => new(w, w, z, z); + public readonly int4 aaba => new(w, w, z, w); + public readonly int4 aaar => new(w, w, w, x); + public readonly int4 aaag => new(w, w, w, y); + public readonly int4 aaab => new(w, w, w, z); + public readonly int4 aaaa => new(w, w, w, w); +} + +public partial struct bool2 +{ + public bool2(bool2 xy) : this(xy.x, xy.y) { } + public bool r { readonly get => x; set => x = value; } + public bool g { readonly get => y; set => y = value; } + public readonly bool2 xx => new(x, x); + public bool2 xy { readonly get => new(x, y); set { x = value.x; y = value.y; } } + public bool2 yx { readonly get => new(y, x); set { y = value.x; x = value.y; } } + public readonly bool2 yy => new(y, y); + public readonly bool3 xxx => new(x, x, x); + public readonly bool3 xxy => new(x, x, y); + public readonly bool3 xyx => new(x, y, x); + public readonly bool3 xyy => new(x, y, y); + public readonly bool3 yxx => new(y, x, x); + public readonly bool3 yxy => new(y, x, y); + public readonly bool3 yyx => new(y, y, x); + public readonly bool3 yyy => new(y, y, y); + public readonly bool4 xxxx => new(x, x, x, x); + public readonly bool4 xxxy => new(x, x, x, y); + public readonly bool4 xxyx => new(x, x, y, x); + public readonly bool4 xxyy => new(x, x, y, y); + public readonly bool4 xyxx => new(x, y, x, x); + public readonly bool4 xyxy => new(x, y, x, y); + public readonly bool4 xyyx => new(x, y, y, x); + public readonly bool4 xyyy => new(x, y, y, y); + public readonly bool4 yxxx => new(y, x, x, x); + public readonly bool4 yxxy => new(y, x, x, y); + public readonly bool4 yxyx => new(y, x, y, x); + public readonly bool4 yxyy => new(y, x, y, y); + public readonly bool4 yyxx => new(y, y, x, x); + public readonly bool4 yyxy => new(y, y, x, y); + public readonly bool4 yyyx => new(y, y, y, x); + public readonly bool4 yyyy => new(y, y, y, y); + public readonly bool2 rr => new(x, x); + public bool2 rg { readonly get => new(x, y); set { x = value.x; y = value.y; } } + public bool2 gr { readonly get => new(y, x); set { y = value.x; x = value.y; } } + public readonly bool2 gg => new(y, y); + public readonly bool3 rrr => new(x, x, x); + public readonly bool3 rrg => new(x, x, y); + public readonly bool3 rgr => new(x, y, x); + public readonly bool3 rgg => new(x, y, y); + public readonly bool3 grr => new(y, x, x); + public readonly bool3 grg => new(y, x, y); + public readonly bool3 ggr => new(y, y, x); + public readonly bool3 ggg => new(y, y, y); + public readonly bool4 rrrr => new(x, x, x, x); + public readonly bool4 rrrg => new(x, x, x, y); + public readonly bool4 rrgr => new(x, x, y, x); + public readonly bool4 rrgg => new(x, x, y, y); + public readonly bool4 rgrr => new(x, y, x, x); + public readonly bool4 rgrg => new(x, y, x, y); + public readonly bool4 rggr => new(x, y, y, x); + public readonly bool4 rggg => new(x, y, y, y); + public readonly bool4 grrr => new(y, x, x, x); + public readonly bool4 grrg => new(y, x, x, y); + public readonly bool4 grgr => new(y, x, y, x); + public readonly bool4 grgg => new(y, x, y, y); + public readonly bool4 ggrr => new(y, y, x, x); + public readonly bool4 ggrg => new(y, y, x, y); + public readonly bool4 gggr => new(y, y, y, x); + public readonly bool4 gggg => new(y, y, y, y); +} + +public partial struct bool3 +{ + public bool3(bool x, bool2 yz) : this(x, yz.x, yz.y) { } + public bool3(bool2 xy, bool z) : this(xy.x, xy.y, z) { } + public bool3(bool3 xyz) : this(xyz.x, xyz.y, xyz.z) { } + public bool r { readonly get => x; set => x = value; } + public bool g { readonly get => y; set => y = value; } + public bool b { readonly get => z; set => z = value; } + public readonly bool2 xx => new(x, x); + public bool2 xy { readonly get => new(x, y); set { x = value.x; y = value.y; } } + public bool2 xz { readonly get => new(x, z); set { x = value.x; z = value.y; } } + public bool2 yx { readonly get => new(y, x); set { y = value.x; x = value.y; } } + public readonly bool2 yy => new(y, y); + public bool2 yz { readonly get => new(y, z); set { y = value.x; z = value.y; } } + public bool2 zx { readonly get => new(z, x); set { z = value.x; x = value.y; } } + public bool2 zy { readonly get => new(z, y); set { z = value.x; y = value.y; } } + public readonly bool2 zz => new(z, z); + public readonly bool3 xxx => new(x, x, x); + public readonly bool3 xxy => new(x, x, y); + public readonly bool3 xxz => new(x, x, z); + public readonly bool3 xyx => new(x, y, x); + public readonly bool3 xyy => new(x, y, y); + public bool3 xyz { readonly get => new(x, y, z); set { x = value.x; y = value.y; z = value.z; } } + public readonly bool3 xzx => new(x, z, x); + public bool3 xzy { readonly get => new(x, z, y); set { x = value.x; z = value.y; y = value.z; } } + public readonly bool3 xzz => new(x, z, z); + public readonly bool3 yxx => new(y, x, x); + public readonly bool3 yxy => new(y, x, y); + public bool3 yxz { readonly get => new(y, x, z); set { y = value.x; x = value.y; z = value.z; } } + public readonly bool3 yyx => new(y, y, x); + public readonly bool3 yyy => new(y, y, y); + public readonly bool3 yyz => new(y, y, z); + public bool3 yzx { readonly get => new(y, z, x); set { y = value.x; z = value.y; x = value.z; } } + public readonly bool3 yzy => new(y, z, y); + public readonly bool3 yzz => new(y, z, z); + public readonly bool3 zxx => new(z, x, x); + public bool3 zxy { readonly get => new(z, x, y); set { z = value.x; x = value.y; y = value.z; } } + public readonly bool3 zxz => new(z, x, z); + public bool3 zyx { readonly get => new(z, y, x); set { z = value.x; y = value.y; x = value.z; } } + public readonly bool3 zyy => new(z, y, y); + public readonly bool3 zyz => new(z, y, z); + public readonly bool3 zzx => new(z, z, x); + public readonly bool3 zzy => new(z, z, y); + public readonly bool3 zzz => new(z, z, z); + public readonly bool4 xxxx => new(x, x, x, x); + public readonly bool4 xxxy => new(x, x, x, y); + public readonly bool4 xxxz => new(x, x, x, z); + public readonly bool4 xxyx => new(x, x, y, x); + public readonly bool4 xxyy => new(x, x, y, y); + public readonly bool4 xxyz => new(x, x, y, z); + public readonly bool4 xxzx => new(x, x, z, x); + public readonly bool4 xxzy => new(x, x, z, y); + public readonly bool4 xxzz => new(x, x, z, z); + public readonly bool4 xyxx => new(x, y, x, x); + public readonly bool4 xyxy => new(x, y, x, y); + public readonly bool4 xyxz => new(x, y, x, z); + public readonly bool4 xyyx => new(x, y, y, x); + public readonly bool4 xyyy => new(x, y, y, y); + public readonly bool4 xyyz => new(x, y, y, z); + public readonly bool4 xyzx => new(x, y, z, x); + public readonly bool4 xyzy => new(x, y, z, y); + public readonly bool4 xyzz => new(x, y, z, z); + public readonly bool4 xzxx => new(x, z, x, x); + public readonly bool4 xzxy => new(x, z, x, y); + public readonly bool4 xzxz => new(x, z, x, z); + public readonly bool4 xzyx => new(x, z, y, x); + public readonly bool4 xzyy => new(x, z, y, y); + public readonly bool4 xzyz => new(x, z, y, z); + public readonly bool4 xzzx => new(x, z, z, x); + public readonly bool4 xzzy => new(x, z, z, y); + public readonly bool4 xzzz => new(x, z, z, z); + public readonly bool4 yxxx => new(y, x, x, x); + public readonly bool4 yxxy => new(y, x, x, y); + public readonly bool4 yxxz => new(y, x, x, z); + public readonly bool4 yxyx => new(y, x, y, x); + public readonly bool4 yxyy => new(y, x, y, y); + public readonly bool4 yxyz => new(y, x, y, z); + public readonly bool4 yxzx => new(y, x, z, x); + public readonly bool4 yxzy => new(y, x, z, y); + public readonly bool4 yxzz => new(y, x, z, z); + public readonly bool4 yyxx => new(y, y, x, x); + public readonly bool4 yyxy => new(y, y, x, y); + public readonly bool4 yyxz => new(y, y, x, z); + public readonly bool4 yyyx => new(y, y, y, x); + public readonly bool4 yyyy => new(y, y, y, y); + public readonly bool4 yyyz => new(y, y, y, z); + public readonly bool4 yyzx => new(y, y, z, x); + public readonly bool4 yyzy => new(y, y, z, y); + public readonly bool4 yyzz => new(y, y, z, z); + public readonly bool4 yzxx => new(y, z, x, x); + public readonly bool4 yzxy => new(y, z, x, y); + public readonly bool4 yzxz => new(y, z, x, z); + public readonly bool4 yzyx => new(y, z, y, x); + public readonly bool4 yzyy => new(y, z, y, y); + public readonly bool4 yzyz => new(y, z, y, z); + public readonly bool4 yzzx => new(y, z, z, x); + public readonly bool4 yzzy => new(y, z, z, y); + public readonly bool4 yzzz => new(y, z, z, z); + public readonly bool4 zxxx => new(z, x, x, x); + public readonly bool4 zxxy => new(z, x, x, y); + public readonly bool4 zxxz => new(z, x, x, z); + public readonly bool4 zxyx => new(z, x, y, x); + public readonly bool4 zxyy => new(z, x, y, y); + public readonly bool4 zxyz => new(z, x, y, z); + public readonly bool4 zxzx => new(z, x, z, x); + public readonly bool4 zxzy => new(z, x, z, y); + public readonly bool4 zxzz => new(z, x, z, z); + public readonly bool4 zyxx => new(z, y, x, x); + public readonly bool4 zyxy => new(z, y, x, y); + public readonly bool4 zyxz => new(z, y, x, z); + public readonly bool4 zyyx => new(z, y, y, x); + public readonly bool4 zyyy => new(z, y, y, y); + public readonly bool4 zyyz => new(z, y, y, z); + public readonly bool4 zyzx => new(z, y, z, x); + public readonly bool4 zyzy => new(z, y, z, y); + public readonly bool4 zyzz => new(z, y, z, z); + public readonly bool4 zzxx => new(z, z, x, x); + public readonly bool4 zzxy => new(z, z, x, y); + public readonly bool4 zzxz => new(z, z, x, z); + public readonly bool4 zzyx => new(z, z, y, x); + public readonly bool4 zzyy => new(z, z, y, y); + public readonly bool4 zzyz => new(z, z, y, z); + public readonly bool4 zzzx => new(z, z, z, x); + public readonly bool4 zzzy => new(z, z, z, y); + public readonly bool4 zzzz => new(z, z, z, z); + public readonly bool2 rr => new(x, x); + public bool2 rg { readonly get => new(x, y); set { x = value.x; y = value.y; } } + public bool2 rb { readonly get => new(x, z); set { x = value.x; z = value.y; } } + public bool2 gr { readonly get => new(y, x); set { y = value.x; x = value.y; } } + public readonly bool2 gg => new(y, y); + public bool2 gb { readonly get => new(y, z); set { y = value.x; z = value.y; } } + public bool2 br { readonly get => new(z, x); set { z = value.x; x = value.y; } } + public bool2 bg { readonly get => new(z, y); set { z = value.x; y = value.y; } } + public readonly bool2 bb => new(z, z); + public readonly bool3 rrr => new(x, x, x); + public readonly bool3 rrg => new(x, x, y); + public readonly bool3 rrb => new(x, x, z); + public readonly bool3 rgr => new(x, y, x); + public readonly bool3 rgg => new(x, y, y); + public bool3 rgb { readonly get => new(x, y, z); set { x = value.x; y = value.y; z = value.z; } } + public readonly bool3 rbr => new(x, z, x); + public bool3 rbg { readonly get => new(x, z, y); set { x = value.x; z = value.y; y = value.z; } } + public readonly bool3 rbb => new(x, z, z); + public readonly bool3 grr => new(y, x, x); + public readonly bool3 grg => new(y, x, y); + public bool3 grb { readonly get => new(y, x, z); set { y = value.x; x = value.y; z = value.z; } } + public readonly bool3 ggr => new(y, y, x); + public readonly bool3 ggg => new(y, y, y); + public readonly bool3 ggb => new(y, y, z); + public bool3 gbr { readonly get => new(y, z, x); set { y = value.x; z = value.y; x = value.z; } } + public readonly bool3 gbg => new(y, z, y); + public readonly bool3 gbb => new(y, z, z); + public readonly bool3 brr => new(z, x, x); + public bool3 brg { readonly get => new(z, x, y); set { z = value.x; x = value.y; y = value.z; } } + public readonly bool3 brb => new(z, x, z); + public bool3 bgr { readonly get => new(z, y, x); set { z = value.x; y = value.y; x = value.z; } } + public readonly bool3 bgg => new(z, y, y); + public readonly bool3 bgb => new(z, y, z); + public readonly bool3 bbr => new(z, z, x); + public readonly bool3 bbg => new(z, z, y); + public readonly bool3 bbb => new(z, z, z); + public readonly bool4 rrrr => new(x, x, x, x); + public readonly bool4 rrrg => new(x, x, x, y); + public readonly bool4 rrrb => new(x, x, x, z); + public readonly bool4 rrgr => new(x, x, y, x); + public readonly bool4 rrgg => new(x, x, y, y); + public readonly bool4 rrgb => new(x, x, y, z); + public readonly bool4 rrbr => new(x, x, z, x); + public readonly bool4 rrbg => new(x, x, z, y); + public readonly bool4 rrbb => new(x, x, z, z); + public readonly bool4 rgrr => new(x, y, x, x); + public readonly bool4 rgrg => new(x, y, x, y); + public readonly bool4 rgrb => new(x, y, x, z); + public readonly bool4 rggr => new(x, y, y, x); + public readonly bool4 rggg => new(x, y, y, y); + public readonly bool4 rggb => new(x, y, y, z); + public readonly bool4 rgbr => new(x, y, z, x); + public readonly bool4 rgbg => new(x, y, z, y); + public readonly bool4 rgbb => new(x, y, z, z); + public readonly bool4 rbrr => new(x, z, x, x); + public readonly bool4 rbrg => new(x, z, x, y); + public readonly bool4 rbrb => new(x, z, x, z); + public readonly bool4 rbgr => new(x, z, y, x); + public readonly bool4 rbgg => new(x, z, y, y); + public readonly bool4 rbgb => new(x, z, y, z); + public readonly bool4 rbbr => new(x, z, z, x); + public readonly bool4 rbbg => new(x, z, z, y); + public readonly bool4 rbbb => new(x, z, z, z); + public readonly bool4 grrr => new(y, x, x, x); + public readonly bool4 grrg => new(y, x, x, y); + public readonly bool4 grrb => new(y, x, x, z); + public readonly bool4 grgr => new(y, x, y, x); + public readonly bool4 grgg => new(y, x, y, y); + public readonly bool4 grgb => new(y, x, y, z); + public readonly bool4 grbr => new(y, x, z, x); + public readonly bool4 grbg => new(y, x, z, y); + public readonly bool4 grbb => new(y, x, z, z); + public readonly bool4 ggrr => new(y, y, x, x); + public readonly bool4 ggrg => new(y, y, x, y); + public readonly bool4 ggrb => new(y, y, x, z); + public readonly bool4 gggr => new(y, y, y, x); + public readonly bool4 gggg => new(y, y, y, y); + public readonly bool4 gggb => new(y, y, y, z); + public readonly bool4 ggbr => new(y, y, z, x); + public readonly bool4 ggbg => new(y, y, z, y); + public readonly bool4 ggbb => new(y, y, z, z); + public readonly bool4 gbrr => new(y, z, x, x); + public readonly bool4 gbrg => new(y, z, x, y); + public readonly bool4 gbrb => new(y, z, x, z); + public readonly bool4 gbgr => new(y, z, y, x); + public readonly bool4 gbgg => new(y, z, y, y); + public readonly bool4 gbgb => new(y, z, y, z); + public readonly bool4 gbbr => new(y, z, z, x); + public readonly bool4 gbbg => new(y, z, z, y); + public readonly bool4 gbbb => new(y, z, z, z); + public readonly bool4 brrr => new(z, x, x, x); + public readonly bool4 brrg => new(z, x, x, y); + public readonly bool4 brrb => new(z, x, x, z); + public readonly bool4 brgr => new(z, x, y, x); + public readonly bool4 brgg => new(z, x, y, y); + public readonly bool4 brgb => new(z, x, y, z); + public readonly bool4 brbr => new(z, x, z, x); + public readonly bool4 brbg => new(z, x, z, y); + public readonly bool4 brbb => new(z, x, z, z); + public readonly bool4 bgrr => new(z, y, x, x); + public readonly bool4 bgrg => new(z, y, x, y); + public readonly bool4 bgrb => new(z, y, x, z); + public readonly bool4 bggr => new(z, y, y, x); + public readonly bool4 bggg => new(z, y, y, y); + public readonly bool4 bggb => new(z, y, y, z); + public readonly bool4 bgbr => new(z, y, z, x); + public readonly bool4 bgbg => new(z, y, z, y); + public readonly bool4 bgbb => new(z, y, z, z); + public readonly bool4 bbrr => new(z, z, x, x); + public readonly bool4 bbrg => new(z, z, x, y); + public readonly bool4 bbrb => new(z, z, x, z); + public readonly bool4 bbgr => new(z, z, y, x); + public readonly bool4 bbgg => new(z, z, y, y); + public readonly bool4 bbgb => new(z, z, y, z); + public readonly bool4 bbbr => new(z, z, z, x); + public readonly bool4 bbbg => new(z, z, z, y); + public readonly bool4 bbbb => new(z, z, z, z); +} + +public partial struct bool4 +{ + public bool4(bool x, bool y, bool2 zw) : this(x, y, zw.x, zw.y) { } + public bool4(bool x, bool2 yz, bool w) : this(x, yz.x, yz.y, w) { } + public bool4(bool x, bool3 yzw) : this(x, yzw.x, yzw.y, yzw.z) { } + public bool4(bool2 xy, bool z, bool w) : this(xy.x, xy.y, z, w) { } + public bool4(bool2 xy, bool2 zw) : this(xy.x, xy.y, zw.x, zw.y) { } + public bool4(bool3 xyz, bool w) : this(xyz.x, xyz.y, xyz.z, w) { } + public bool4(bool4 xyzw) : this(xyzw.x, xyzw.y, xyzw.z, xyzw.w) { } + public bool r { readonly get => x; set => x = value; } + public bool g { readonly get => y; set => y = value; } + public bool b { readonly get => z; set => z = value; } + public bool a { readonly get => w; set => w = value; } + public readonly bool2 xx => new(x, x); + public bool2 xy { readonly get => new(x, y); set { x = value.x; y = value.y; } } + public bool2 xz { readonly get => new(x, z); set { x = value.x; z = value.y; } } + public bool2 xw { readonly get => new(x, w); set { x = value.x; w = value.y; } } + public bool2 yx { readonly get => new(y, x); set { y = value.x; x = value.y; } } + public readonly bool2 yy => new(y, y); + public bool2 yz { readonly get => new(y, z); set { y = value.x; z = value.y; } } + public bool2 yw { readonly get => new(y, w); set { y = value.x; w = value.y; } } + public bool2 zx { readonly get => new(z, x); set { z = value.x; x = value.y; } } + public bool2 zy { readonly get => new(z, y); set { z = value.x; y = value.y; } } + public readonly bool2 zz => new(z, z); + public bool2 zw { readonly get => new(z, w); set { z = value.x; w = value.y; } } + public bool2 wx { readonly get => new(w, x); set { w = value.x; x = value.y; } } + public bool2 wy { readonly get => new(w, y); set { w = value.x; y = value.y; } } + public bool2 wz { readonly get => new(w, z); set { w = value.x; z = value.y; } } + public readonly bool2 ww => new(w, w); + public readonly bool3 xxx => new(x, x, x); + public readonly bool3 xxy => new(x, x, y); + public readonly bool3 xxz => new(x, x, z); + public readonly bool3 xxw => new(x, x, w); + public readonly bool3 xyx => new(x, y, x); + public readonly bool3 xyy => new(x, y, y); + public bool3 xyz { readonly get => new(x, y, z); set { x = value.x; y = value.y; z = value.z; } } + public bool3 xyw { readonly get => new(x, y, w); set { x = value.x; y = value.y; w = value.z; } } + public readonly bool3 xzx => new(x, z, x); + public bool3 xzy { readonly get => new(x, z, y); set { x = value.x; z = value.y; y = value.z; } } + public readonly bool3 xzz => new(x, z, z); + public bool3 xzw { readonly get => new(x, z, w); set { x = value.x; z = value.y; w = value.z; } } + public readonly bool3 xwx => new(x, w, x); + public bool3 xwy { readonly get => new(x, w, y); set { x = value.x; w = value.y; y = value.z; } } + public bool3 xwz { readonly get => new(x, w, z); set { x = value.x; w = value.y; z = value.z; } } + public readonly bool3 xww => new(x, w, w); + public readonly bool3 yxx => new(y, x, x); + public readonly bool3 yxy => new(y, x, y); + public bool3 yxz { readonly get => new(y, x, z); set { y = value.x; x = value.y; z = value.z; } } + public bool3 yxw { readonly get => new(y, x, w); set { y = value.x; x = value.y; w = value.z; } } + public readonly bool3 yyx => new(y, y, x); + public readonly bool3 yyy => new(y, y, y); + public readonly bool3 yyz => new(y, y, z); + public readonly bool3 yyw => new(y, y, w); + public bool3 yzx { readonly get => new(y, z, x); set { y = value.x; z = value.y; x = value.z; } } + public readonly bool3 yzy => new(y, z, y); + public readonly bool3 yzz => new(y, z, z); + public bool3 yzw { readonly get => new(y, z, w); set { y = value.x; z = value.y; w = value.z; } } + public bool3 ywx { readonly get => new(y, w, x); set { y = value.x; w = value.y; x = value.z; } } + public readonly bool3 ywy => new(y, w, y); + public bool3 ywz { readonly get => new(y, w, z); set { y = value.x; w = value.y; z = value.z; } } + public readonly bool3 yww => new(y, w, w); + public readonly bool3 zxx => new(z, x, x); + public bool3 zxy { readonly get => new(z, x, y); set { z = value.x; x = value.y; y = value.z; } } + public readonly bool3 zxz => new(z, x, z); + public bool3 zxw { readonly get => new(z, x, w); set { z = value.x; x = value.y; w = value.z; } } + public bool3 zyx { readonly get => new(z, y, x); set { z = value.x; y = value.y; x = value.z; } } + public readonly bool3 zyy => new(z, y, y); + public readonly bool3 zyz => new(z, y, z); + public bool3 zyw { readonly get => new(z, y, w); set { z = value.x; y = value.y; w = value.z; } } + public readonly bool3 zzx => new(z, z, x); + public readonly bool3 zzy => new(z, z, y); + public readonly bool3 zzz => new(z, z, z); + public readonly bool3 zzw => new(z, z, w); + public bool3 zwx { readonly get => new(z, w, x); set { z = value.x; w = value.y; x = value.z; } } + public bool3 zwy { readonly get => new(z, w, y); set { z = value.x; w = value.y; y = value.z; } } + public readonly bool3 zwz => new(z, w, z); + public readonly bool3 zww => new(z, w, w); + public readonly bool3 wxx => new(w, x, x); + public bool3 wxy { readonly get => new(w, x, y); set { w = value.x; x = value.y; y = value.z; } } + public bool3 wxz { readonly get => new(w, x, z); set { w = value.x; x = value.y; z = value.z; } } + public readonly bool3 wxw => new(w, x, w); + public bool3 wyx { readonly get => new(w, y, x); set { w = value.x; y = value.y; x = value.z; } } + public readonly bool3 wyy => new(w, y, y); + public bool3 wyz { readonly get => new(w, y, z); set { w = value.x; y = value.y; z = value.z; } } + public readonly bool3 wyw => new(w, y, w); + public bool3 wzx { readonly get => new(w, z, x); set { w = value.x; z = value.y; x = value.z; } } + public bool3 wzy { readonly get => new(w, z, y); set { w = value.x; z = value.y; y = value.z; } } + public readonly bool3 wzz => new(w, z, z); + public readonly bool3 wzw => new(w, z, w); + public readonly bool3 wwx => new(w, w, x); + public readonly bool3 wwy => new(w, w, y); + public readonly bool3 wwz => new(w, w, z); + public readonly bool3 www => new(w, w, w); + public readonly bool4 xxxx => new(x, x, x, x); + public readonly bool4 xxxy => new(x, x, x, y); + public readonly bool4 xxxz => new(x, x, x, z); + public readonly bool4 xxxw => new(x, x, x, w); + public readonly bool4 xxyx => new(x, x, y, x); + public readonly bool4 xxyy => new(x, x, y, y); + public readonly bool4 xxyz => new(x, x, y, z); + public readonly bool4 xxyw => new(x, x, y, w); + public readonly bool4 xxzx => new(x, x, z, x); + public readonly bool4 xxzy => new(x, x, z, y); + public readonly bool4 xxzz => new(x, x, z, z); + public readonly bool4 xxzw => new(x, x, z, w); + public readonly bool4 xxwx => new(x, x, w, x); + public readonly bool4 xxwy => new(x, x, w, y); + public readonly bool4 xxwz => new(x, x, w, z); + public readonly bool4 xxww => new(x, x, w, w); + public readonly bool4 xyxx => new(x, y, x, x); + public readonly bool4 xyxy => new(x, y, x, y); + public readonly bool4 xyxz => new(x, y, x, z); + public readonly bool4 xyxw => new(x, y, x, w); + public readonly bool4 xyyx => new(x, y, y, x); + public readonly bool4 xyyy => new(x, y, y, y); + public readonly bool4 xyyz => new(x, y, y, z); + public readonly bool4 xyyw => new(x, y, y, w); + public readonly bool4 xyzx => new(x, y, z, x); + public readonly bool4 xyzy => new(x, y, z, y); + public readonly bool4 xyzz => new(x, y, z, z); + public bool4 xyzw { readonly get => new(x, y, z, w); set { x = value.x; y = value.y; z = value.z; w = value.w; } } + public readonly bool4 xywx => new(x, y, w, x); + public readonly bool4 xywy => new(x, y, w, y); + public bool4 xywz { readonly get => new(x, y, w, z); set { x = value.x; y = value.y; w = value.z; z = value.w; } } + public readonly bool4 xyww => new(x, y, w, w); + public readonly bool4 xzxx => new(x, z, x, x); + public readonly bool4 xzxy => new(x, z, x, y); + public readonly bool4 xzxz => new(x, z, x, z); + public readonly bool4 xzxw => new(x, z, x, w); + public readonly bool4 xzyx => new(x, z, y, x); + public readonly bool4 xzyy => new(x, z, y, y); + public readonly bool4 xzyz => new(x, z, y, z); + public bool4 xzyw { readonly get => new(x, z, y, w); set { x = value.x; z = value.y; y = value.z; w = value.w; } } + public readonly bool4 xzzx => new(x, z, z, x); + public readonly bool4 xzzy => new(x, z, z, y); + public readonly bool4 xzzz => new(x, z, z, z); + public readonly bool4 xzzw => new(x, z, z, w); + public readonly bool4 xzwx => new(x, z, w, x); + public bool4 xzwy { readonly get => new(x, z, w, y); set { x = value.x; z = value.y; w = value.z; y = value.w; } } + public readonly bool4 xzwz => new(x, z, w, z); + public readonly bool4 xzww => new(x, z, w, w); + public readonly bool4 xwxx => new(x, w, x, x); + public readonly bool4 xwxy => new(x, w, x, y); + public readonly bool4 xwxz => new(x, w, x, z); + public readonly bool4 xwxw => new(x, w, x, w); + public readonly bool4 xwyx => new(x, w, y, x); + public readonly bool4 xwyy => new(x, w, y, y); + public bool4 xwyz { readonly get => new(x, w, y, z); set { x = value.x; w = value.y; y = value.z; z = value.w; } } + public readonly bool4 xwyw => new(x, w, y, w); + public readonly bool4 xwzx => new(x, w, z, x); + public bool4 xwzy { readonly get => new(x, w, z, y); set { x = value.x; w = value.y; z = value.z; y = value.w; } } + public readonly bool4 xwzz => new(x, w, z, z); + public readonly bool4 xwzw => new(x, w, z, w); + public readonly bool4 xwwx => new(x, w, w, x); + public readonly bool4 xwwy => new(x, w, w, y); + public readonly bool4 xwwz => new(x, w, w, z); + public readonly bool4 xwww => new(x, w, w, w); + public readonly bool4 yxxx => new(y, x, x, x); + public readonly bool4 yxxy => new(y, x, x, y); + public readonly bool4 yxxz => new(y, x, x, z); + public readonly bool4 yxxw => new(y, x, x, w); + public readonly bool4 yxyx => new(y, x, y, x); + public readonly bool4 yxyy => new(y, x, y, y); + public readonly bool4 yxyz => new(y, x, y, z); + public readonly bool4 yxyw => new(y, x, y, w); + public readonly bool4 yxzx => new(y, x, z, x); + public readonly bool4 yxzy => new(y, x, z, y); + public readonly bool4 yxzz => new(y, x, z, z); + public bool4 yxzw { readonly get => new(y, x, z, w); set { y = value.x; x = value.y; z = value.z; w = value.w; } } + public readonly bool4 yxwx => new(y, x, w, x); + public readonly bool4 yxwy => new(y, x, w, y); + public bool4 yxwz { readonly get => new(y, x, w, z); set { y = value.x; x = value.y; w = value.z; z = value.w; } } + public readonly bool4 yxww => new(y, x, w, w); + public readonly bool4 yyxx => new(y, y, x, x); + public readonly bool4 yyxy => new(y, y, x, y); + public readonly bool4 yyxz => new(y, y, x, z); + public readonly bool4 yyxw => new(y, y, x, w); + public readonly bool4 yyyx => new(y, y, y, x); + public readonly bool4 yyyy => new(y, y, y, y); + public readonly bool4 yyyz => new(y, y, y, z); + public readonly bool4 yyyw => new(y, y, y, w); + public readonly bool4 yyzx => new(y, y, z, x); + public readonly bool4 yyzy => new(y, y, z, y); + public readonly bool4 yyzz => new(y, y, z, z); + public readonly bool4 yyzw => new(y, y, z, w); + public readonly bool4 yywx => new(y, y, w, x); + public readonly bool4 yywy => new(y, y, w, y); + public readonly bool4 yywz => new(y, y, w, z); + public readonly bool4 yyww => new(y, y, w, w); + public readonly bool4 yzxx => new(y, z, x, x); + public readonly bool4 yzxy => new(y, z, x, y); + public readonly bool4 yzxz => new(y, z, x, z); + public bool4 yzxw { readonly get => new(y, z, x, w); set { y = value.x; z = value.y; x = value.z; w = value.w; } } + public readonly bool4 yzyx => new(y, z, y, x); + public readonly bool4 yzyy => new(y, z, y, y); + public readonly bool4 yzyz => new(y, z, y, z); + public readonly bool4 yzyw => new(y, z, y, w); + public readonly bool4 yzzx => new(y, z, z, x); + public readonly bool4 yzzy => new(y, z, z, y); + public readonly bool4 yzzz => new(y, z, z, z); + public readonly bool4 yzzw => new(y, z, z, w); + public bool4 yzwx { readonly get => new(y, z, w, x); set { y = value.x; z = value.y; w = value.z; x = value.w; } } + public readonly bool4 yzwy => new(y, z, w, y); + public readonly bool4 yzwz => new(y, z, w, z); + public readonly bool4 yzww => new(y, z, w, w); + public readonly bool4 ywxx => new(y, w, x, x); + public readonly bool4 ywxy => new(y, w, x, y); + public bool4 ywxz { readonly get => new(y, w, x, z); set { y = value.x; w = value.y; x = value.z; z = value.w; } } + public readonly bool4 ywxw => new(y, w, x, w); + public readonly bool4 ywyx => new(y, w, y, x); + public readonly bool4 ywyy => new(y, w, y, y); + public readonly bool4 ywyz => new(y, w, y, z); + public readonly bool4 ywyw => new(y, w, y, w); + public bool4 ywzx { readonly get => new(y, w, z, x); set { y = value.x; w = value.y; z = value.z; x = value.w; } } + public readonly bool4 ywzy => new(y, w, z, y); + public readonly bool4 ywzz => new(y, w, z, z); + public readonly bool4 ywzw => new(y, w, z, w); + public readonly bool4 ywwx => new(y, w, w, x); + public readonly bool4 ywwy => new(y, w, w, y); + public readonly bool4 ywwz => new(y, w, w, z); + public readonly bool4 ywww => new(y, w, w, w); + public readonly bool4 zxxx => new(z, x, x, x); + public readonly bool4 zxxy => new(z, x, x, y); + public readonly bool4 zxxz => new(z, x, x, z); + public readonly bool4 zxxw => new(z, x, x, w); + public readonly bool4 zxyx => new(z, x, y, x); + public readonly bool4 zxyy => new(z, x, y, y); + public readonly bool4 zxyz => new(z, x, y, z); + public bool4 zxyw { readonly get => new(z, x, y, w); set { z = value.x; x = value.y; y = value.z; w = value.w; } } + public readonly bool4 zxzx => new(z, x, z, x); + public readonly bool4 zxzy => new(z, x, z, y); + public readonly bool4 zxzz => new(z, x, z, z); + public readonly bool4 zxzw => new(z, x, z, w); + public readonly bool4 zxwx => new(z, x, w, x); + public bool4 zxwy { readonly get => new(z, x, w, y); set { z = value.x; x = value.y; w = value.z; y = value.w; } } + public readonly bool4 zxwz => new(z, x, w, z); + public readonly bool4 zxww => new(z, x, w, w); + public readonly bool4 zyxx => new(z, y, x, x); + public readonly bool4 zyxy => new(z, y, x, y); + public readonly bool4 zyxz => new(z, y, x, z); + public bool4 zyxw { readonly get => new(z, y, x, w); set { z = value.x; y = value.y; x = value.z; w = value.w; } } + public readonly bool4 zyyx => new(z, y, y, x); + public readonly bool4 zyyy => new(z, y, y, y); + public readonly bool4 zyyz => new(z, y, y, z); + public readonly bool4 zyyw => new(z, y, y, w); + public readonly bool4 zyzx => new(z, y, z, x); + public readonly bool4 zyzy => new(z, y, z, y); + public readonly bool4 zyzz => new(z, y, z, z); + public readonly bool4 zyzw => new(z, y, z, w); + public bool4 zywx { readonly get => new(z, y, w, x); set { z = value.x; y = value.y; w = value.z; x = value.w; } } + public readonly bool4 zywy => new(z, y, w, y); + public readonly bool4 zywz => new(z, y, w, z); + public readonly bool4 zyww => new(z, y, w, w); + public readonly bool4 zzxx => new(z, z, x, x); + public readonly bool4 zzxy => new(z, z, x, y); + public readonly bool4 zzxz => new(z, z, x, z); + public readonly bool4 zzxw => new(z, z, x, w); + public readonly bool4 zzyx => new(z, z, y, x); + public readonly bool4 zzyy => new(z, z, y, y); + public readonly bool4 zzyz => new(z, z, y, z); + public readonly bool4 zzyw => new(z, z, y, w); + public readonly bool4 zzzx => new(z, z, z, x); + public readonly bool4 zzzy => new(z, z, z, y); + public readonly bool4 zzzz => new(z, z, z, z); + public readonly bool4 zzzw => new(z, z, z, w); + public readonly bool4 zzwx => new(z, z, w, x); + public readonly bool4 zzwy => new(z, z, w, y); + public readonly bool4 zzwz => new(z, z, w, z); + public readonly bool4 zzww => new(z, z, w, w); + public readonly bool4 zwxx => new(z, w, x, x); + public bool4 zwxy { readonly get => new(z, w, x, y); set { z = value.x; w = value.y; x = value.z; y = value.w; } } + public readonly bool4 zwxz => new(z, w, x, z); + public readonly bool4 zwxw => new(z, w, x, w); + public bool4 zwyx { readonly get => new(z, w, y, x); set { z = value.x; w = value.y; y = value.z; x = value.w; } } + public readonly bool4 zwyy => new(z, w, y, y); + public readonly bool4 zwyz => new(z, w, y, z); + public readonly bool4 zwyw => new(z, w, y, w); + public readonly bool4 zwzx => new(z, w, z, x); + public readonly bool4 zwzy => new(z, w, z, y); + public readonly bool4 zwzz => new(z, w, z, z); + public readonly bool4 zwzw => new(z, w, z, w); + public readonly bool4 zwwx => new(z, w, w, x); + public readonly bool4 zwwy => new(z, w, w, y); + public readonly bool4 zwwz => new(z, w, w, z); + public readonly bool4 zwww => new(z, w, w, w); + public readonly bool4 wxxx => new(w, x, x, x); + public readonly bool4 wxxy => new(w, x, x, y); + public readonly bool4 wxxz => new(w, x, x, z); + public readonly bool4 wxxw => new(w, x, x, w); + public readonly bool4 wxyx => new(w, x, y, x); + public readonly bool4 wxyy => new(w, x, y, y); + public bool4 wxyz { readonly get => new(w, x, y, z); set { w = value.x; x = value.y; y = value.z; z = value.w; } } + public readonly bool4 wxyw => new(w, x, y, w); + public readonly bool4 wxzx => new(w, x, z, x); + public bool4 wxzy { readonly get => new(w, x, z, y); set { w = value.x; x = value.y; z = value.z; y = value.w; } } + public readonly bool4 wxzz => new(w, x, z, z); + public readonly bool4 wxzw => new(w, x, z, w); + public readonly bool4 wxwx => new(w, x, w, x); + public readonly bool4 wxwy => new(w, x, w, y); + public readonly bool4 wxwz => new(w, x, w, z); + public readonly bool4 wxww => new(w, x, w, w); + public readonly bool4 wyxx => new(w, y, x, x); + public readonly bool4 wyxy => new(w, y, x, y); + public bool4 wyxz { readonly get => new(w, y, x, z); set { w = value.x; y = value.y; x = value.z; z = value.w; } } + public readonly bool4 wyxw => new(w, y, x, w); + public readonly bool4 wyyx => new(w, y, y, x); + public readonly bool4 wyyy => new(w, y, y, y); + public readonly bool4 wyyz => new(w, y, y, z); + public readonly bool4 wyyw => new(w, y, y, w); + public bool4 wyzx { readonly get => new(w, y, z, x); set { w = value.x; y = value.y; z = value.z; x = value.w; } } + public readonly bool4 wyzy => new(w, y, z, y); + public readonly bool4 wyzz => new(w, y, z, z); + public readonly bool4 wyzw => new(w, y, z, w); + public readonly bool4 wywx => new(w, y, w, x); + public readonly bool4 wywy => new(w, y, w, y); + public readonly bool4 wywz => new(w, y, w, z); + public readonly bool4 wyww => new(w, y, w, w); + public readonly bool4 wzxx => new(w, z, x, x); + public bool4 wzxy { readonly get => new(w, z, x, y); set { w = value.x; z = value.y; x = value.z; y = value.w; } } + public readonly bool4 wzxz => new(w, z, x, z); + public readonly bool4 wzxw => new(w, z, x, w); + public bool4 wzyx { readonly get => new(w, z, y, x); set { w = value.x; z = value.y; y = value.z; x = value.w; } } + public readonly bool4 wzyy => new(w, z, y, y); + public readonly bool4 wzyz => new(w, z, y, z); + public readonly bool4 wzyw => new(w, z, y, w); + public readonly bool4 wzzx => new(w, z, z, x); + public readonly bool4 wzzy => new(w, z, z, y); + public readonly bool4 wzzz => new(w, z, z, z); + public readonly bool4 wzzw => new(w, z, z, w); + public readonly bool4 wzwx => new(w, z, w, x); + public readonly bool4 wzwy => new(w, z, w, y); + public readonly bool4 wzwz => new(w, z, w, z); + public readonly bool4 wzww => new(w, z, w, w); + public readonly bool4 wwxx => new(w, w, x, x); + public readonly bool4 wwxy => new(w, w, x, y); + public readonly bool4 wwxz => new(w, w, x, z); + public readonly bool4 wwxw => new(w, w, x, w); + public readonly bool4 wwyx => new(w, w, y, x); + public readonly bool4 wwyy => new(w, w, y, y); + public readonly bool4 wwyz => new(w, w, y, z); + public readonly bool4 wwyw => new(w, w, y, w); + public readonly bool4 wwzx => new(w, w, z, x); + public readonly bool4 wwzy => new(w, w, z, y); + public readonly bool4 wwzz => new(w, w, z, z); + public readonly bool4 wwzw => new(w, w, z, w); + public readonly bool4 wwwx => new(w, w, w, x); + public readonly bool4 wwwy => new(w, w, w, y); + public readonly bool4 wwwz => new(w, w, w, z); + public readonly bool4 wwww => new(w, w, w, w); + public readonly bool2 rr => new(x, x); + public bool2 rg { readonly get => new(x, y); set { x = value.x; y = value.y; } } + public bool2 rb { readonly get => new(x, z); set { x = value.x; z = value.y; } } + public bool2 ra { readonly get => new(x, w); set { x = value.x; w = value.y; } } + public bool2 gr { readonly get => new(y, x); set { y = value.x; x = value.y; } } + public readonly bool2 gg => new(y, y); + public bool2 gb { readonly get => new(y, z); set { y = value.x; z = value.y; } } + public bool2 ga { readonly get => new(y, w); set { y = value.x; w = value.y; } } + public bool2 br { readonly get => new(z, x); set { z = value.x; x = value.y; } } + public bool2 bg { readonly get => new(z, y); set { z = value.x; y = value.y; } } + public readonly bool2 bb => new(z, z); + public bool2 ba { readonly get => new(z, w); set { z = value.x; w = value.y; } } + public bool2 ar { readonly get => new(w, x); set { w = value.x; x = value.y; } } + public bool2 ag { readonly get => new(w, y); set { w = value.x; y = value.y; } } + public bool2 ab { readonly get => new(w, z); set { w = value.x; z = value.y; } } + public readonly bool2 aa => new(w, w); + public readonly bool3 rrr => new(x, x, x); + public readonly bool3 rrg => new(x, x, y); + public readonly bool3 rrb => new(x, x, z); + public readonly bool3 rra => new(x, x, w); + public readonly bool3 rgr => new(x, y, x); + public readonly bool3 rgg => new(x, y, y); + public bool3 rgb { readonly get => new(x, y, z); set { x = value.x; y = value.y; z = value.z; } } + public bool3 rga { readonly get => new(x, y, w); set { x = value.x; y = value.y; w = value.z; } } + public readonly bool3 rbr => new(x, z, x); + public bool3 rbg { readonly get => new(x, z, y); set { x = value.x; z = value.y; y = value.z; } } + public readonly bool3 rbb => new(x, z, z); + public bool3 rba { readonly get => new(x, z, w); set { x = value.x; z = value.y; w = value.z; } } + public readonly bool3 rar => new(x, w, x); + public bool3 rag { readonly get => new(x, w, y); set { x = value.x; w = value.y; y = value.z; } } + public bool3 rab { readonly get => new(x, w, z); set { x = value.x; w = value.y; z = value.z; } } + public readonly bool3 raa => new(x, w, w); + public readonly bool3 grr => new(y, x, x); + public readonly bool3 grg => new(y, x, y); + public bool3 grb { readonly get => new(y, x, z); set { y = value.x; x = value.y; z = value.z; } } + public bool3 gra { readonly get => new(y, x, w); set { y = value.x; x = value.y; w = value.z; } } + public readonly bool3 ggr => new(y, y, x); + public readonly bool3 ggg => new(y, y, y); + public readonly bool3 ggb => new(y, y, z); + public readonly bool3 gga => new(y, y, w); + public bool3 gbr { readonly get => new(y, z, x); set { y = value.x; z = value.y; x = value.z; } } + public readonly bool3 gbg => new(y, z, y); + public readonly bool3 gbb => new(y, z, z); + public bool3 gba { readonly get => new(y, z, w); set { y = value.x; z = value.y; w = value.z; } } + public bool3 gar { readonly get => new(y, w, x); set { y = value.x; w = value.y; x = value.z; } } + public readonly bool3 gag => new(y, w, y); + public bool3 gab { readonly get => new(y, w, z); set { y = value.x; w = value.y; z = value.z; } } + public readonly bool3 gaa => new(y, w, w); + public readonly bool3 brr => new(z, x, x); + public bool3 brg { readonly get => new(z, x, y); set { z = value.x; x = value.y; y = value.z; } } + public readonly bool3 brb => new(z, x, z); + public bool3 bra { readonly get => new(z, x, w); set { z = value.x; x = value.y; w = value.z; } } + public bool3 bgr { readonly get => new(z, y, x); set { z = value.x; y = value.y; x = value.z; } } + public readonly bool3 bgg => new(z, y, y); + public readonly bool3 bgb => new(z, y, z); + public bool3 bga { readonly get => new(z, y, w); set { z = value.x; y = value.y; w = value.z; } } + public readonly bool3 bbr => new(z, z, x); + public readonly bool3 bbg => new(z, z, y); + public readonly bool3 bbb => new(z, z, z); + public readonly bool3 bba => new(z, z, w); + public bool3 bar { readonly get => new(z, w, x); set { z = value.x; w = value.y; x = value.z; } } + public bool3 bag { readonly get => new(z, w, y); set { z = value.x; w = value.y; y = value.z; } } + public readonly bool3 bab => new(z, w, z); + public readonly bool3 baa => new(z, w, w); + public readonly bool3 arr => new(w, x, x); + public bool3 arg { readonly get => new(w, x, y); set { w = value.x; x = value.y; y = value.z; } } + public bool3 arb { readonly get => new(w, x, z); set { w = value.x; x = value.y; z = value.z; } } + public readonly bool3 ara => new(w, x, w); + public bool3 agr { readonly get => new(w, y, x); set { w = value.x; y = value.y; x = value.z; } } + public readonly bool3 agg => new(w, y, y); + public bool3 agb { readonly get => new(w, y, z); set { w = value.x; y = value.y; z = value.z; } } + public readonly bool3 aga => new(w, y, w); + public bool3 abr { readonly get => new(w, z, x); set { w = value.x; z = value.y; x = value.z; } } + public bool3 abg { readonly get => new(w, z, y); set { w = value.x; z = value.y; y = value.z; } } + public readonly bool3 abb => new(w, z, z); + public readonly bool3 aba => new(w, z, w); + public readonly bool3 aar => new(w, w, x); + public readonly bool3 aag => new(w, w, y); + public readonly bool3 aab => new(w, w, z); + public readonly bool3 aaa => new(w, w, w); + public readonly bool4 rrrr => new(x, x, x, x); + public readonly bool4 rrrg => new(x, x, x, y); + public readonly bool4 rrrb => new(x, x, x, z); + public readonly bool4 rrra => new(x, x, x, w); + public readonly bool4 rrgr => new(x, x, y, x); + public readonly bool4 rrgg => new(x, x, y, y); + public readonly bool4 rrgb => new(x, x, y, z); + public readonly bool4 rrga => new(x, x, y, w); + public readonly bool4 rrbr => new(x, x, z, x); + public readonly bool4 rrbg => new(x, x, z, y); + public readonly bool4 rrbb => new(x, x, z, z); + public readonly bool4 rrba => new(x, x, z, w); + public readonly bool4 rrar => new(x, x, w, x); + public readonly bool4 rrag => new(x, x, w, y); + public readonly bool4 rrab => new(x, x, w, z); + public readonly bool4 rraa => new(x, x, w, w); + public readonly bool4 rgrr => new(x, y, x, x); + public readonly bool4 rgrg => new(x, y, x, y); + public readonly bool4 rgrb => new(x, y, x, z); + public readonly bool4 rgra => new(x, y, x, w); + public readonly bool4 rggr => new(x, y, y, x); + public readonly bool4 rggg => new(x, y, y, y); + public readonly bool4 rggb => new(x, y, y, z); + public readonly bool4 rgga => new(x, y, y, w); + public readonly bool4 rgbr => new(x, y, z, x); + public readonly bool4 rgbg => new(x, y, z, y); + public readonly bool4 rgbb => new(x, y, z, z); + public bool4 rgba { readonly get => new(x, y, z, w); set { x = value.x; y = value.y; z = value.z; w = value.w; } } + public readonly bool4 rgar => new(x, y, w, x); + public readonly bool4 rgag => new(x, y, w, y); + public bool4 rgab { readonly get => new(x, y, w, z); set { x = value.x; y = value.y; w = value.z; z = value.w; } } + public readonly bool4 rgaa => new(x, y, w, w); + public readonly bool4 rbrr => new(x, z, x, x); + public readonly bool4 rbrg => new(x, z, x, y); + public readonly bool4 rbrb => new(x, z, x, z); + public readonly bool4 rbra => new(x, z, x, w); + public readonly bool4 rbgr => new(x, z, y, x); + public readonly bool4 rbgg => new(x, z, y, y); + public readonly bool4 rbgb => new(x, z, y, z); + public bool4 rbga { readonly get => new(x, z, y, w); set { x = value.x; z = value.y; y = value.z; w = value.w; } } + public readonly bool4 rbbr => new(x, z, z, x); + public readonly bool4 rbbg => new(x, z, z, y); + public readonly bool4 rbbb => new(x, z, z, z); + public readonly bool4 rbba => new(x, z, z, w); + public readonly bool4 rbar => new(x, z, w, x); + public bool4 rbag { readonly get => new(x, z, w, y); set { x = value.x; z = value.y; w = value.z; y = value.w; } } + public readonly bool4 rbab => new(x, z, w, z); + public readonly bool4 rbaa => new(x, z, w, w); + public readonly bool4 rarr => new(x, w, x, x); + public readonly bool4 rarg => new(x, w, x, y); + public readonly bool4 rarb => new(x, w, x, z); + public readonly bool4 rara => new(x, w, x, w); + public readonly bool4 ragr => new(x, w, y, x); + public readonly bool4 ragg => new(x, w, y, y); + public bool4 ragb { readonly get => new(x, w, y, z); set { x = value.x; w = value.y; y = value.z; z = value.w; } } + public readonly bool4 raga => new(x, w, y, w); + public readonly bool4 rabr => new(x, w, z, x); + public bool4 rabg { readonly get => new(x, w, z, y); set { x = value.x; w = value.y; z = value.z; y = value.w; } } + public readonly bool4 rabb => new(x, w, z, z); + public readonly bool4 raba => new(x, w, z, w); + public readonly bool4 raar => new(x, w, w, x); + public readonly bool4 raag => new(x, w, w, y); + public readonly bool4 raab => new(x, w, w, z); + public readonly bool4 raaa => new(x, w, w, w); + public readonly bool4 grrr => new(y, x, x, x); + public readonly bool4 grrg => new(y, x, x, y); + public readonly bool4 grrb => new(y, x, x, z); + public readonly bool4 grra => new(y, x, x, w); + public readonly bool4 grgr => new(y, x, y, x); + public readonly bool4 grgg => new(y, x, y, y); + public readonly bool4 grgb => new(y, x, y, z); + public readonly bool4 grga => new(y, x, y, w); + public readonly bool4 grbr => new(y, x, z, x); + public readonly bool4 grbg => new(y, x, z, y); + public readonly bool4 grbb => new(y, x, z, z); + public bool4 grba { readonly get => new(y, x, z, w); set { y = value.x; x = value.y; z = value.z; w = value.w; } } + public readonly bool4 grar => new(y, x, w, x); + public readonly bool4 grag => new(y, x, w, y); + public bool4 grab { readonly get => new(y, x, w, z); set { y = value.x; x = value.y; w = value.z; z = value.w; } } + public readonly bool4 graa => new(y, x, w, w); + public readonly bool4 ggrr => new(y, y, x, x); + public readonly bool4 ggrg => new(y, y, x, y); + public readonly bool4 ggrb => new(y, y, x, z); + public readonly bool4 ggra => new(y, y, x, w); + public readonly bool4 gggr => new(y, y, y, x); + public readonly bool4 gggg => new(y, y, y, y); + public readonly bool4 gggb => new(y, y, y, z); + public readonly bool4 ggga => new(y, y, y, w); + public readonly bool4 ggbr => new(y, y, z, x); + public readonly bool4 ggbg => new(y, y, z, y); + public readonly bool4 ggbb => new(y, y, z, z); + public readonly bool4 ggba => new(y, y, z, w); + public readonly bool4 ggar => new(y, y, w, x); + public readonly bool4 ggag => new(y, y, w, y); + public readonly bool4 ggab => new(y, y, w, z); + public readonly bool4 ggaa => new(y, y, w, w); + public readonly bool4 gbrr => new(y, z, x, x); + public readonly bool4 gbrg => new(y, z, x, y); + public readonly bool4 gbrb => new(y, z, x, z); + public bool4 gbra { readonly get => new(y, z, x, w); set { y = value.x; z = value.y; x = value.z; w = value.w; } } + public readonly bool4 gbgr => new(y, z, y, x); + public readonly bool4 gbgg => new(y, z, y, y); + public readonly bool4 gbgb => new(y, z, y, z); + public readonly bool4 gbga => new(y, z, y, w); + public readonly bool4 gbbr => new(y, z, z, x); + public readonly bool4 gbbg => new(y, z, z, y); + public readonly bool4 gbbb => new(y, z, z, z); + public readonly bool4 gbba => new(y, z, z, w); + public bool4 gbar { readonly get => new(y, z, w, x); set { y = value.x; z = value.y; w = value.z; x = value.w; } } + public readonly bool4 gbag => new(y, z, w, y); + public readonly bool4 gbab => new(y, z, w, z); + public readonly bool4 gbaa => new(y, z, w, w); + public readonly bool4 garr => new(y, w, x, x); + public readonly bool4 garg => new(y, w, x, y); + public bool4 garb { readonly get => new(y, w, x, z); set { y = value.x; w = value.y; x = value.z; z = value.w; } } + public readonly bool4 gara => new(y, w, x, w); + public readonly bool4 gagr => new(y, w, y, x); + public readonly bool4 gagg => new(y, w, y, y); + public readonly bool4 gagb => new(y, w, y, z); + public readonly bool4 gaga => new(y, w, y, w); + public bool4 gabr { readonly get => new(y, w, z, x); set { y = value.x; w = value.y; z = value.z; x = value.w; } } + public readonly bool4 gabg => new(y, w, z, y); + public readonly bool4 gabb => new(y, w, z, z); + public readonly bool4 gaba => new(y, w, z, w); + public readonly bool4 gaar => new(y, w, w, x); + public readonly bool4 gaag => new(y, w, w, y); + public readonly bool4 gaab => new(y, w, w, z); + public readonly bool4 gaaa => new(y, w, w, w); + public readonly bool4 brrr => new(z, x, x, x); + public readonly bool4 brrg => new(z, x, x, y); + public readonly bool4 brrb => new(z, x, x, z); + public readonly bool4 brra => new(z, x, x, w); + public readonly bool4 brgr => new(z, x, y, x); + public readonly bool4 brgg => new(z, x, y, y); + public readonly bool4 brgb => new(z, x, y, z); + public bool4 brga { readonly get => new(z, x, y, w); set { z = value.x; x = value.y; y = value.z; w = value.w; } } + public readonly bool4 brbr => new(z, x, z, x); + public readonly bool4 brbg => new(z, x, z, y); + public readonly bool4 brbb => new(z, x, z, z); + public readonly bool4 brba => new(z, x, z, w); + public readonly bool4 brar => new(z, x, w, x); + public bool4 brag { readonly get => new(z, x, w, y); set { z = value.x; x = value.y; w = value.z; y = value.w; } } + public readonly bool4 brab => new(z, x, w, z); + public readonly bool4 braa => new(z, x, w, w); + public readonly bool4 bgrr => new(z, y, x, x); + public readonly bool4 bgrg => new(z, y, x, y); + public readonly bool4 bgrb => new(z, y, x, z); + public bool4 bgra { readonly get => new(z, y, x, w); set { z = value.x; y = value.y; x = value.z; w = value.w; } } + public readonly bool4 bggr => new(z, y, y, x); + public readonly bool4 bggg => new(z, y, y, y); + public readonly bool4 bggb => new(z, y, y, z); + public readonly bool4 bgga => new(z, y, y, w); + public readonly bool4 bgbr => new(z, y, z, x); + public readonly bool4 bgbg => new(z, y, z, y); + public readonly bool4 bgbb => new(z, y, z, z); + public readonly bool4 bgba => new(z, y, z, w); + public bool4 bgar { readonly get => new(z, y, w, x); set { z = value.x; y = value.y; w = value.z; x = value.w; } } + public readonly bool4 bgag => new(z, y, w, y); + public readonly bool4 bgab => new(z, y, w, z); + public readonly bool4 bgaa => new(z, y, w, w); + public readonly bool4 bbrr => new(z, z, x, x); + public readonly bool4 bbrg => new(z, z, x, y); + public readonly bool4 bbrb => new(z, z, x, z); + public readonly bool4 bbra => new(z, z, x, w); + public readonly bool4 bbgr => new(z, z, y, x); + public readonly bool4 bbgg => new(z, z, y, y); + public readonly bool4 bbgb => new(z, z, y, z); + public readonly bool4 bbga => new(z, z, y, w); + public readonly bool4 bbbr => new(z, z, z, x); + public readonly bool4 bbbg => new(z, z, z, y); + public readonly bool4 bbbb => new(z, z, z, z); + public readonly bool4 bbba => new(z, z, z, w); + public readonly bool4 bbar => new(z, z, w, x); + public readonly bool4 bbag => new(z, z, w, y); + public readonly bool4 bbab => new(z, z, w, z); + public readonly bool4 bbaa => new(z, z, w, w); + public readonly bool4 barr => new(z, w, x, x); + public bool4 barg { readonly get => new(z, w, x, y); set { z = value.x; w = value.y; x = value.z; y = value.w; } } + public readonly bool4 barb => new(z, w, x, z); + public readonly bool4 bara => new(z, w, x, w); + public bool4 bagr { readonly get => new(z, w, y, x); set { z = value.x; w = value.y; y = value.z; x = value.w; } } + public readonly bool4 bagg => new(z, w, y, y); + public readonly bool4 bagb => new(z, w, y, z); + public readonly bool4 baga => new(z, w, y, w); + public readonly bool4 babr => new(z, w, z, x); + public readonly bool4 babg => new(z, w, z, y); + public readonly bool4 babb => new(z, w, z, z); + public readonly bool4 baba => new(z, w, z, w); + public readonly bool4 baar => new(z, w, w, x); + public readonly bool4 baag => new(z, w, w, y); + public readonly bool4 baab => new(z, w, w, z); + public readonly bool4 baaa => new(z, w, w, w); + public readonly bool4 arrr => new(w, x, x, x); + public readonly bool4 arrg => new(w, x, x, y); + public readonly bool4 arrb => new(w, x, x, z); + public readonly bool4 arra => new(w, x, x, w); + public readonly bool4 argr => new(w, x, y, x); + public readonly bool4 argg => new(w, x, y, y); + public bool4 argb { readonly get => new(w, x, y, z); set { w = value.x; x = value.y; y = value.z; z = value.w; } } + public readonly bool4 arga => new(w, x, y, w); + public readonly bool4 arbr => new(w, x, z, x); + public bool4 arbg { readonly get => new(w, x, z, y); set { w = value.x; x = value.y; z = value.z; y = value.w; } } + public readonly bool4 arbb => new(w, x, z, z); + public readonly bool4 arba => new(w, x, z, w); + public readonly bool4 arar => new(w, x, w, x); + public readonly bool4 arag => new(w, x, w, y); + public readonly bool4 arab => new(w, x, w, z); + public readonly bool4 araa => new(w, x, w, w); + public readonly bool4 agrr => new(w, y, x, x); + public readonly bool4 agrg => new(w, y, x, y); + public bool4 agrb { readonly get => new(w, y, x, z); set { w = value.x; y = value.y; x = value.z; z = value.w; } } + public readonly bool4 agra => new(w, y, x, w); + public readonly bool4 aggr => new(w, y, y, x); + public readonly bool4 aggg => new(w, y, y, y); + public readonly bool4 aggb => new(w, y, y, z); + public readonly bool4 agga => new(w, y, y, w); + public bool4 agbr { readonly get => new(w, y, z, x); set { w = value.x; y = value.y; z = value.z; x = value.w; } } + public readonly bool4 agbg => new(w, y, z, y); + public readonly bool4 agbb => new(w, y, z, z); + public readonly bool4 agba => new(w, y, z, w); + public readonly bool4 agar => new(w, y, w, x); + public readonly bool4 agag => new(w, y, w, y); + public readonly bool4 agab => new(w, y, w, z); + public readonly bool4 agaa => new(w, y, w, w); + public readonly bool4 abrr => new(w, z, x, x); + public bool4 abrg { readonly get => new(w, z, x, y); set { w = value.x; z = value.y; x = value.z; y = value.w; } } + public readonly bool4 abrb => new(w, z, x, z); + public readonly bool4 abra => new(w, z, x, w); + public bool4 abgr { readonly get => new(w, z, y, x); set { w = value.x; z = value.y; y = value.z; x = value.w; } } + public readonly bool4 abgg => new(w, z, y, y); + public readonly bool4 abgb => new(w, z, y, z); + public readonly bool4 abga => new(w, z, y, w); + public readonly bool4 abbr => new(w, z, z, x); + public readonly bool4 abbg => new(w, z, z, y); + public readonly bool4 abbb => new(w, z, z, z); + public readonly bool4 abba => new(w, z, z, w); + public readonly bool4 abar => new(w, z, w, x); + public readonly bool4 abag => new(w, z, w, y); + public readonly bool4 abab => new(w, z, w, z); + public readonly bool4 abaa => new(w, z, w, w); + public readonly bool4 aarr => new(w, w, x, x); + public readonly bool4 aarg => new(w, w, x, y); + public readonly bool4 aarb => new(w, w, x, z); + public readonly bool4 aara => new(w, w, x, w); + public readonly bool4 aagr => new(w, w, y, x); + public readonly bool4 aagg => new(w, w, y, y); + public readonly bool4 aagb => new(w, w, y, z); + public readonly bool4 aaga => new(w, w, y, w); + public readonly bool4 aabr => new(w, w, z, x); + public readonly bool4 aabg => new(w, w, z, y); + public readonly bool4 aabb => new(w, w, z, z); + public readonly bool4 aaba => new(w, w, z, w); + public readonly bool4 aaar => new(w, w, w, x); + public readonly bool4 aaag => new(w, w, w, y); + public readonly bool4 aaab => new(w, w, w, z); + public readonly bool4 aaaa => new(w, w, w, w); +} + +public partial struct double2 +{ + public double2(double2 xy) : this(xy.x, xy.y) { } + public double r { readonly get => x; set => x = value; } + public double g { readonly get => y; set => y = value; } + public readonly double2 xx => new(x, x); + public double2 xy { readonly get => new(x, y); set { x = value.x; y = value.y; } } + public double2 yx { readonly get => new(y, x); set { y = value.x; x = value.y; } } + public readonly double2 yy => new(y, y); + public readonly double3 xxx => new(x, x, x); + public readonly double3 xxy => new(x, x, y); + public readonly double3 xyx => new(x, y, x); + public readonly double3 xyy => new(x, y, y); + public readonly double3 yxx => new(y, x, x); + public readonly double3 yxy => new(y, x, y); + public readonly double3 yyx => new(y, y, x); + public readonly double3 yyy => new(y, y, y); + public readonly double2 rr => new(x, x); + public double2 rg { readonly get => new(x, y); set { x = value.x; y = value.y; } } + public double2 gr { readonly get => new(y, x); set { y = value.x; x = value.y; } } + public readonly double2 gg => new(y, y); + public readonly double3 rrr => new(x, x, x); + public readonly double3 rrg => new(x, x, y); + public readonly double3 rgr => new(x, y, x); + public readonly double3 rgg => new(x, y, y); + public readonly double3 grr => new(y, x, x); + public readonly double3 grg => new(y, x, y); + public readonly double3 ggr => new(y, y, x); + public readonly double3 ggg => new(y, y, y); +} + +public partial struct double3 +{ + public double3(double x, double2 yz) : this(x, yz.x, yz.y) { } + public double3(double2 xy, double z) : this(xy.x, xy.y, z) { } + public double3(double3 xyz) : this(xyz.x, xyz.y, xyz.z) { } + public double r { readonly get => x; set => x = value; } + public double g { readonly get => y; set => y = value; } + public double b { readonly get => z; set => z = value; } + public readonly double2 xx => new(x, x); + public double2 xy { readonly get => new(x, y); set { x = value.x; y = value.y; } } + public double2 xz { readonly get => new(x, z); set { x = value.x; z = value.y; } } + public double2 yx { readonly get => new(y, x); set { y = value.x; x = value.y; } } + public readonly double2 yy => new(y, y); + public double2 yz { readonly get => new(y, z); set { y = value.x; z = value.y; } } + public double2 zx { readonly get => new(z, x); set { z = value.x; x = value.y; } } + public double2 zy { readonly get => new(z, y); set { z = value.x; y = value.y; } } + public readonly double2 zz => new(z, z); + public readonly double3 xxx => new(x, x, x); + public readonly double3 xxy => new(x, x, y); + public readonly double3 xxz => new(x, x, z); + public readonly double3 xyx => new(x, y, x); + public readonly double3 xyy => new(x, y, y); + public double3 xyz { readonly get => new(x, y, z); set { x = value.x; y = value.y; z = value.z; } } + public readonly double3 xzx => new(x, z, x); + public double3 xzy { readonly get => new(x, z, y); set { x = value.x; z = value.y; y = value.z; } } + public readonly double3 xzz => new(x, z, z); + public readonly double3 yxx => new(y, x, x); + public readonly double3 yxy => new(y, x, y); + public double3 yxz { readonly get => new(y, x, z); set { y = value.x; x = value.y; z = value.z; } } + public readonly double3 yyx => new(y, y, x); + public readonly double3 yyy => new(y, y, y); + public readonly double3 yyz => new(y, y, z); + public double3 yzx { readonly get => new(y, z, x); set { y = value.x; z = value.y; x = value.z; } } + public readonly double3 yzy => new(y, z, y); + public readonly double3 yzz => new(y, z, z); + public readonly double3 zxx => new(z, x, x); + public double3 zxy { readonly get => new(z, x, y); set { z = value.x; x = value.y; y = value.z; } } + public readonly double3 zxz => new(z, x, z); + public double3 zyx { readonly get => new(z, y, x); set { z = value.x; y = value.y; x = value.z; } } + public readonly double3 zyy => new(z, y, y); + public readonly double3 zyz => new(z, y, z); + public readonly double3 zzx => new(z, z, x); + public readonly double3 zzy => new(z, z, y); + public readonly double3 zzz => new(z, z, z); + public readonly double2 rr => new(x, x); + public double2 rg { readonly get => new(x, y); set { x = value.x; y = value.y; } } + public double2 rb { readonly get => new(x, z); set { x = value.x; z = value.y; } } + public double2 gr { readonly get => new(y, x); set { y = value.x; x = value.y; } } + public readonly double2 gg => new(y, y); + public double2 gb { readonly get => new(y, z); set { y = value.x; z = value.y; } } + public double2 br { readonly get => new(z, x); set { z = value.x; x = value.y; } } + public double2 bg { readonly get => new(z, y); set { z = value.x; y = value.y; } } + public readonly double2 bb => new(z, z); + public readonly double3 rrr => new(x, x, x); + public readonly double3 rrg => new(x, x, y); + public readonly double3 rrb => new(x, x, z); + public readonly double3 rgr => new(x, y, x); + public readonly double3 rgg => new(x, y, y); + public double3 rgb { readonly get => new(x, y, z); set { x = value.x; y = value.y; z = value.z; } } + public readonly double3 rbr => new(x, z, x); + public double3 rbg { readonly get => new(x, z, y); set { x = value.x; z = value.y; y = value.z; } } + public readonly double3 rbb => new(x, z, z); + public readonly double3 grr => new(y, x, x); + public readonly double3 grg => new(y, x, y); + public double3 grb { readonly get => new(y, x, z); set { y = value.x; x = value.y; z = value.z; } } + public readonly double3 ggr => new(y, y, x); + public readonly double3 ggg => new(y, y, y); + public readonly double3 ggb => new(y, y, z); + public double3 gbr { readonly get => new(y, z, x); set { y = value.x; z = value.y; x = value.z; } } + public readonly double3 gbg => new(y, z, y); + public readonly double3 gbb => new(y, z, z); + public readonly double3 brr => new(z, x, x); + public double3 brg { readonly get => new(z, x, y); set { z = value.x; x = value.y; y = value.z; } } + public readonly double3 brb => new(z, x, z); + public double3 bgr { readonly get => new(z, y, x); set { z = value.x; y = value.y; x = value.z; } } + public readonly double3 bgg => new(z, y, y); + public readonly double3 bgb => new(z, y, z); + public readonly double3 bbr => new(z, z, x); + public readonly double3 bbg => new(z, z, y); + public readonly double3 bbb => new(z, z, z); +} + +public static partial class math +{ + public static float2 float2(float2 xy) => new(xy); + public static float3 float3(float x, float2 yz) => new(x, yz); + public static float3 float3(float2 xy, float z) => new(xy, z); + public static float3 float3(float3 xyz) => new(xyz); + public static float4 float4(float x, float y, float2 zw) => new(x, y, zw); + public static float4 float4(float x, float2 yz, float w) => new(x, yz, w); + public static float4 float4(float x, float3 yzw) => new(x, yzw); + public static float4 float4(float2 xy, float z, float w) => new(xy, z, w); + public static float4 float4(float2 xy, float2 zw) => new(xy, zw); + public static float4 float4(float3 xyz, float w) => new(xyz, w); + public static float4 float4(float4 xyzw) => new(xyzw); + public static int2 int2(int2 xy) => new(xy); + public static int3 int3(int x, int2 yz) => new(x, yz); + public static int3 int3(int2 xy, int z) => new(xy, z); + public static int3 int3(int3 xyz) => new(xyz); + public static int4 int4(int x, int y, int2 zw) => new(x, y, zw); + public static int4 int4(int x, int2 yz, int w) => new(x, yz, w); + public static int4 int4(int x, int3 yzw) => new(x, yzw); + public static int4 int4(int2 xy, int z, int w) => new(xy, z, w); + public static int4 int4(int2 xy, int2 zw) => new(xy, zw); + public static int4 int4(int3 xyz, int w) => new(xyz, w); + public static int4 int4(int4 xyzw) => new(xyzw); + public static bool2 bool2(bool2 xy) => new(xy); + public static bool3 bool3(bool x, bool2 yz) => new(x, yz); + public static bool3 bool3(bool2 xy, bool z) => new(xy, z); + public static bool3 bool3(bool3 xyz) => new(xyz); + public static bool4 bool4(bool x, bool y, bool2 zw) => new(x, y, zw); + public static bool4 bool4(bool x, bool2 yz, bool w) => new(x, yz, w); + public static bool4 bool4(bool x, bool3 yzw) => new(x, yzw); + public static bool4 bool4(bool2 xy, bool z, bool w) => new(xy, z, w); + public static bool4 bool4(bool2 xy, bool2 zw) => new(xy, zw); + public static bool4 bool4(bool3 xyz, bool w) => new(xyz, w); + public static bool4 bool4(bool4 xyzw) => new(xyzw); +} diff --git a/packages/cultmath/src/CultMath/Swizzles.g.cs.meta b/packages/cultmath/src/CultMath/Swizzles.g.cs.meta new file mode 100644 index 00000000..035e577f --- /dev/null +++ b/packages/cultmath/src/CultMath/Swizzles.g.cs.meta @@ -0,0 +1,2 @@ +fileFormatVersion: 2 +guid: e89d6089b354459788eb6f0faebfd6c0 diff --git a/packages/cultmath/src/CultMath/UnscopedRefAttribute.cs b/packages/cultmath/src/CultMath/UnscopedRefAttribute.cs new file mode 100644 index 00000000..6e1e3148 --- /dev/null +++ b/packages/cultmath/src/CultMath/UnscopedRefAttribute.cs @@ -0,0 +1,10 @@ +#if !NET7_0_OR_GREATER +namespace System.Diagnostics.CodeAnalysis; + +// Compiler-recognized polyfill so netstandard2.1 builds can return refs into a struct's +// own fields (the matrix row indexer). net7.0+ supplies the real attribute. +[AttributeUsage(AttributeTargets.Method | AttributeTargets.Property | AttributeTargets.Parameter, AllowMultiple = false, Inherited = false)] +internal sealed class UnscopedRefAttribute : Attribute +{ +} +#endif diff --git a/packages/cultmath/src/CultMath/UnscopedRefAttribute.cs.meta b/packages/cultmath/src/CultMath/UnscopedRefAttribute.cs.meta new file mode 100644 index 00000000..61df7771 --- /dev/null +++ b/packages/cultmath/src/CultMath/UnscopedRefAttribute.cs.meta @@ -0,0 +1,2 @@ +fileFormatVersion: 2 +guid: 3f204000f9854517a8dd754e2d9dc975 diff --git a/packages/cultmath/src/CultMath/Voronoi.cs b/packages/cultmath/src/CultMath/Voronoi.cs new file mode 100644 index 00000000..38b52d81 --- /dev/null +++ b/packages/cultmath/src/CultMath/Voronoi.cs @@ -0,0 +1,229 @@ +using System; +using System.Runtime.InteropServices; + +namespace CultMath; + +public static class Voronoi +{ + public static bool NativeAvailable { get; private set; } = true; + + public static Color32 SampleTone(float x, float y, float resolutionY, int frameIndex, CultMathTone tone, float span) + { + Span xs = stackalloc[] { x }; + Span ys = stackalloc[] { y }; + Span tones = stackalloc[] { tone }; + Span spans = stackalloc[] { span }; + Span colors = stackalloc Color32[1]; + SampleTones(xs, ys, tones, spans, resolutionY, frameIndex, colors); + return colors[0]; + } + + public static void SampleTones( + ReadOnlySpan xs, + ReadOnlySpan ys, + ReadOnlySpan tones, + ReadOnlySpan spans, + float resolutionY, + int frameIndex, + Span colors) + { + if (xs.Length != ys.Length || xs.Length != tones.Length || xs.Length != spans.Length || xs.Length != colors.Length) + { + throw new ArgumentException("CultMath Voronoi batch spans must have equal length."); + } + + if (xs.IsEmpty) + { + return; + } + + if (NativeAvailable && TrySampleNative(xs, ys, tones, spans, resolutionY, frameIndex, colors)) + { + return; + } + + for (var index = 0; index < xs.Length; index++) + { + colors[index] = SampleManaged(xs[index], ys[index], resolutionY, frameIndex, tones[index], spans[index]); + } + } + + private static unsafe bool TrySampleNative( + ReadOnlySpan xs, + ReadOnlySpan ys, + ReadOnlySpan tones, + ReadOnlySpan spans, + float resolutionY, + int frameIndex, + Span colors) + { + try + { + fixed (float* xPtr = xs) + fixed (float* yPtr = ys) + fixed (CultMathTone* tonePtr = tones) + fixed (float* spanPtr = spans) + fixed (Color32* colorPtr = colors) + { + var result = Native.cultmath_apollonian_voronoi_tones( + xPtr, + yPtr, + (byte*)tonePtr, + spanPtr, + (nuint)xs.Length, + resolutionY, + frameIndex, + colorPtr); + if (result == 0) + { + return true; + } + } + } + catch (DllNotFoundException) + { + NativeAvailable = false; + } + catch (EntryPointNotFoundException) + { + NativeAvailable = false; + } + + return false; + } + + private static Color32 SampleManaged(float x, float y, float resolutionY, int frameIndex, CultMathTone tone, float span) + { + var sample = StableSample(x, y, resolutionY, frameIndex, tone, span); + var luminance = math.saturate(sample.x * 0.299f + sample.y * 0.587f + sample.z * 0.114f); + if (tone == CultMathTone.Header) + { + var glow = 0.62f + luminance * 0.38f; + return new Color32((byte)(255 * glow), (byte)(96 * glow), (byte)(14 * glow)); + } + + if (tone == CultMathTone.Body) + { + var glow = 0.68f + luminance * 0.32f; + return new Color32((byte)(245 * glow), (byte)(248 * glow), (byte)(252 * glow)); + } + + var gain = tone switch + { + CultMathTone.Panel => 0.42f, + CultMathTone.Edge => 1.05f, + _ => 0.24f, + }; + var lift = tone switch + { + CultMathTone.Panel => 14.0f, + CultMathTone.Edge => 24.0f, + _ => 4.0f, + }; + return new Color32( + (byte)Math.Clamp(lift + sample.x * 255.0f * gain, 0, 255), + (byte)Math.Clamp(lift + sample.y * 255.0f * gain, 0, 255), + (byte)Math.Clamp(lift + sample.z * 255.0f * gain, 0, 255)); + } + + private static float3 StableSample(float x, float y, float resolutionY, int frameIndex, CultMathTone tone, float span) + { + var seed = x * 12.9898f + y * 78.233f + Hash1(((byte)tone + 1.0f) * 19.19f); + var jitter = new float2((Hash1(seed) - 0.5f) * span, (Hash1(seed + 37.17f) - 0.5f) * span); + return SampleField(x + jitter.x, y + jitter.y, resolutionY, frameIndex); + } + + private static float3 SampleField(float px, float py, float resolutionY, int frameIndex) + { + var time = frameIndex / 120.0f; + var p = new float2(6.0f * px / Math.Max(1.0f, resolutionY), 6.0f * py / Math.Max(1.0f, resolutionY)); + var n = math.floor(p); + var f = math.frac(p); + var distance = 8.0f; + var color = float3.zero; + const float smoothness = 0.005f; + for (var j = -2; j <= 2; j++) + { + for (var i = -2; i <= 2; i++) + { + var g = new float2(i, j); + var o = Hash2(n + g); + var weight = o.x * 0.5f + 0.5f; + o = 0.5f + 0.5f * new float2(math.sin(time + 6.2831f * o.x), math.sin(time + 6.2831f * o.y)); + var d2 = math.abs(g - f + o); + var d = Math.Max(d2.x, d2.y) * weight; + var seed = Hash1(math.dot(n + g, new float2(7.0f, 113.0f))); + var hue = math.frac((n.x + g.x) * 0.173f + (n.y + g.y) * 0.379f + seed * 0.431f); + var candidate = PastelSpectrum(hue); + var h = math.smoothstep(0.0f, 1.0f, 0.5f + 0.5f * (distance - d) / smoothness); + var correction = h * (1.0f - h) * smoothness / (1.0f + 3.0f * smoothness); + distance = math.lerp(distance, d, h) - correction; + color = math.lerp(color, candidate, h) - correction; + } + } + + return math.max(float3.zero, color * (1.0f - 0.1f * math.smoothstep(0.04f, 0.05f, distance))); + } + + private static float Hash1(float value) => math.frac(math.sin(value) * 43758.5453f); + + private static float3 PastelSpectrum(float hue) + { + return HsvToRgb(hue, 0.52f, 0.98f); + } + + private static float3 HsvToRgb(float hue, float saturation, float value) + { + var h = math.frac(hue) * 6.0f; + var c = value * saturation; + var x = c * (1.0f - Math.Abs(h % 2.0f - 1.0f)); + var m = value - c; + float3 rgb; + if (h < 1.0f) + { + rgb = new float3(c, x, 0.0f); + } + else if (h < 2.0f) + { + rgb = new float3(x, c, 0.0f); + } + else if (h < 3.0f) + { + rgb = new float3(0.0f, c, x); + } + else if (h < 4.0f) + { + rgb = new float3(0.0f, x, c); + } + else if (h < 5.0f) + { + rgb = new float3(x, 0.0f, c); + } + else + { + rgb = new float3(c, 0.0f, x); + } + + return rgb + m; + } + + private static float2 Hash2(float2 value) + { + var p = new float2(math.dot(value, new float2(127.1f, 311.7f)), math.dot(value, new float2(269.5f, 183.3f))); + return math.frac(new float2(math.sin(p.x) * 43758.5453f, math.sin(p.y) * 43758.5453f)); + } + + private static unsafe partial class Native + { + [DllImport("cultmath_core", CallingConvention = CallingConvention.Cdecl)] + public static extern int cultmath_apollonian_voronoi_tones( + float* xs, + float* ys, + byte* tones, + float* spans, + nuint count, + float resolutionY, + int frameIndex, + Color32* outColors); + } +} diff --git a/packages/cultmath/src/CultMath/Voronoi.cs.meta b/packages/cultmath/src/CultMath/Voronoi.cs.meta new file mode 100644 index 00000000..6c738085 --- /dev/null +++ b/packages/cultmath/src/CultMath/Voronoi.cs.meta @@ -0,0 +1,2 @@ +fileFormatVersion: 2 +guid: cb530f1428ac8ff43939b986986ae959 \ No newline at end of file diff --git a/packages/cultmath/src/CultMath/bool2.cs b/packages/cultmath/src/CultMath/bool2.cs new file mode 100644 index 00000000..29710f0c --- /dev/null +++ b/packages/cultmath/src/CultMath/bool2.cs @@ -0,0 +1,51 @@ +using System; + +namespace CultMath; + +public partial struct bool2 : IEquatable +{ + public static readonly bool2 @false = new(false, false); + public static readonly bool2 @true = new(true, true); + + public bool x; + public bool y; + + public bool2(bool x, bool y) + { + this.x = x; + this.y = y; + } + + public bool this[int index] + { + readonly get => index switch + { + 0 => x, + 1 => y, + _ => throw new ArgumentOutOfRangeException(nameof(index)), + }; + set + { + switch (index) + { + case 0: x = value; break; + case 1: y = value; break; + default: throw new ArgumentOutOfRangeException(nameof(index)); + } + } + } + + public static implicit operator bool2(bool value) => new(value, value); + + public static bool2 operator !(bool2 value) => new(!value.x, !value.y); + public static bool2 operator &(bool2 left, bool2 right) => new(left.x & right.x, left.y & right.y); + public static bool2 operator |(bool2 left, bool2 right) => new(left.x | right.x, left.y | right.y); + public static bool2 operator ^(bool2 left, bool2 right) => new(left.x ^ right.x, left.y ^ right.y); + public static bool2 operator ==(bool2 left, bool2 right) => new(left.x == right.x, left.y == right.y); + public static bool2 operator !=(bool2 left, bool2 right) => new(left.x != right.x, left.y != right.y); + + public readonly bool Equals(bool2 other) => x == other.x && y == other.y; + public override readonly bool Equals(object? obj) => obj is bool2 other && Equals(other); + public override readonly int GetHashCode() => HashCode.Combine(x, y); + public override readonly string ToString() => $"bool2({x}, {y})"; +} diff --git a/packages/cultmath/src/CultMath/bool2.cs.meta b/packages/cultmath/src/CultMath/bool2.cs.meta new file mode 100644 index 00000000..477ff144 --- /dev/null +++ b/packages/cultmath/src/CultMath/bool2.cs.meta @@ -0,0 +1,2 @@ +fileFormatVersion: 2 +guid: 725efb2725e987449a3782452f5a95e0 \ No newline at end of file diff --git a/packages/cultmath/src/CultMath/bool3.cs b/packages/cultmath/src/CultMath/bool3.cs new file mode 100644 index 00000000..9e4029c7 --- /dev/null +++ b/packages/cultmath/src/CultMath/bool3.cs @@ -0,0 +1,55 @@ +using System; + +namespace CultMath; + +public partial struct bool3 : IEquatable +{ + public static readonly bool3 @false = new(false, false, false); + public static readonly bool3 @true = new(true, true, true); + + public bool x; + public bool y; + public bool z; + + public bool3(bool x, bool y, bool z) + { + this.x = x; + this.y = y; + this.z = z; + } + + public bool this[int index] + { + readonly get => index switch + { + 0 => x, + 1 => y, + 2 => z, + _ => throw new ArgumentOutOfRangeException(nameof(index)), + }; + set + { + switch (index) + { + case 0: x = value; break; + case 1: y = value; break; + case 2: z = value; break; + default: throw new ArgumentOutOfRangeException(nameof(index)); + } + } + } + + public static implicit operator bool3(bool value) => new(value, value, value); + + public static bool3 operator !(bool3 value) => new(!value.x, !value.y, !value.z); + public static bool3 operator &(bool3 left, bool3 right) => new(left.x & right.x, left.y & right.y, left.z & right.z); + public static bool3 operator |(bool3 left, bool3 right) => new(left.x | right.x, left.y | right.y, left.z | right.z); + public static bool3 operator ^(bool3 left, bool3 right) => new(left.x ^ right.x, left.y ^ right.y, left.z ^ right.z); + public static bool3 operator ==(bool3 left, bool3 right) => new(left.x == right.x, left.y == right.y, left.z == right.z); + public static bool3 operator !=(bool3 left, bool3 right) => new(left.x != right.x, left.y != right.y, left.z != right.z); + + public readonly bool Equals(bool3 other) => x == other.x && y == other.y && z == other.z; + public override readonly bool Equals(object? obj) => obj is bool3 other && Equals(other); + public override readonly int GetHashCode() => HashCode.Combine(x, y, z); + public override readonly string ToString() => $"bool3({x}, {y}, {z})"; +} diff --git a/packages/cultmath/src/CultMath/bool3.cs.meta b/packages/cultmath/src/CultMath/bool3.cs.meta new file mode 100644 index 00000000..ada81b6d --- /dev/null +++ b/packages/cultmath/src/CultMath/bool3.cs.meta @@ -0,0 +1,2 @@ +fileFormatVersion: 2 +guid: 323ec670fc374c8ba4d87b4bf4c91f34 diff --git a/packages/cultmath/src/CultMath/bool4.cs b/packages/cultmath/src/CultMath/bool4.cs new file mode 100644 index 00000000..b12f15b6 --- /dev/null +++ b/packages/cultmath/src/CultMath/bool4.cs @@ -0,0 +1,59 @@ +using System; + +namespace CultMath; + +public partial struct bool4 : IEquatable +{ + public static readonly bool4 @false = new(false, false, false, false); + public static readonly bool4 @true = new(true, true, true, true); + + public bool x; + public bool y; + public bool z; + public bool w; + + public bool4(bool x, bool y, bool z, bool w) + { + this.x = x; + this.y = y; + this.z = z; + this.w = w; + } + + public bool this[int index] + { + readonly get => index switch + { + 0 => x, + 1 => y, + 2 => z, + 3 => w, + _ => throw new ArgumentOutOfRangeException(nameof(index)), + }; + set + { + switch (index) + { + case 0: x = value; break; + case 1: y = value; break; + case 2: z = value; break; + case 3: w = value; break; + default: throw new ArgumentOutOfRangeException(nameof(index)); + } + } + } + + public static implicit operator bool4(bool value) => new(value, value, value, value); + + public static bool4 operator !(bool4 value) => new(!value.x, !value.y, !value.z, !value.w); + public static bool4 operator &(bool4 left, bool4 right) => new(left.x & right.x, left.y & right.y, left.z & right.z, left.w & right.w); + public static bool4 operator |(bool4 left, bool4 right) => new(left.x | right.x, left.y | right.y, left.z | right.z, left.w | right.w); + public static bool4 operator ^(bool4 left, bool4 right) => new(left.x ^ right.x, left.y ^ right.y, left.z ^ right.z, left.w ^ right.w); + public static bool4 operator ==(bool4 left, bool4 right) => new(left.x == right.x, left.y == right.y, left.z == right.z, left.w == right.w); + public static bool4 operator !=(bool4 left, bool4 right) => new(left.x != right.x, left.y != right.y, left.z != right.z, left.w != right.w); + + public readonly bool Equals(bool4 other) => x == other.x && y == other.y && z == other.z && w == other.w; + public override readonly bool Equals(object? obj) => obj is bool4 other && Equals(other); + public override readonly int GetHashCode() => HashCode.Combine(x, y, z, w); + public override readonly string ToString() => $"bool4({x}, {y}, {z}, {w})"; +} diff --git a/packages/cultmath/src/CultMath/bool4.cs.meta b/packages/cultmath/src/CultMath/bool4.cs.meta new file mode 100644 index 00000000..625bb46b --- /dev/null +++ b/packages/cultmath/src/CultMath/bool4.cs.meta @@ -0,0 +1,2 @@ +fileFormatVersion: 2 +guid: d38b9f70d88c459584441b85ecbfd92a diff --git a/packages/cultmath/src/CultMath/double2.cs b/packages/cultmath/src/CultMath/double2.cs new file mode 100644 index 00000000..958b409d --- /dev/null +++ b/packages/cultmath/src/CultMath/double2.cs @@ -0,0 +1,58 @@ +using System; + +namespace CultMath; + +public partial struct double2 : IEquatable +{ + public static readonly double2 zero = new(0.0, 0.0); + public static readonly double2 one = new(1.0, 1.0); + + public double x; + public double y; + + public double2(double x, double y) + { + this.x = x; + this.y = y; + } + + public double this[int index] + { + readonly get => index switch + { + 0 => x, + 1 => y, + _ => throw new ArgumentOutOfRangeException(nameof(index)), + }; + set + { + switch (index) + { + case 0: x = value; break; + case 1: y = value; break; + default: throw new ArgumentOutOfRangeException(nameof(index)); + } + } + } + + public static implicit operator double2(double value) => new(value, value); + + public static double2 operator +(double2 value) => value; + public static double2 operator -(double2 value) => new(-value.x, -value.y); + public static double2 operator +(double2 left, double2 right) => new(left.x + right.x, left.y + right.y); + public static double2 operator -(double2 left, double2 right) => new(left.x - right.x, left.y - right.y); + public static double2 operator *(double2 left, double2 right) => new(left.x * right.x, left.y * right.y); + public static double2 operator /(double2 left, double2 right) => new(left.x / right.x, left.y / right.y); + + public static bool2 operator <(double2 left, double2 right) => new(left.x < right.x, left.y < right.y); + public static bool2 operator >(double2 left, double2 right) => new(left.x > right.x, left.y > right.y); + public static bool2 operator <=(double2 left, double2 right) => new(left.x <= right.x, left.y <= right.y); + public static bool2 operator >=(double2 left, double2 right) => new(left.x >= right.x, left.y >= right.y); + public static bool2 operator ==(double2 left, double2 right) => new(left.x == right.x, left.y == right.y); + public static bool2 operator !=(double2 left, double2 right) => new(left.x != right.x, left.y != right.y); + + public readonly bool Equals(double2 other) => x.Equals(other.x) && y.Equals(other.y); + public override readonly bool Equals(object? obj) => obj is double2 other && Equals(other); + public override readonly int GetHashCode() => HashCode.Combine(x, y); + public override readonly string ToString() => FormattableString.Invariant($"double2({x:R}, {y:R})"); +} diff --git a/packages/cultmath/src/CultMath/double2.cs.meta b/packages/cultmath/src/CultMath/double2.cs.meta new file mode 100644 index 00000000..61243b2a --- /dev/null +++ b/packages/cultmath/src/CultMath/double2.cs.meta @@ -0,0 +1,2 @@ +fileFormatVersion: 2 +guid: bbe739a5f0216c3459762d2b5ee7401a \ No newline at end of file diff --git a/packages/cultmath/src/CultMath/double3.cs b/packages/cultmath/src/CultMath/double3.cs new file mode 100644 index 00000000..5fbe9c65 --- /dev/null +++ b/packages/cultmath/src/CultMath/double3.cs @@ -0,0 +1,62 @@ +using System; + +namespace CultMath; + +public partial struct double3 : IEquatable +{ + public static readonly double3 zero = new(0.0, 0.0, 0.0); + public static readonly double3 one = new(1.0, 1.0, 1.0); + + public double x; + public double y; + public double z; + + public double3(double x, double y, double z) + { + this.x = x; + this.y = y; + this.z = z; + } + + public double this[int index] + { + readonly get => index switch + { + 0 => x, + 1 => y, + 2 => z, + _ => throw new ArgumentOutOfRangeException(nameof(index)), + }; + set + { + switch (index) + { + case 0: x = value; break; + case 1: y = value; break; + case 2: z = value; break; + default: throw new ArgumentOutOfRangeException(nameof(index)); + } + } + } + + public static implicit operator double3(double value) => new(value, value, value); + + public static double3 operator +(double3 value) => value; + public static double3 operator -(double3 value) => new(-value.x, -value.y, -value.z); + public static double3 operator +(double3 left, double3 right) => new(left.x + right.x, left.y + right.y, left.z + right.z); + public static double3 operator -(double3 left, double3 right) => new(left.x - right.x, left.y - right.y, left.z - right.z); + public static double3 operator *(double3 left, double3 right) => new(left.x * right.x, left.y * right.y, left.z * right.z); + public static double3 operator /(double3 left, double3 right) => new(left.x / right.x, left.y / right.y, left.z / right.z); + + public static bool3 operator <(double3 left, double3 right) => new(left.x < right.x, left.y < right.y, left.z < right.z); + public static bool3 operator >(double3 left, double3 right) => new(left.x > right.x, left.y > right.y, left.z > right.z); + public static bool3 operator <=(double3 left, double3 right) => new(left.x <= right.x, left.y <= right.y, left.z <= right.z); + public static bool3 operator >=(double3 left, double3 right) => new(left.x >= right.x, left.y >= right.y, left.z >= right.z); + public static bool3 operator ==(double3 left, double3 right) => new(left.x == right.x, left.y == right.y, left.z == right.z); + public static bool3 operator !=(double3 left, double3 right) => new(left.x != right.x, left.y != right.y, left.z != right.z); + + public readonly bool Equals(double3 other) => x.Equals(other.x) && y.Equals(other.y) && z.Equals(other.z); + public override readonly bool Equals(object? obj) => obj is double3 other && Equals(other); + public override readonly int GetHashCode() => HashCode.Combine(x, y, z); + public override readonly string ToString() => FormattableString.Invariant($"double3({x:R}, {y:R}, {z:R})"); +} diff --git a/packages/cultmath/src/CultMath/double3.cs.meta b/packages/cultmath/src/CultMath/double3.cs.meta new file mode 100644 index 00000000..302d13d8 --- /dev/null +++ b/packages/cultmath/src/CultMath/double3.cs.meta @@ -0,0 +1,2 @@ +fileFormatVersion: 2 +guid: 1cbaafd219703ee43ae279aa5cbf407f \ No newline at end of file diff --git a/packages/cultmath/src/CultMath/float2.cs b/packages/cultmath/src/CultMath/float2.cs new file mode 100644 index 00000000..e33655a8 --- /dev/null +++ b/packages/cultmath/src/CultMath/float2.cs @@ -0,0 +1,84 @@ +using System; + +namespace CultMath; + +public partial struct float2 : IEquatable +{ + public static readonly float2 zero = new(0.0f, 0.0f); + public static readonly float2 one = new(1.0f, 1.0f); + + public float x; + public float y; + + public float2(float x, float y) + { + this.x = x; + this.y = y; + } + + public float this[int index] + { + readonly get => index switch + { + 0 => x, + 1 => y, + _ => throw new ArgumentOutOfRangeException(nameof(index)), + }; + set + { + switch (index) + { + case 0: x = value; break; + case 1: y = value; break; + default: throw new ArgumentOutOfRangeException(nameof(index)); + } + } + } + + public static implicit operator float2(float value) => new(value, value); + public static explicit operator System.Numerics.Vector2(float2 value) => new(value.x, value.y); + public static explicit operator float2(System.Numerics.Vector2 value) => new(value.X, value.Y); + + public static float2 operator +(float2 value) => value; + public static float2 operator -(float2 value) => new(-value.x, -value.y); + + public static float2 operator +(float2 left, float2 right) => new(left.x + right.x, left.y + right.y); + public static float2 operator +(float2 left, float right) => new(left.x + right, left.y + right); + public static float2 operator +(float left, float2 right) => new(left + right.x, left + right.y); + public static float2 operator -(float2 left, float2 right) => new(left.x - right.x, left.y - right.y); + public static float2 operator -(float2 left, float right) => new(left.x - right, left.y - right); + public static float2 operator -(float left, float2 right) => new(left - right.x, left - right.y); + public static float2 operator *(float2 left, float2 right) => new(left.x * right.x, left.y * right.y); + public static float2 operator *(float2 left, float right) => new(left.x * right, left.y * right); + public static float2 operator *(float left, float2 right) => new(left * right.x, left * right.y); + public static float2 operator /(float2 left, float2 right) => new(left.x / right.x, left.y / right.y); + public static float2 operator /(float2 left, float right) => new(left.x / right, left.y / right); + public static float2 operator /(float left, float2 right) => new(left / right.x, left / right.y); + public static float2 operator %(float2 left, float2 right) => new(left.x % right.x, left.y % right.y); + public static float2 operator %(float2 left, float right) => new(left.x % right, left.y % right); + public static float2 operator %(float left, float2 right) => new(left % right.x, left % right.y); + + public static bool2 operator <(float2 left, float2 right) => new(left.x < right.x, left.y < right.y); + public static bool2 operator <(float2 left, float right) => new(left.x < right, left.y < right); + public static bool2 operator <(float left, float2 right) => new(left < right.x, left < right.y); + public static bool2 operator >(float2 left, float2 right) => new(left.x > right.x, left.y > right.y); + public static bool2 operator >(float2 left, float right) => new(left.x > right, left.y > right); + public static bool2 operator >(float left, float2 right) => new(left > right.x, left > right.y); + public static bool2 operator <=(float2 left, float2 right) => new(left.x <= right.x, left.y <= right.y); + public static bool2 operator <=(float2 left, float right) => new(left.x <= right, left.y <= right); + public static bool2 operator <=(float left, float2 right) => new(left <= right.x, left <= right.y); + public static bool2 operator >=(float2 left, float2 right) => new(left.x >= right.x, left.y >= right.y); + public static bool2 operator >=(float2 left, float right) => new(left.x >= right, left.y >= right); + public static bool2 operator >=(float left, float2 right) => new(left >= right.x, left >= right.y); + public static bool2 operator ==(float2 left, float2 right) => new(left.x == right.x, left.y == right.y); + public static bool2 operator ==(float2 left, float right) => new(left.x == right, left.y == right); + public static bool2 operator ==(float left, float2 right) => new(left == right.x, left == right.y); + public static bool2 operator !=(float2 left, float2 right) => new(left.x != right.x, left.y != right.y); + public static bool2 operator !=(float2 left, float right) => new(left.x != right, left.y != right); + public static bool2 operator !=(float left, float2 right) => new(left != right.x, left != right.y); + + public readonly bool Equals(float2 other) => x.Equals(other.x) && y.Equals(other.y); + public override readonly bool Equals(object? obj) => obj is float2 other && Equals(other); + public override readonly int GetHashCode() => HashCode.Combine(x, y); + public override readonly string ToString() => FormattableString.Invariant($"float2({x:R}, {y:R})"); +} diff --git a/packages/cultmath/src/CultMath/float2.cs.meta b/packages/cultmath/src/CultMath/float2.cs.meta new file mode 100644 index 00000000..c82f69ae --- /dev/null +++ b/packages/cultmath/src/CultMath/float2.cs.meta @@ -0,0 +1,2 @@ +fileFormatVersion: 2 +guid: fa066a0aed5494f499d462a2ebc51e50 \ No newline at end of file diff --git a/packages/cultmath/src/CultMath/float2x2.cs b/packages/cultmath/src/CultMath/float2x2.cs new file mode 100644 index 00000000..bd8673ae --- /dev/null +++ b/packages/cultmath/src/CultMath/float2x2.cs @@ -0,0 +1,67 @@ +using System; +using System.Diagnostics.CodeAnalysis; + +namespace CultMath; + +/// +/// HLSL float2x2: row-major. Constructors take rows, m[i] is a writable reference to +/// row i (so m[1][0] = s works), and _mRC names row R, column C exactly as HLSL +/// does. Use for products; there is no * +/// matrix product because HLSL's * is component-wise. +/// +public struct float2x2 : IEquatable +{ + public static readonly float2x2 identity = new(1.0f, 0.0f, 0.0f, 1.0f); + public static readonly float2x2 zero = new(0.0f, 0.0f, 0.0f, 0.0f); + + private float2 _row0; + private float2 _row1; + + public float2x2(float m00, float m01, float m10, float m11) + { + _row0 = new float2(m00, m01); + _row1 = new float2(m10, m11); + } + + public float2x2(float2 row0, float2 row1) + { + _row0 = row0; + _row1 = row1; + } + + public float _m00 { readonly get => _row0.x; set => _row0.x = value; } + public float _m01 { readonly get => _row0.y; set => _row0.y = value; } + public float _m10 { readonly get => _row1.x; set => _row1.x = value; } + public float _m11 { readonly get => _row1.y; set => _row1.y = value; } + + /// Writes through a readonly field, in parameter, property, or identity hit a defensive copy and are lost; write to a local and assign it back. + [UnscopedRef] + public ref float2 this[int row] + { + get + { + switch (row) + { + case 0: return ref _row0; + case 1: return ref _row1; + default: throw new ArgumentOutOfRangeException(nameof(row)); + } + } + } + + /// + /// Counter-clockwise rotation for column vectors: mul(Rotate(a), v) equals + /// math.rotate(v, a); mul(v, Rotate(a)) rotates the other way. + /// + public static float2x2 Rotate(float radians) + { + var s = MathF.Sin(radians); + var c = MathF.Cos(radians); + return new float2x2(c, -s, s, c); + } + + public readonly bool Equals(float2x2 other) => _row0.Equals(other._row0) && _row1.Equals(other._row1); + public override readonly bool Equals(object? obj) => obj is float2x2 other && Equals(other); + public override readonly int GetHashCode() => HashCode.Combine(_row0, _row1); + public override readonly string ToString() => FormattableString.Invariant($"float2x2({_row0.x:R}, {_row0.y:R}, {_row1.x:R}, {_row1.y:R})"); +} diff --git a/packages/cultmath/src/CultMath/float2x2.cs.meta b/packages/cultmath/src/CultMath/float2x2.cs.meta new file mode 100644 index 00000000..56840df7 --- /dev/null +++ b/packages/cultmath/src/CultMath/float2x2.cs.meta @@ -0,0 +1,2 @@ +fileFormatVersion: 2 +guid: c0dcec14c0b7432dbd8ba0bebb0a7c2f diff --git a/packages/cultmath/src/CultMath/float3.cs b/packages/cultmath/src/CultMath/float3.cs new file mode 100644 index 00000000..ca83628b --- /dev/null +++ b/packages/cultmath/src/CultMath/float3.cs @@ -0,0 +1,88 @@ +using System; + +namespace CultMath; + +public partial struct float3 : IEquatable +{ + public static readonly float3 zero = new(0.0f, 0.0f, 0.0f); + public static readonly float3 one = new(1.0f, 1.0f, 1.0f); + + public float x; + public float y; + public float z; + + public float3(float x, float y, float z) + { + this.x = x; + this.y = y; + this.z = z; + } + + public float this[int index] + { + readonly get => index switch + { + 0 => x, + 1 => y, + 2 => z, + _ => throw new ArgumentOutOfRangeException(nameof(index)), + }; + set + { + switch (index) + { + case 0: x = value; break; + case 1: y = value; break; + case 2: z = value; break; + default: throw new ArgumentOutOfRangeException(nameof(index)); + } + } + } + + public static implicit operator float3(float value) => new(value, value, value); + public static explicit operator System.Numerics.Vector3(float3 value) => new(value.x, value.y, value.z); + public static explicit operator float3(System.Numerics.Vector3 value) => new(value.X, value.Y, value.Z); + + public static float3 operator +(float3 value) => value; + public static float3 operator -(float3 value) => new(-value.x, -value.y, -value.z); + + public static float3 operator +(float3 left, float3 right) => new(left.x + right.x, left.y + right.y, left.z + right.z); + public static float3 operator +(float3 left, float right) => new(left.x + right, left.y + right, left.z + right); + public static float3 operator +(float left, float3 right) => new(left + right.x, left + right.y, left + right.z); + public static float3 operator -(float3 left, float3 right) => new(left.x - right.x, left.y - right.y, left.z - right.z); + public static float3 operator -(float3 left, float right) => new(left.x - right, left.y - right, left.z - right); + public static float3 operator -(float left, float3 right) => new(left - right.x, left - right.y, left - right.z); + public static float3 operator *(float3 left, float3 right) => new(left.x * right.x, left.y * right.y, left.z * right.z); + public static float3 operator *(float3 left, float right) => new(left.x * right, left.y * right, left.z * right); + public static float3 operator *(float left, float3 right) => new(left * right.x, left * right.y, left * right.z); + public static float3 operator /(float3 left, float3 right) => new(left.x / right.x, left.y / right.y, left.z / right.z); + public static float3 operator /(float3 left, float right) => new(left.x / right, left.y / right, left.z / right); + public static float3 operator /(float left, float3 right) => new(left / right.x, left / right.y, left / right.z); + public static float3 operator %(float3 left, float3 right) => new(left.x % right.x, left.y % right.y, left.z % right.z); + public static float3 operator %(float3 left, float right) => new(left.x % right, left.y % right, left.z % right); + public static float3 operator %(float left, float3 right) => new(left % right.x, left % right.y, left % right.z); + + public static bool3 operator <(float3 left, float3 right) => new(left.x < right.x, left.y < right.y, left.z < right.z); + public static bool3 operator <(float3 left, float right) => new(left.x < right, left.y < right, left.z < right); + public static bool3 operator <(float left, float3 right) => new(left < right.x, left < right.y, left < right.z); + public static bool3 operator >(float3 left, float3 right) => new(left.x > right.x, left.y > right.y, left.z > right.z); + public static bool3 operator >(float3 left, float right) => new(left.x > right, left.y > right, left.z > right); + public static bool3 operator >(float left, float3 right) => new(left > right.x, left > right.y, left > right.z); + public static bool3 operator <=(float3 left, float3 right) => new(left.x <= right.x, left.y <= right.y, left.z <= right.z); + public static bool3 operator <=(float3 left, float right) => new(left.x <= right, left.y <= right, left.z <= right); + public static bool3 operator <=(float left, float3 right) => new(left <= right.x, left <= right.y, left <= right.z); + public static bool3 operator >=(float3 left, float3 right) => new(left.x >= right.x, left.y >= right.y, left.z >= right.z); + public static bool3 operator >=(float3 left, float right) => new(left.x >= right, left.y >= right, left.z >= right); + public static bool3 operator >=(float left, float3 right) => new(left >= right.x, left >= right.y, left >= right.z); + public static bool3 operator ==(float3 left, float3 right) => new(left.x == right.x, left.y == right.y, left.z == right.z); + public static bool3 operator ==(float3 left, float right) => new(left.x == right, left.y == right, left.z == right); + public static bool3 operator ==(float left, float3 right) => new(left == right.x, left == right.y, left == right.z); + public static bool3 operator !=(float3 left, float3 right) => new(left.x != right.x, left.y != right.y, left.z != right.z); + public static bool3 operator !=(float3 left, float right) => new(left.x != right, left.y != right, left.z != right); + public static bool3 operator !=(float left, float3 right) => new(left != right.x, left != right.y, left != right.z); + + public readonly bool Equals(float3 other) => x.Equals(other.x) && y.Equals(other.y) && z.Equals(other.z); + public override readonly bool Equals(object? obj) => obj is float3 other && Equals(other); + public override readonly int GetHashCode() => HashCode.Combine(x, y, z); + public override readonly string ToString() => FormattableString.Invariant($"float3({x:R}, {y:R}, {z:R})"); +} diff --git a/packages/cultmath/src/CultMath/float3.cs.meta b/packages/cultmath/src/CultMath/float3.cs.meta new file mode 100644 index 00000000..3c649c4a --- /dev/null +++ b/packages/cultmath/src/CultMath/float3.cs.meta @@ -0,0 +1,2 @@ +fileFormatVersion: 2 +guid: 58a355dd1cf6b9f48a7595eea0c0e1f2 \ No newline at end of file diff --git a/packages/cultmath/src/CultMath/float3x3.cs b/packages/cultmath/src/CultMath/float3x3.cs new file mode 100644 index 00000000..d7aad21c --- /dev/null +++ b/packages/cultmath/src/CultMath/float3x3.cs @@ -0,0 +1,116 @@ +using System; +using System.Diagnostics.CodeAnalysis; + +namespace CultMath; + +/// +/// HLSL float3x3: row-major. Constructors take rows, m[i] is a writable reference to +/// row i (so m[1][2] = s works), and _mRC names row R, column C exactly as HLSL +/// does. Use for products; there is no * +/// matrix product because HLSL's * is component-wise. +/// +public struct float3x3 : IEquatable +{ + public static readonly float3x3 identity = new(1.0f, 0.0f, 0.0f, 0.0f, 1.0f, 0.0f, 0.0f, 0.0f, 1.0f); + public static readonly float3x3 zero = new(0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f); + + private float3 _row0; + private float3 _row1; + private float3 _row2; + + public float3x3(float m00, float m01, float m02, float m10, float m11, float m12, float m20, float m21, float m22) + { + _row0 = new float3(m00, m01, m02); + _row1 = new float3(m10, m11, m12); + _row2 = new float3(m20, m21, m22); + } + + public float3x3(float3 row0, float3 row1, float3 row2) + { + _row0 = row0; + _row1 = row1; + _row2 = row2; + } + + public float _m00 { readonly get => _row0.x; set => _row0.x = value; } + public float _m01 { readonly get => _row0.y; set => _row0.y = value; } + public float _m02 { readonly get => _row0.z; set => _row0.z = value; } + public float _m10 { readonly get => _row1.x; set => _row1.x = value; } + public float _m11 { readonly get => _row1.y; set => _row1.y = value; } + public float _m12 { readonly get => _row1.z; set => _row1.z = value; } + public float _m20 { readonly get => _row2.x; set => _row2.x = value; } + public float _m21 { readonly get => _row2.y; set => _row2.y = value; } + public float _m22 { readonly get => _row2.z; set => _row2.z = value; } + + /// Writes through a readonly field, in parameter, property, or identity hit a defensive copy and are lost; write to a local and assign it back. + [UnscopedRef] + public ref float3 this[int row] + { + get + { + switch (row) + { + case 0: return ref _row0; + case 1: return ref _row1; + case 2: return ref _row2; + default: throw new ArgumentOutOfRangeException(nameof(row)); + } + } + } + + /// Rotation about +X for column vectors (mul(m, v)). + public static float3x3 RotateX(float radians) + { + var s = MathF.Sin(radians); + var c = MathF.Cos(radians); + return new float3x3(1.0f, 0.0f, 0.0f, 0.0f, c, -s, 0.0f, s, c); + } + + /// Rotation about +Y for column vectors (mul(m, v)). + public static float3x3 RotateY(float radians) + { + var s = MathF.Sin(radians); + var c = MathF.Cos(radians); + return new float3x3(c, 0.0f, s, 0.0f, 1.0f, 0.0f, -s, 0.0f, c); + } + + /// Rotation about +Z for column vectors (mul(m, v)). + public static float3x3 RotateZ(float radians) + { + var s = MathF.Sin(radians); + var c = MathF.Cos(radians); + return new float3x3(c, -s, 0.0f, s, c, 0.0f, 0.0f, 0.0f, 1.0f); + } + + /// + /// Extrinsic Euler rotation in radians. The order names the axis applied first: + /// rotates about Y, then X, then Z, so the + /// matrix is mul(RotateZ, mul(RotateX, RotateY)) for column vectors. + /// + public static float3x3 Euler(float3 xyz, math.RotationOrder order = math.RotationOrder.Default) + { + var x = RotateX(xyz.x); + var y = RotateY(xyz.y); + var z = RotateZ(xyz.z); + return order switch + { + math.RotationOrder.XYZ => math.mul(z, math.mul(y, x)), + math.RotationOrder.XZY => math.mul(y, math.mul(z, x)), + math.RotationOrder.YXZ => math.mul(z, math.mul(x, y)), + math.RotationOrder.YZX => math.mul(x, math.mul(z, y)), + math.RotationOrder.ZXY => math.mul(y, math.mul(x, z)), + math.RotationOrder.ZYX => math.mul(x, math.mul(y, z)), + _ => throw new ArgumentOutOfRangeException(nameof(order)), + }; + } + + public static float3x3 Euler(float x, float y, float z, math.RotationOrder order = math.RotationOrder.Default) => + Euler(new float3(x, y, z), order); + + public readonly bool Equals(float3x3 other) => _row0.Equals(other._row0) && _row1.Equals(other._row1) && _row2.Equals(other._row2); + public override readonly bool Equals(object? obj) => obj is float3x3 other && Equals(other); + public override readonly int GetHashCode() => HashCode.Combine(_row0, _row1, _row2); + + public override readonly string ToString() => + FormattableString.Invariant($"float3x3({_row0.x:R}, {_row0.y:R}, {_row0.z:R}, {_row1.x:R}, {_row1.y:R}, {_row1.z:R}, {_row2.x:R}, {_row2.y:R}, {_row2.z:R})"); +} diff --git a/packages/cultmath/src/CultMath/float3x3.cs.meta b/packages/cultmath/src/CultMath/float3x3.cs.meta new file mode 100644 index 00000000..99b25ed3 --- /dev/null +++ b/packages/cultmath/src/CultMath/float3x3.cs.meta @@ -0,0 +1,2 @@ +fileFormatVersion: 2 +guid: 9ca36a9bfff144a9b51478a6d7b19c8e diff --git a/packages/cultmath/src/CultMath/float4.cs b/packages/cultmath/src/CultMath/float4.cs new file mode 100644 index 00000000..3f647971 --- /dev/null +++ b/packages/cultmath/src/CultMath/float4.cs @@ -0,0 +1,92 @@ +using System; + +namespace CultMath; + +public partial struct float4 : IEquatable +{ + public static readonly float4 zero = new(0.0f, 0.0f, 0.0f, 0.0f); + public static readonly float4 one = new(1.0f, 1.0f, 1.0f, 1.0f); + + public float x; + public float y; + public float z; + public float w; + + public float4(float x, float y, float z, float w) + { + this.x = x; + this.y = y; + this.z = z; + this.w = w; + } + + public float this[int index] + { + readonly get => index switch + { + 0 => x, + 1 => y, + 2 => z, + 3 => w, + _ => throw new ArgumentOutOfRangeException(nameof(index)), + }; + set + { + switch (index) + { + case 0: x = value; break; + case 1: y = value; break; + case 2: z = value; break; + case 3: w = value; break; + default: throw new ArgumentOutOfRangeException(nameof(index)); + } + } + } + + public static implicit operator float4(float value) => new(value, value, value, value); + public static explicit operator System.Numerics.Vector4(float4 value) => new(value.x, value.y, value.z, value.w); + public static explicit operator float4(System.Numerics.Vector4 value) => new(value.X, value.Y, value.Z, value.W); + + public static float4 operator +(float4 value) => value; + public static float4 operator -(float4 value) => new(-value.x, -value.y, -value.z, -value.w); + + public static float4 operator +(float4 left, float4 right) => new(left.x + right.x, left.y + right.y, left.z + right.z, left.w + right.w); + public static float4 operator +(float4 left, float right) => new(left.x + right, left.y + right, left.z + right, left.w + right); + public static float4 operator +(float left, float4 right) => new(left + right.x, left + right.y, left + right.z, left + right.w); + public static float4 operator -(float4 left, float4 right) => new(left.x - right.x, left.y - right.y, left.z - right.z, left.w - right.w); + public static float4 operator -(float4 left, float right) => new(left.x - right, left.y - right, left.z - right, left.w - right); + public static float4 operator -(float left, float4 right) => new(left - right.x, left - right.y, left - right.z, left - right.w); + public static float4 operator *(float4 left, float4 right) => new(left.x * right.x, left.y * right.y, left.z * right.z, left.w * right.w); + public static float4 operator *(float4 left, float right) => new(left.x * right, left.y * right, left.z * right, left.w * right); + public static float4 operator *(float left, float4 right) => new(left * right.x, left * right.y, left * right.z, left * right.w); + public static float4 operator /(float4 left, float4 right) => new(left.x / right.x, left.y / right.y, left.z / right.z, left.w / right.w); + public static float4 operator /(float4 left, float right) => new(left.x / right, left.y / right, left.z / right, left.w / right); + public static float4 operator /(float left, float4 right) => new(left / right.x, left / right.y, left / right.z, left / right.w); + public static float4 operator %(float4 left, float4 right) => new(left.x % right.x, left.y % right.y, left.z % right.z, left.w % right.w); + public static float4 operator %(float4 left, float right) => new(left.x % right, left.y % right, left.z % right, left.w % right); + public static float4 operator %(float left, float4 right) => new(left % right.x, left % right.y, left % right.z, left % right.w); + + public static bool4 operator <(float4 left, float4 right) => new(left.x < right.x, left.y < right.y, left.z < right.z, left.w < right.w); + public static bool4 operator <(float4 left, float right) => new(left.x < right, left.y < right, left.z < right, left.w < right); + public static bool4 operator <(float left, float4 right) => new(left < right.x, left < right.y, left < right.z, left < right.w); + public static bool4 operator >(float4 left, float4 right) => new(left.x > right.x, left.y > right.y, left.z > right.z, left.w > right.w); + public static bool4 operator >(float4 left, float right) => new(left.x > right, left.y > right, left.z > right, left.w > right); + public static bool4 operator >(float left, float4 right) => new(left > right.x, left > right.y, left > right.z, left > right.w); + public static bool4 operator <=(float4 left, float4 right) => new(left.x <= right.x, left.y <= right.y, left.z <= right.z, left.w <= right.w); + public static bool4 operator <=(float4 left, float right) => new(left.x <= right, left.y <= right, left.z <= right, left.w <= right); + public static bool4 operator <=(float left, float4 right) => new(left <= right.x, left <= right.y, left <= right.z, left <= right.w); + public static bool4 operator >=(float4 left, float4 right) => new(left.x >= right.x, left.y >= right.y, left.z >= right.z, left.w >= right.w); + public static bool4 operator >=(float4 left, float right) => new(left.x >= right, left.y >= right, left.z >= right, left.w >= right); + public static bool4 operator >=(float left, float4 right) => new(left >= right.x, left >= right.y, left >= right.z, left >= right.w); + public static bool4 operator ==(float4 left, float4 right) => new(left.x == right.x, left.y == right.y, left.z == right.z, left.w == right.w); + public static bool4 operator ==(float4 left, float right) => new(left.x == right, left.y == right, left.z == right, left.w == right); + public static bool4 operator ==(float left, float4 right) => new(left == right.x, left == right.y, left == right.z, left == right.w); + public static bool4 operator !=(float4 left, float4 right) => new(left.x != right.x, left.y != right.y, left.z != right.z, left.w != right.w); + public static bool4 operator !=(float4 left, float right) => new(left.x != right, left.y != right, left.z != right, left.w != right); + public static bool4 operator !=(float left, float4 right) => new(left != right.x, left != right.y, left != right.z, left != right.w); + + public readonly bool Equals(float4 other) => x.Equals(other.x) && y.Equals(other.y) && z.Equals(other.z) && w.Equals(other.w); + public override readonly bool Equals(object? obj) => obj is float4 other && Equals(other); + public override readonly int GetHashCode() => HashCode.Combine(x, y, z, w); + public override readonly string ToString() => FormattableString.Invariant($"float4({x:R}, {y:R}, {z:R}, {w:R})"); +} diff --git a/packages/cultmath/src/CultMath/float4.cs.meta b/packages/cultmath/src/CultMath/float4.cs.meta new file mode 100644 index 00000000..206444e0 --- /dev/null +++ b/packages/cultmath/src/CultMath/float4.cs.meta @@ -0,0 +1,2 @@ +fileFormatVersion: 2 +guid: fe24fbdab33538849bc6be0afbb0a692 \ No newline at end of file diff --git a/packages/cultmath/src/CultMath/int2.cs b/packages/cultmath/src/CultMath/int2.cs new file mode 100644 index 00000000..f690a25d --- /dev/null +++ b/packages/cultmath/src/CultMath/int2.cs @@ -0,0 +1,62 @@ +using System; + +namespace CultMath; + +public partial struct int2 : IEquatable +{ + public static readonly int2 zero = new(0, 0); + public static readonly int2 one = new(1, 1); + + public int x; + public int y; + + public int2(int x, int y) + { + this.x = x; + this.y = y; + } + + public int this[int index] + { + readonly get => index switch + { + 0 => x, + 1 => y, + _ => throw new ArgumentOutOfRangeException(nameof(index)), + }; + set + { + switch (index) + { + case 0: x = value; break; + case 1: y = value; break; + default: throw new ArgumentOutOfRangeException(nameof(index)); + } + } + } + + public static implicit operator int2(int value) => new(value, value); + public static implicit operator float2(int2 value) => new(value.x, value.y); + public static explicit operator System.Numerics.Vector2(int2 value) => new(value.x, value.y); + public static explicit operator int2(System.Numerics.Vector2 value) => new((int)value.X, (int)value.Y); + + public static int2 operator +(int2 value) => value; + public static int2 operator -(int2 value) => new(-value.x, -value.y); + public static int2 operator +(int2 left, int2 right) => new(left.x + right.x, left.y + right.y); + public static int2 operator -(int2 left, int2 right) => new(left.x - right.x, left.y - right.y); + public static int2 operator *(int2 left, int2 right) => new(left.x * right.x, left.y * right.y); + public static int2 operator /(int2 left, int2 right) => new(left.x / right.x, left.y / right.y); + public static int2 operator %(int2 left, int2 right) => new(left.x % right.x, left.y % right.y); + + public static bool2 operator <(int2 left, int2 right) => new(left.x < right.x, left.y < right.y); + public static bool2 operator >(int2 left, int2 right) => new(left.x > right.x, left.y > right.y); + public static bool2 operator <=(int2 left, int2 right) => new(left.x <= right.x, left.y <= right.y); + public static bool2 operator >=(int2 left, int2 right) => new(left.x >= right.x, left.y >= right.y); + public static bool2 operator ==(int2 left, int2 right) => new(left.x == right.x, left.y == right.y); + public static bool2 operator !=(int2 left, int2 right) => new(left.x != right.x, left.y != right.y); + + public readonly bool Equals(int2 other) => x == other.x && y == other.y; + public override readonly bool Equals(object? obj) => obj is int2 other && Equals(other); + public override readonly int GetHashCode() => HashCode.Combine(x, y); + public override readonly string ToString() => FormattableString.Invariant($"int2({x}, {y})"); +} diff --git a/packages/cultmath/src/CultMath/int2.cs.meta b/packages/cultmath/src/CultMath/int2.cs.meta new file mode 100644 index 00000000..5920328a --- /dev/null +++ b/packages/cultmath/src/CultMath/int2.cs.meta @@ -0,0 +1,2 @@ +fileFormatVersion: 2 +guid: 577b174a998aaa4478ec26e7a6782f46 \ No newline at end of file diff --git a/packages/cultmath/src/CultMath/int3.cs b/packages/cultmath/src/CultMath/int3.cs new file mode 100644 index 00000000..454a8970 --- /dev/null +++ b/packages/cultmath/src/CultMath/int3.cs @@ -0,0 +1,64 @@ +using System; + +namespace CultMath; + +public partial struct int3 : IEquatable +{ + public static readonly int3 zero = new(0, 0, 0); + public static readonly int3 one = new(1, 1, 1); + + public int x; + public int y; + public int z; + + public int3(int x, int y, int z) + { + this.x = x; + this.y = y; + this.z = z; + } + + public int this[int index] + { + readonly get => index switch + { + 0 => x, + 1 => y, + 2 => z, + _ => throw new ArgumentOutOfRangeException(nameof(index)), + }; + set + { + switch (index) + { + case 0: x = value; break; + case 1: y = value; break; + case 2: z = value; break; + default: throw new ArgumentOutOfRangeException(nameof(index)); + } + } + } + + public static implicit operator int3(int value) => new(value, value, value); + public static implicit operator float3(int3 value) => new(value.x, value.y, value.z); + + public static int3 operator +(int3 value) => value; + public static int3 operator -(int3 value) => new(-value.x, -value.y, -value.z); + public static int3 operator +(int3 left, int3 right) => new(left.x + right.x, left.y + right.y, left.z + right.z); + public static int3 operator -(int3 left, int3 right) => new(left.x - right.x, left.y - right.y, left.z - right.z); + public static int3 operator *(int3 left, int3 right) => new(left.x * right.x, left.y * right.y, left.z * right.z); + public static int3 operator /(int3 left, int3 right) => new(left.x / right.x, left.y / right.y, left.z / right.z); + public static int3 operator %(int3 left, int3 right) => new(left.x % right.x, left.y % right.y, left.z % right.z); + + public static bool3 operator <(int3 left, int3 right) => new(left.x < right.x, left.y < right.y, left.z < right.z); + public static bool3 operator >(int3 left, int3 right) => new(left.x > right.x, left.y > right.y, left.z > right.z); + public static bool3 operator <=(int3 left, int3 right) => new(left.x <= right.x, left.y <= right.y, left.z <= right.z); + public static bool3 operator >=(int3 left, int3 right) => new(left.x >= right.x, left.y >= right.y, left.z >= right.z); + public static bool3 operator ==(int3 left, int3 right) => new(left.x == right.x, left.y == right.y, left.z == right.z); + public static bool3 operator !=(int3 left, int3 right) => new(left.x != right.x, left.y != right.y, left.z != right.z); + + public readonly bool Equals(int3 other) => x == other.x && y == other.y && z == other.z; + public override readonly bool Equals(object? obj) => obj is int3 other && Equals(other); + public override readonly int GetHashCode() => HashCode.Combine(x, y, z); + public override readonly string ToString() => FormattableString.Invariant($"int3({x}, {y}, {z})"); +} diff --git a/packages/cultmath/src/CultMath/int3.cs.meta b/packages/cultmath/src/CultMath/int3.cs.meta new file mode 100644 index 00000000..d0400303 --- /dev/null +++ b/packages/cultmath/src/CultMath/int3.cs.meta @@ -0,0 +1,2 @@ +fileFormatVersion: 2 +guid: 42ee98aace354db19b3ae87bdcbc1df9 diff --git a/packages/cultmath/src/CultMath/int4.cs b/packages/cultmath/src/CultMath/int4.cs new file mode 100644 index 00000000..caaaeb9e --- /dev/null +++ b/packages/cultmath/src/CultMath/int4.cs @@ -0,0 +1,68 @@ +using System; + +namespace CultMath; + +public partial struct int4 : IEquatable +{ + public static readonly int4 zero = new(0, 0, 0, 0); + public static readonly int4 one = new(1, 1, 1, 1); + + public int x; + public int y; + public int z; + public int w; + + public int4(int x, int y, int z, int w) + { + this.x = x; + this.y = y; + this.z = z; + this.w = w; + } + + public int this[int index] + { + readonly get => index switch + { + 0 => x, + 1 => y, + 2 => z, + 3 => w, + _ => throw new ArgumentOutOfRangeException(nameof(index)), + }; + set + { + switch (index) + { + case 0: x = value; break; + case 1: y = value; break; + case 2: z = value; break; + case 3: w = value; break; + default: throw new ArgumentOutOfRangeException(nameof(index)); + } + } + } + + public static implicit operator int4(int value) => new(value, value, value, value); + public static implicit operator float4(int4 value) => new(value.x, value.y, value.z, value.w); + + public static int4 operator +(int4 value) => value; + public static int4 operator -(int4 value) => new(-value.x, -value.y, -value.z, -value.w); + public static int4 operator +(int4 left, int4 right) => new(left.x + right.x, left.y + right.y, left.z + right.z, left.w + right.w); + public static int4 operator -(int4 left, int4 right) => new(left.x - right.x, left.y - right.y, left.z - right.z, left.w - right.w); + public static int4 operator *(int4 left, int4 right) => new(left.x * right.x, left.y * right.y, left.z * right.z, left.w * right.w); + public static int4 operator /(int4 left, int4 right) => new(left.x / right.x, left.y / right.y, left.z / right.z, left.w / right.w); + public static int4 operator %(int4 left, int4 right) => new(left.x % right.x, left.y % right.y, left.z % right.z, left.w % right.w); + + public static bool4 operator <(int4 left, int4 right) => new(left.x < right.x, left.y < right.y, left.z < right.z, left.w < right.w); + public static bool4 operator >(int4 left, int4 right) => new(left.x > right.x, left.y > right.y, left.z > right.z, left.w > right.w); + public static bool4 operator <=(int4 left, int4 right) => new(left.x <= right.x, left.y <= right.y, left.z <= right.z, left.w <= right.w); + public static bool4 operator >=(int4 left, int4 right) => new(left.x >= right.x, left.y >= right.y, left.z >= right.z, left.w >= right.w); + public static bool4 operator ==(int4 left, int4 right) => new(left.x == right.x, left.y == right.y, left.z == right.z, left.w == right.w); + public static bool4 operator !=(int4 left, int4 right) => new(left.x != right.x, left.y != right.y, left.z != right.z, left.w != right.w); + + public readonly bool Equals(int4 other) => x == other.x && y == other.y && z == other.z && w == other.w; + public override readonly bool Equals(object? obj) => obj is int4 other && Equals(other); + public override readonly int GetHashCode() => HashCode.Combine(x, y, z, w); + public override readonly string ToString() => FormattableString.Invariant($"int4({x}, {y}, {z}, {w})"); +} diff --git a/packages/cultmath/src/CultMath/int4.cs.meta b/packages/cultmath/src/CultMath/int4.cs.meta new file mode 100644 index 00000000..48d0eaba --- /dev/null +++ b/packages/cultmath/src/CultMath/int4.cs.meta @@ -0,0 +1,2 @@ +fileFormatVersion: 2 +guid: 1a313709f76a430eb83901bb2d05c74e diff --git a/packages/cultmath/src/CultMath/math.cs b/packages/cultmath/src/CultMath/math.cs new file mode 100644 index 00000000..ca74857f --- /dev/null +++ b/packages/cultmath/src/CultMath/math.cs @@ -0,0 +1,623 @@ +using System; + +namespace CultMath; + +public static partial class math +{ + public const float PI = MathF.PI; + public const float TAU = MathF.PI * 2.0f; + public const float HALF_PI = MathF.PI * 0.5f; + + /// + /// Extrinsic Euler order for : + /// the first letter is the axis applied first. Nested here so using static CultMath.math; + /// brings it into scope with the rest of the shader vocabulary. + /// + public enum RotationOrder : byte + { + XYZ, + XZY, + YXZ, + YZX, + ZXY, + ZYX, + Default = ZXY, + } + + public static float2 float2(float x, float y) => new(x, y); + public static float2 float2(float value) => new(value, value); + public static bool2 bool2(bool x, bool y) => new(x, y); + public static bool2 bool2(bool value) => new(value, value); + public static int2 int2(int x, int y) => new(x, y); + public static int2 int2(int value) => new(value, value); + // Mixed constructor functions (float4(float2, float2), ...) are generated in Swizzles.g.cs. + public static float3 float3(float x, float y, float z) => new(x, y, z); + public static float3 float3(float value) => new(value, value, value); + public static float4 float4(float x, float y, float z, float w) => new(x, y, z, w); + public static float4 float4(float value) => new(value, value, value, value); + public static bool3 bool3(bool x, bool y, bool z) => new(x, y, z); + public static bool3 bool3(bool value) => new(value, value, value); + public static bool4 bool4(bool x, bool y, bool z, bool w) => new(x, y, z, w); + public static bool4 bool4(bool value) => new(value, value, value, value); + public static int3 int3(int x, int y, int z) => new(x, y, z); + public static int3 int3(int value) => new(value, value, value); + public static int4 int4(int x, int y, int z, int w) => new(x, y, z, w); + public static int4 int4(int value) => new(value, value, value, value); + + // Matrix constructor functions take rows, as HLSL does. Under `using static CultMath.math;` + // these method groups hide the type names in member access, so static members need a + // qualified type: `CultMath.float3x3.Euler(...)`, `CultMath.float2x2.Rotate(...)`. + public static float2x2 float2x2(float m00, float m01, float m10, float m11) => new(m00, m01, m10, m11); + public static float2x2 float2x2(float2 row0, float2 row1) => new(row0, row1); + public static float3x3 float3x3(float m00, float m01, float m02, float m10, float m11, float m12, float m20, float m21, float m22) => + new(m00, m01, m02, m10, m11, m12, m20, m21, m22); + public static float3x3 float3x3(float3 row0, float3 row1, float3 row2) => new(row0, row1, row2); + public static float radians(float degrees) => degrees * (PI / 180.0f); + public static float degrees(float radians) => radians * (180.0f / PI); + public static float sin(float value) => MathF.Sin(value); + public static float cos(float value) => MathF.Cos(value); + public static float tan(float value) => MathF.Tan(value); + public static float asin(float value) => MathF.Asin(value); + public static float acos(float value) => MathF.Acos(value); + public static float atan(float value) => MathF.Atan(value); + public static float atan2(float y, float x) => MathF.Atan2(y, x); + public static float sqrt(float value) => MathF.Sqrt(value); + public static float pow(float value, float power) => MathF.Pow(value, power); + public static bool isinf(float value) => float.IsInfinity(value); + public static float exp(float value) => MathF.Exp(value); + public static double exp(double value) => Math.Exp(value); + public static float2 exp(float2 value) => new(exp(value.x), exp(value.y)); + public static float3 exp(float3 value) => new(exp(value.x), exp(value.y), exp(value.z)); + public static float4 exp(float4 value) => new(exp(value.x), exp(value.y), exp(value.z), exp(value.w)); + public static double2 exp(double2 value) => new(exp(value.x), exp(value.y)); + public static double3 exp(double3 value) => new(exp(value.x), exp(value.y), exp(value.z)); + + public static float abs(float value) => MathF.Abs(value); + public static float2 abs(float2 value) => new(abs(value.x), abs(value.y)); + public static float3 abs(float3 value) => new(abs(value.x), abs(value.y), abs(value.z)); + public static float4 abs(float4 value) => new(abs(value.x), abs(value.y), abs(value.z), abs(value.w)); + public static int abs(int value) => Math.Abs(value); + public static int2 abs(int2 value) => new(abs(value.x), abs(value.y)); + public static int3 abs(int3 value) => new(abs(value.x), abs(value.y), abs(value.z)); + public static int4 abs(int4 value) => new(abs(value.x), abs(value.y), abs(value.z), abs(value.w)); + + // HLSL sign returns int. NaN compares false both ways, so it yields 0 instead of throwing. + public static int sign(float value) => value > 0.0f ? 1 : value < 0.0f ? -1 : 0; + public static int2 sign(float2 value) => new(sign(value.x), sign(value.y)); + public static int3 sign(float3 value) => new(sign(value.x), sign(value.y), sign(value.z)); + public static int4 sign(float4 value) => new(sign(value.x), sign(value.y), sign(value.z), sign(value.w)); + public static int sign(int value) => value > 0 ? 1 : value < 0 ? -1 : 0; + public static int2 sign(int2 value) => new(sign(value.x), sign(value.y)); + public static int3 sign(int3 value) => new(sign(value.x), sign(value.y), sign(value.z)); + public static int4 sign(int4 value) => new(sign(value.x), sign(value.y), sign(value.z), sign(value.w)); + + public static float floor(float value) => MathF.Floor(value); + public static float2 floor(float2 value) => new(floor(value.x), floor(value.y)); + public static float3 floor(float3 value) => new(floor(value.x), floor(value.y), floor(value.z)); + public static float4 floor(float4 value) => new(floor(value.x), floor(value.y), floor(value.z), floor(value.w)); + + public static float ceil(float value) => MathF.Ceiling(value); + public static float2 ceil(float2 value) => new(ceil(value.x), ceil(value.y)); + public static float3 ceil(float3 value) => new(ceil(value.x), ceil(value.y), ceil(value.z)); + public static float4 ceil(float4 value) => new(ceil(value.x), ceil(value.y), ceil(value.z), ceil(value.w)); + + public static float frac(float value) => value - floor(value); + public static float2 frac(float2 value) => value - floor(value); + public static float3 frac(float3 value) => value - floor(value); + public static float4 frac(float4 value) => value - floor(value); + + // dxc lowers float min/max/clamp to DXIL FMin/FMax (saturate to Saturate, defined as FMin(1, FMax(0, x))). + // DXIL.rst: FMin(a, b) is a < b ? a : b, FMax(a, b) is a >= b ? a : b, and a NaN operand returns the other. + public static float min(float left, float right) => left < right || float.IsNaN(right) ? left : right; + public static int min(int left, int right) => Math.Min(left, right); + public static float2 min(float2 left, float2 right) => new(min(left.x, right.x), min(left.y, right.y)); + public static float3 min(float3 left, float3 right) => new(min(left.x, right.x), min(left.y, right.y), min(left.z, right.z)); + public static float4 min(float4 left, float4 right) => new(min(left.x, right.x), min(left.y, right.y), min(left.z, right.z), min(left.w, right.w)); + public static int2 min(int2 left, int2 right) => new(min(left.x, right.x), min(left.y, right.y)); + public static int3 min(int3 left, int3 right) => new(min(left.x, right.x), min(left.y, right.y), min(left.z, right.z)); + public static int4 min(int4 left, int4 right) => new(min(left.x, right.x), min(left.y, right.y), min(left.z, right.z), min(left.w, right.w)); + + public static float max(float left, float right) => left >= right || float.IsNaN(right) ? left : right; + public static int max(int left, int right) => Math.Max(left, right); + public static float2 max(float2 left, float2 right) => new(max(left.x, right.x), max(left.y, right.y)); + public static float3 max(float3 left, float3 right) => new(max(left.x, right.x), max(left.y, right.y), max(left.z, right.z)); + public static float4 max(float4 left, float4 right) => new(max(left.x, right.x), max(left.y, right.y), max(left.z, right.z), max(left.w, right.w)); + public static int2 max(int2 left, int2 right) => new(max(left.x, right.x), max(left.y, right.y)); + public static int3 max(int3 left, int3 right) => new(max(left.x, right.x), max(left.y, right.y), max(left.z, right.z)); + public static int4 max(int4 left, int4 right) => new(max(left.x, right.x), max(left.y, right.y), max(left.z, right.z), max(left.w, right.w)); + + public static float clamp(float value, float minimum, float maximum) => min(max(value, minimum), maximum); + public static float2 clamp(float2 value, float2 minimum, float2 maximum) => min(max(value, minimum), maximum); + public static float3 clamp(float3 value, float3 minimum, float3 maximum) => min(max(value, minimum), maximum); + public static float4 clamp(float4 value, float4 minimum, float4 maximum) => min(max(value, minimum), maximum); + + // DXIL Saturate is FMin(1, FMax(0, x)); that operand order, unlike clamp(x, 0, 1), maps -0 to +0. + public static float saturate(float value) => min(1.0f, max(0.0f, value)); + public static float2 saturate(float2 value) => min(1.0f, max(0.0f, value)); + public static float3 saturate(float3 value) => min(1.0f, max(0.0f, value)); + public static float4 saturate(float4 value) => min(1.0f, max(0.0f, value)); + + public static float lerp(float start, float end, float amount) => start + (end - start) * amount; + public static double lerp(double start, double end, double amount) => start + (end - start) * amount; + public static float2 lerp(float2 start, float2 end, float2 amount) => start + (end - start) * amount; + public static float3 lerp(float3 start, float3 end, float3 amount) => start + (end - start) * amount; + public static float4 lerp(float4 start, float4 end, float4 amount) => start + (end - start) * amount; + public static double2 lerp(double2 start, double2 end, double2 amount) => start + (end - start) * amount; + public static double3 lerp(double3 start, double3 end, double3 amount) => start + (end - start) * amount; + + public static float unlerp(float start, float end, float value) => (value - start) / (end - start); + public static double unlerp(double start, double end, double value) => (value - start) / (end - start); + public static float2 unlerp(float2 start, float2 end, float2 value) => (value - start) / (end - start); + public static float3 unlerp(float3 start, float3 end, float3 value) => (value - start) / (end - start); + public static float4 unlerp(float4 start, float4 end, float4 value) => (value - start) / (end - start); + public static double2 unlerp(double2 start, double2 end, double2 value) => (value - start) / (end - start); + public static double3 unlerp(double3 start, double3 end, double3 value) => (value - start) / (end - start); + + // dxc lowers step(y, x) to x < y ? 0 : 1 (fcmp olt, then select), so a NaN on either side yields 1. + public static float step(float edge, float value) => value < edge ? 0.0f : 1.0f; + public static float2 step(float2 edge, float2 value) => new(step(edge.x, value.x), step(edge.y, value.y)); + public static float3 step(float3 edge, float3 value) => new(step(edge.x, value.x), step(edge.y, value.y), step(edge.z, value.z)); + public static float4 step(float4 edge, float4 value) => new(step(edge.x, value.x), step(edge.y, value.y), step(edge.z, value.z), step(edge.w, value.w)); + + public static float smoothstep(float minimum, float maximum, float value) + { + var t = saturate((value - minimum) / (maximum - minimum)); + return t * t * (3.0f - 2.0f * t); + } + + public static float2 smoothstep(float2 minimum, float2 maximum, float2 value) + { + var t = saturate((value - minimum) / (maximum - minimum)); + return t * t * (3.0f - 2.0f * t); + } + + public static float3 smoothstep(float3 minimum, float3 maximum, float3 value) + { + var t = saturate((value - minimum) / (maximum - minimum)); + return t * t * (3.0f - 2.0f * t); + } + + public static float4 smoothstep(float4 minimum, float4 maximum, float4 value) + { + var t = saturate((value - minimum) / (maximum - minimum)); + return t * t * (3.0f - 2.0f * t); + } + + public static float dot(float2 left, float2 right) => left.x * right.x + left.y * right.y; + public static float dot(float3 left, float3 right) => left.x * right.x + left.y * right.y + left.z * right.z; + public static float dot(float4 left, float4 right) => left.x * right.x + left.y * right.y + left.z * right.z + left.w * right.w; + + public static float3 cross(float3 left, float3 right) => + new( + left.y * right.z - left.z * right.y, + left.z * right.x - left.x * right.z, + left.x * right.y - left.y * right.x); + + public static float lengthsq(float2 value) => dot(value, value); + public static float lengthsq(float3 value) => dot(value, value); + public static float lengthsq(float4 value) => dot(value, value); + + public static float length(float2 value) => MathF.Sqrt(lengthsq(value)); + public static float length(float3 value) => MathF.Sqrt(lengthsq(value)); + public static float length(float4 value) => MathF.Sqrt(lengthsq(value)); + + public static float distance(float2 left, float2 right) => length(left - right); + public static float distance(float3 left, float3 right) => length(left - right); + public static float distance(float4 left, float4 right) => length(left - right); + + public static float2 normalize(float2 value) => value / max(length(value), 1.0e-20f); + public static float3 normalize(float3 value) => value / max(length(value), 1.0e-20f); + public static float4 normalize(float4 value) => value / max(length(value), 1.0e-20f); + public static quaternion normalize(quaternion value) + { + var length = MathF.Sqrt(value.x * value.x + value.y * value.y + value.z * value.z + value.w * value.w); + if (length <= 1.0e-20f) + { + return quaternion.identity; + } + + var invLength = 1.0f / length; + return new quaternion(value.x * invLength, value.y * invLength, value.z * invLength, value.w * invLength); + } + + public static float2 reflect(float2 incident, float2 normal) => incident - 2.0f * dot(normal, incident) * normal; + public static float3 reflect(float3 incident, float3 normal) => incident - 2.0f * dot(normal, incident) * normal; + + public static float2 rotate(float2 value, float radians) + { + var s = sin(radians); + var c = cos(radians); + return new float2(value.x * c - value.y * s, value.x * s + value.y * c); + } + + public static float2 rotate_degrees(float2 value, float degrees) => rotate(value, radians(degrees)); + + public static float csum(float2 value) => value.x + value.y; + public static float csum(float3 value) => value.x + value.y + value.z; + public static float csum(float4 value) => value.x + value.y + value.z + value.w; + + public static float decay(float source, float lambda, float dt) => source * exp(-lambda * dt); + public static float2 decay(float2 source, float lambda, float dt) => source * exp(-lambda * dt); + public static float3 decay(float3 source, float lambda, float dt) => source * exp(-lambda * dt); + public static float4 decay(float4 source, float lambda, float dt) => source * exp(-lambda * dt); + public static float2 decay(float2 source, float2 lambda, float dt) => source * exp(-lambda * dt); + public static float3 decay(float3 source, float3 lambda, float dt) => source * exp(-lambda * dt); + public static float4 decay(float4 source, float4 lambda, float dt) => source * exp(-lambda * dt); + + public static float damp(float start, float end, float lambda, float dt) => lerp(start, end, 1.0f - exp(-lambda * dt)); + public static float2 damp(float2 start, float2 end, float lambda, float dt) => lerp(start, end, 1.0f - exp(-lambda * dt)); + public static float3 damp(float3 start, float3 end, float lambda, float dt) => lerp(start, end, 1.0f - exp(-lambda * dt)); + public static float4 damp(float4 start, float4 end, float lambda, float dt) => lerp(start, end, 1.0f - exp(-lambda * dt)); + + public static float first_order_intercept_time(float shotSpeed, float3 targetRelativePosition, float3 targetRelativeVelocity) + { + var velocitySquared = lengthsq(targetRelativeVelocity); + if (velocitySquared < 0.001f) + { + return 0.0f; + } + + var a = velocitySquared - shotSpeed * shotSpeed; + if (abs(a) < 0.001f) + { + var time = -lengthsq(targetRelativePosition) / (2.0f * dot(targetRelativeVelocity, targetRelativePosition)); + return max(time, 0.0f); + } + + var b = 2.0f * dot(targetRelativeVelocity, targetRelativePosition); + var c = lengthsq(targetRelativePosition); + var determinant = b * b - 4.0f * a * c; + + if (determinant > 0.0f) + { + var root = sqrt(determinant); + var t1 = (-b + root) / (2.0f * a); + var t2 = (-b - root) / (2.0f * a); + if (t1 > 0.0f) + { + return t2 > 0.0f ? min(t1, t2) : t1; + } + + return max(t2, 0.0f); + } + + if (determinant < 0.0f) + { + return 0.0f; + } + + return max(-b / (2.0f * a), 0.0f); + } + + public static float3 first_order_intercept( + float3 shooterPosition, + float3 shooterVelocity, + float shotSpeed, + float3 targetPosition, + float3 targetVelocity) + { + var targetRelativePosition = targetPosition - shooterPosition; + var targetRelativeVelocity = targetVelocity - shooterVelocity; + var time = first_order_intercept_time(shotSpeed, targetRelativePosition, targetRelativeVelocity); + return targetPosition + time * targetRelativeVelocity; + } + + public static float distance_to_segment(float2 point, float2 start, float2 end, out float2 closest) + { + var segment = end - start; + var segmentLengthSq = lengthsq(segment); + if (segmentLengthSq <= 0.0f) + { + closest = start; + return distance(point, start); + } + + var t = saturate(dot(point - start, segment) / segmentLengthSq); + closest = start + segment * t; + return distance(point, closest); + } + + public static float catmullrom(float p0, float p1, float p2, float p3, float t) + { + var t2 = t * t; + var t3 = t2 * t; + return 0.5f * ((2.0f * p1) + (p2 - p0) * t + (2.0f * p0 - 5.0f * p1 + 4.0f * p2 - p3) * t2 + (-p0 + 3.0f * p1 - 3.0f * p2 + p3) * t3); + } + + public static float2 catmullrom(float2 p0, float2 p1, float2 p2, float2 p3, float t) => + 0.5f * ((2.0f * p1) + (p2 - p0) * t + (2.0f * p0 - 5.0f * p1 + 4.0f * p2 - p3) * (t * t) + (-p0 + 3.0f * p1 - 3.0f * p2 + p3) * (t * t * t)); + + public static float3 catmullrom(float3 p0, float3 p1, float3 p2, float3 p3, float t) => + 0.5f * ((2.0f * p1) + (p2 - p0) * t + (2.0f * p0 - 5.0f * p1 + 4.0f * p2 - p3) * (t * t) + (-p0 + 3.0f * p1 - 3.0f * p2 + p3) * (t * t * t)); + + public static float4 catmullrom(float4 p0, float4 p1, float4 p2, float4 p3, float t) => + 0.5f * ((2.0f * p1) + (p2 - p0) * t + (2.0f * p0 - 5.0f * p1 + 4.0f * p2 - p3) * (t * t) + (-p0 + 3.0f * p1 - 3.0f * p2 + p3) * (t * t * t)); + + public static float quadratic_bezier(float p0, float p1, float p2, float t) + { + t = saturate(t); + var inv = 1.0f - t; + return inv * inv * p0 + 2.0f * inv * t * p1 + t * t * p2; + } + + public static float3 quadratic_bezier(float3 p0, float3 p1, float3 p2, float t) + { + t = saturate(t); + var inv = 1.0f - t; + return inv * inv * p0 + 2.0f * inv * t * p1 + t * t * p2; + } + + public static float cubic_bezier(float p0, float p1, float p2, float p3, float t) + { + t = saturate(t); + var inv = 1.0f - t; + return inv * inv * inv * p0 + 3.0f * inv * inv * t * p1 + 3.0f * inv * t * t * p2 + t * t * t * p3; + } + + public static float3 cubic_bezier(float3 p0, float3 p1, float3 p2, float3 p3, float t) + { + t = saturate(t); + var inv = 1.0f - t; + return inv * inv * inv * p0 + 3.0f * inv * inv * t * p1 + 3.0f * inv * t * t * p2 + t * t * t * p3; + } + + public static float smoothstep01(float value) => smoothstep(0.0f, 1.0f, value); + + public static float smootherstep(float value) + { + value = saturate(value); + return value * value * value * (value * (value * 6.0f - 15.0f) + 10.0f); + } + + public static float hash(float value) => frac(sin(value) * 43758.5453f); + public static float hash(float2 value) => hash(dot(value, new float2(127.1f, 311.7f))); + public static float hash(float3 value) => hash(dot(value, new float3(127.1f, 311.7f, 74.7f))); + + public static float log(float value) => MathF.Log(value); + public static float2 log(float2 value) => new(log(value.x), log(value.y)); + public static float3 log(float3 value) => new(log(value.x), log(value.y), log(value.z)); + public static float4 log(float4 value) => new(log(value.x), log(value.y), log(value.z), log(value.w)); + + public static float2 pow(float2 value, float2 power) => new(pow(value.x, power.x), pow(value.y, power.y)); + public static float3 pow(float3 value, float3 power) => new(pow(value.x, power.x), pow(value.y, power.y), pow(value.z, power.z)); + public static float4 pow(float4 value, float4 power) => new(pow(value.x, power.x), pow(value.y, power.y), pow(value.z, power.z), pow(value.w, power.w)); + + public static float2 sqrt(float2 value) => new(sqrt(value.x), sqrt(value.y)); + public static float3 sqrt(float3 value) => new(sqrt(value.x), sqrt(value.y), sqrt(value.z)); + public static float4 sqrt(float4 value) => new(sqrt(value.x), sqrt(value.y), sqrt(value.z), sqrt(value.w)); + + public static float2 sin(float2 value) => new(sin(value.x), sin(value.y)); + public static float3 sin(float3 value) => new(sin(value.x), sin(value.y), sin(value.z)); + public static float4 sin(float4 value) => new(sin(value.x), sin(value.y), sin(value.z), sin(value.w)); + + public static float2 cos(float2 value) => new(cos(value.x), cos(value.y)); + public static float3 cos(float3 value) => new(cos(value.x), cos(value.y), cos(value.z)); + public static float4 cos(float4 value) => new(cos(value.x), cos(value.y), cos(value.z), cos(value.w)); + + public static float2 acos(float2 value) => new(acos(value.x), acos(value.y)); + public static float3 acos(float3 value) => new(acos(value.x), acos(value.y), acos(value.z)); + public static float4 acos(float4 value) => new(acos(value.x), acos(value.y), acos(value.z), acos(value.w)); + + public static float2 atan2(float2 y, float2 x) => new(atan2(y.x, x.x), atan2(y.y, x.y)); + public static float3 atan2(float3 y, float3 x) => new(atan2(y.x, x.x), atan2(y.y, x.y), atan2(y.z, x.z)); + public static float4 atan2(float4 y, float4 x) => new(atan2(y.x, x.x), atan2(y.y, x.y), atan2(y.z, x.z), atan2(y.w, x.w)); + + // DXIL FMin/FMax take double overloads with the same rules as float (see min/max above). + public static double min(double left, double right) => left < right || double.IsNaN(right) ? left : right; + public static double max(double left, double right) => left >= right || double.IsNaN(right) ? left : right; + + public static bool any(bool2 value) => value.x || value.y; + public static bool any(bool3 value) => value.x || value.y || value.z; + public static bool any(bool4 value) => value.x || value.y || value.z || value.w; + public static bool all(bool2 value) => value.x && value.y; + public static bool all(bool3 value) => value.x && value.y && value.z; + public static bool all(bool4 value) => value.x && value.y && value.z && value.w; + + // HLSL any/all on numeric vectors test components against zero (NaN != 0 counts as true). + public static bool any(float2 value) => value.x != 0.0f || value.y != 0.0f; + public static bool any(float3 value) => value.x != 0.0f || value.y != 0.0f || value.z != 0.0f; + public static bool any(float4 value) => value.x != 0.0f || value.y != 0.0f || value.z != 0.0f || value.w != 0.0f; + public static bool all(float2 value) => value.x != 0.0f && value.y != 0.0f; + public static bool all(float3 value) => value.x != 0.0f && value.y != 0.0f && value.z != 0.0f; + public static bool all(float4 value) => value.x != 0.0f && value.y != 0.0f && value.z != 0.0f && value.w != 0.0f; + public static bool any(int2 value) => value.x != 0 || value.y != 0; + public static bool any(int3 value) => value.x != 0 || value.y != 0 || value.z != 0; + public static bool any(int4 value) => value.x != 0 || value.y != 0 || value.z != 0 || value.w != 0; + public static bool all(int2 value) => value.x != 0 && value.y != 0; + public static bool all(int3 value) => value.x != 0 && value.y != 0 && value.z != 0; + public static bool all(int4 value) => value.x != 0 && value.y != 0 && value.z != 0 && value.w != 0; + + // HLSL 2021 argument order: select(condition, whenTrue, whenFalse). + // Unity.Mathematics' select(falseValue, trueValue, condition) is the reverse. + public static float select(bool condition, float whenTrue, float whenFalse) => condition ? whenTrue : whenFalse; + public static float2 select(bool2 condition, float2 whenTrue, float2 whenFalse) => + new(condition.x ? whenTrue.x : whenFalse.x, condition.y ? whenTrue.y : whenFalse.y); + public static float3 select(bool3 condition, float3 whenTrue, float3 whenFalse) => + new(condition.x ? whenTrue.x : whenFalse.x, condition.y ? whenTrue.y : whenFalse.y, condition.z ? whenTrue.z : whenFalse.z); + public static float4 select(bool4 condition, float4 whenTrue, float4 whenFalse) => + new(condition.x ? whenTrue.x : whenFalse.x, condition.y ? whenTrue.y : whenFalse.y, condition.z ? whenTrue.z : whenFalse.z, condition.w ? whenTrue.w : whenFalse.w); + + // HLSL mul: mul(m, v) treats v as a column vector, mul(v, m) as a row vector. + public static float mul(float left, float right) => left * right; + + public static float2 mul(float2x2 m, float2 v) => + new(m._m00 * v.x + m._m01 * v.y, m._m10 * v.x + m._m11 * v.y); + + public static float2 mul(float2 v, float2x2 m) => + new(v.x * m._m00 + v.y * m._m10, v.x * m._m01 + v.y * m._m11); + + public static float2x2 mul(float2x2 a, float2x2 b) => + new( + a._m00 * b._m00 + a._m01 * b._m10, a._m00 * b._m01 + a._m01 * b._m11, + a._m10 * b._m00 + a._m11 * b._m10, a._m10 * b._m01 + a._m11 * b._m11); + + public static float3 mul(float3x3 m, float3 v) => + new( + m._m00 * v.x + m._m01 * v.y + m._m02 * v.z, + m._m10 * v.x + m._m11 * v.y + m._m12 * v.z, + m._m20 * v.x + m._m21 * v.y + m._m22 * v.z); + + public static float3 mul(float3 v, float3x3 m) => + new( + v.x * m._m00 + v.y * m._m10 + v.z * m._m20, + v.x * m._m01 + v.y * m._m11 + v.z * m._m21, + v.x * m._m02 + v.y * m._m12 + v.z * m._m22); + + public static float3x3 mul(float3x3 a, float3x3 b) => + new( + a._m00 * b._m00 + a._m01 * b._m10 + a._m02 * b._m20, + a._m00 * b._m01 + a._m01 * b._m11 + a._m02 * b._m21, + a._m00 * b._m02 + a._m01 * b._m12 + a._m02 * b._m22, + a._m10 * b._m00 + a._m11 * b._m10 + a._m12 * b._m20, + a._m10 * b._m01 + a._m11 * b._m11 + a._m12 * b._m21, + a._m10 * b._m02 + a._m11 * b._m12 + a._m12 * b._m22, + a._m20 * b._m00 + a._m21 * b._m10 + a._m22 * b._m20, + a._m20 * b._m01 + a._m21 * b._m11 + a._m22 * b._m21, + a._m20 * b._m02 + a._m21 * b._m12 + a._m22 * b._m22); + + // Ashima Arts / Ian McEwan 3D simplex noise (MIT), mirrored by cultmath_snoise(float3) + // in shaders/CultMath.hlsl with the same float32 evaluation order. + public static float snoise(float3 value) + { + const float cx = 1.0f / 6.0f; + const float cy = 1.0f / 3.0f; + var i = floor(value + dot(value, new float3(cy, cy, cy))); + var x0 = value - i + dot(i, new float3(cx, cx, cx)); + var g = step(new float3(x0.y, x0.z, x0.x), x0); + var l = 1.0f - g; + var lzxy = new float3(l.z, l.x, l.y); + var i1 = min(g, lzxy); + var i2 = max(g, lzxy); + var x1 = x0 - i1 + cx; + var x2 = x0 - i2 + cy; + var x3 = x0 - 0.5f; + + i = snoise_mod289(i); + var p = snoise_permute(snoise_permute(snoise_permute( + i.z + new float4(0.0f, i1.z, i2.z, 1.0f)) + i.y + new float4(0.0f, i1.y, i2.y, 1.0f)) + i.x + new float4(0.0f, i1.x, i2.x, 1.0f)); + + const float n = 0.142857142857f; + var ns = new float3(2.0f * n, 0.5f * n - 1.0f, n); + var j = p - 49.0f * floor(p * ns.z * ns.z); + var xs = floor(j * ns.z); + var ys = floor(j - 7.0f * xs); + var x = xs * ns.x + ns.y; + var y = ys * ns.x + ns.y; + var h = 1.0f - abs(x) - abs(y); + + var b0 = new float4(x.x, x.y, y.x, y.y); + var b1 = new float4(x.z, x.w, y.z, y.w); + var s0 = floor(b0) * 2.0f + 1.0f; + var s1 = floor(b1) * 2.0f + 1.0f; + var sh = -step(h, 0.0f); + var a0 = new float4(b0.x, b0.z, b0.y, b0.w) + new float4(s0.x, s0.z, s0.y, s0.w) * new float4(sh.x, sh.x, sh.y, sh.y); + var a1 = new float4(b1.x, b1.z, b1.y, b1.w) + new float4(s1.x, s1.z, s1.y, s1.w) * new float4(sh.z, sh.z, sh.w, sh.w); + + var p0 = new float3(a0.x, a0.y, h.x); + var p1 = new float3(a0.z, a0.w, h.y); + var p2 = new float3(a1.x, a1.y, h.z); + var p3 = new float3(a1.z, a1.w, h.w); + var norm = 1.79284291400159f - 0.85373472095314f * new float4(dot(p0, p0), dot(p1, p1), dot(p2, p2), dot(p3, p3)); + p0 *= norm.x; + p1 *= norm.y; + p2 *= norm.z; + p3 *= norm.w; + + var m = max(0.6f - new float4(dot(x0, x0), dot(x1, x1), dot(x2, x2), dot(x3, x3)), 0.0f); + m = m * m; + return 42.0f * dot(m * m, new float4(dot(p0, x0), dot(p1, x1), dot(p2, x2), dot(p3, x3))); + } + + // Ashima Arts / Ian McEwan 2D simplex noise, kept component-explicit so + // the C# and HLSL mirrors preserve the same float32 evaluation order. + public static float snoise(float2 value) + { + var c = new float4( + 0.211324865405187f, + 0.366025403784439f, + -0.577350269189626f, + 0.024390243902439f); + var i = floor(value + dot(value, new float2(c.y, c.y))); + var x0 = value - i + dot(i, new float2(c.x, c.x)); + var i1 = x0.x > x0.y ? new float2(1.0f, 0.0f) : new float2(0.0f, 1.0f); + var x12 = new float4( + x0.x + c.x - i1.x, + x0.y + c.x - i1.y, + x0.x + c.z, + x0.y + c.z); + + i = snoise_mod289(i); + var p = snoise_permute(snoise_permute(i.y + new float3(0.0f, i1.y, 1.0f)) + i.x + new float3(0.0f, i1.x, 1.0f)); + var m = max(0.5f - new float3( + dot(x0, x0), + x12.x * x12.x + x12.y * x12.y, + x12.z * x12.z + x12.w * x12.w), 0.0f); + m = m * m; + m = m * m; + + var x = 2.0f * frac(p * c.w) - 1.0f; + var h = abs(x) - 0.5f; + var ox = floor(x + 0.5f); + var a0 = x - ox; + m *= 1.79284291400159f - 0.85373472095314f * (a0 * a0 + h * h); + + var g = new float3( + a0.x * x0.x + h.x * x0.y, + a0.y * x12.x + h.y * x12.y, + a0.z * x12.z + h.z * x12.w); + return 130.0f * dot(m, g); + } + + private static float2 snoise_mod289(float2 value) => value - floor(value * (1.0f / 289.0f)) * 289.0f; + private static float3 snoise_mod289(float3 value) => value - floor(value * (1.0f / 289.0f)) * 289.0f; + private static float4 snoise_mod289(float4 value) => value - floor(value * (1.0f / 289.0f)) * 289.0f; + private static float3 snoise_permute(float3 value) => snoise_mod289(((value * 34.0f) + 1.0f) * value); + private static float4 snoise_permute(float4 value) => snoise_mod289(((value * 34.0f) + 1.0f) * value); + + public static float value_noise(float2 position) + { + var cell = floor(position); + var local = frac(position); + var u = local * local * (3.0f - 2.0f * local); + + var a = hash(cell); + var b = hash(cell + new float2(1.0f, 0.0f)); + var c = hash(cell + new float2(0.0f, 1.0f)); + var d = hash(cell + new float2(1.0f, 1.0f)); + + return lerp(lerp(a, b, u.x), lerp(c, d, u.x), u.y); + } + + public static float value_noise_bicubic(float2 position) + { + var cell = floor(position); + var local = frac(position); + + var y0 = catmullrom( + hash(cell + new float2(-1.0f, -1.0f)), + hash(cell + new float2(0.0f, -1.0f)), + hash(cell + new float2(1.0f, -1.0f)), + hash(cell + new float2(2.0f, -1.0f)), + local.x); + var y1 = catmullrom( + hash(cell + new float2(-1.0f, 0.0f)), + hash(cell + new float2(0.0f, 0.0f)), + hash(cell + new float2(1.0f, 0.0f)), + hash(cell + new float2(2.0f, 0.0f)), + local.x); + var y2 = catmullrom( + hash(cell + new float2(-1.0f, 1.0f)), + hash(cell + new float2(0.0f, 1.0f)), + hash(cell + new float2(1.0f, 1.0f)), + hash(cell + new float2(2.0f, 1.0f)), + local.x); + var y3 = catmullrom( + hash(cell + new float2(-1.0f, 2.0f)), + hash(cell + new float2(0.0f, 2.0f)), + hash(cell + new float2(1.0f, 2.0f)), + hash(cell + new float2(2.0f, 2.0f)), + local.x); + + return catmullrom(y0, y1, y2, y3, local.y); + } +} diff --git a/packages/cultmath/src/CultMath/math.cs.meta b/packages/cultmath/src/CultMath/math.cs.meta new file mode 100644 index 00000000..9222af7d --- /dev/null +++ b/packages/cultmath/src/CultMath/math.cs.meta @@ -0,0 +1,2 @@ +fileFormatVersion: 2 +guid: 0d9b4cacc40d169429ee743d19577cb7 \ No newline at end of file diff --git a/packages/cultmath/src/CultMath/quaternion.cs b/packages/cultmath/src/CultMath/quaternion.cs new file mode 100644 index 00000000..e8c4a48c --- /dev/null +++ b/packages/cultmath/src/CultMath/quaternion.cs @@ -0,0 +1,95 @@ +using System; + +namespace CultMath; + +/// +/// Not an HLSL type: shaders carry rotations as float4. Equality therefore stays +/// scalar (== returns bool) instead of following the component-wise vector rule. +/// +public struct quaternion : IEquatable +{ + public static readonly quaternion identity = new(0.0f, 0.0f, 0.0f, 1.0f); + + public float x; + public float y; + public float z; + public float w; + + public quaternion(float x, float y, float z, float w) + { + this.x = x; + this.y = y; + this.z = z; + this.w = w; + } + + public static quaternion LookRotation(float3 forward, float3 up) + { + forward = math.normalize(forward); + up = math.normalize(up); + + var right = math.normalize(math.cross(up, forward)); + up = math.cross(forward, right); + + var m00 = right.x; + var m01 = up.x; + var m02 = forward.x; + var m10 = right.y; + var m11 = up.y; + var m12 = forward.y; + var m20 = right.z; + var m21 = up.z; + var m22 = forward.z; + + var trace = m00 + m11 + m22; + if (trace > 0.0f) + { + var s = math.sqrt(trace + 1.0f) * 2.0f; + return math.normalize(new quaternion( + (m21 - m12) / s, + (m02 - m20) / s, + (m10 - m01) / s, + 0.25f * s)); + } + + if (m00 > m11 && m00 > m22) + { + var s = math.sqrt(1.0f + m00 - m11 - m22) * 2.0f; + return math.normalize(new quaternion( + 0.25f * s, + (m01 + m10) / s, + (m02 + m20) / s, + (m21 - m12) / s)); + } + + if (m11 > m22) + { + var s = math.sqrt(1.0f + m11 - m00 - m22) * 2.0f; + return math.normalize(new quaternion( + (m01 + m10) / s, + 0.25f * s, + (m12 + m21) / s, + (m02 - m20) / s)); + } + + { + var s = math.sqrt(1.0f + m22 - m00 - m11) * 2.0f; + return math.normalize(new quaternion( + (m02 + m20) / s, + (m12 + m21) / s, + 0.25f * s, + (m10 - m01) / s)); + } + } + + public static implicit operator quaternion(float4 value) => new(value.x, value.y, value.z, value.w); + public static implicit operator float4(quaternion value) => new(value.x, value.y, value.z, value.w); + + public static bool operator ==(quaternion left, quaternion right) => left.Equals(right); + public static bool operator !=(quaternion left, quaternion right) => !left.Equals(right); + + public readonly bool Equals(quaternion other) => x.Equals(other.x) && y.Equals(other.y) && z.Equals(other.z) && w.Equals(other.w); + public override readonly bool Equals(object? obj) => obj is quaternion other && Equals(other); + public override readonly int GetHashCode() => HashCode.Combine(x, y, z, w); + public override readonly string ToString() => FormattableString.Invariant($"quaternion({x:R}, {y:R}, {z:R}, {w:R})"); +} diff --git a/packages/cultmath/src/CultMath/quaternion.cs.meta b/packages/cultmath/src/CultMath/quaternion.cs.meta new file mode 100644 index 00000000..ae6e8a11 --- /dev/null +++ b/packages/cultmath/src/CultMath/quaternion.cs.meta @@ -0,0 +1,2 @@ +fileFormatVersion: 2 +guid: 12f3627d9250dc348be9c1a1056ec7d0 \ No newline at end of file diff --git a/packages/cultmath/src/CultMath/rect.cs b/packages/cultmath/src/CultMath/rect.cs new file mode 100644 index 00000000..51e05c19 --- /dev/null +++ b/packages/cultmath/src/CultMath/rect.cs @@ -0,0 +1,59 @@ +namespace CultMath; + +/// +/// Axis-aligned 2D bounds stored canonically as normalized minimum and maximum +/// corners. Point containment and intersection include the boundary. +/// +public readonly record struct rect +{ + public rect(float minX, float minY, float maxX, float maxY) + : this(new float2(minX, minY), new float2(maxX, maxY)) + { + } + + public rect(float2 min, float2 max) + { + this.min = new float2(MathF.Min(min.x, max.x), MathF.Min(min.y, max.y)); + this.max = new float2(MathF.Max(min.x, max.x), MathF.Max(min.y, max.y)); + } + + public float2 min { get; } + public float2 max { get; } + + public float width => max.x - min.x; + public float height => max.y - min.y; + public float2 size => max - min; + public float2 center => (min + max) * 0.5f; + public float area => width * height; + public bool isEmpty => width == 0.0f || height == 0.0f; + + public bool Contains(float2 point) => + point.x >= min.x && point.x <= max.x && + point.y >= min.y && point.y <= max.y; + + public bool Contains(rect other) => + other.min.x >= min.x && other.max.x <= max.x && + other.min.y >= min.y && other.max.y <= max.y; + + public bool Intersects(rect other) => + min.x <= other.max.x && max.x >= other.min.x && + min.y <= other.max.y && max.y >= other.min.y; + + public rect Intersection(rect other) + { + if (!Intersects(other)) + return new rect(0.0f, 0.0f, 0.0f, 0.0f); + + return new rect( + MathF.Max(min.x, other.min.x), + MathF.Max(min.y, other.min.y), + MathF.Min(max.x, other.max.x), + MathF.Min(max.y, other.max.y)); + } + + public rect Encapsulate(float2 point) => new( + MathF.Min(min.x, point.x), + MathF.Min(min.y, point.y), + MathF.Max(max.x, point.x), + MathF.Max(max.y, point.y)); +} diff --git a/packages/cultmath/src/CultMath/rect.cs.meta b/packages/cultmath/src/CultMath/rect.cs.meta new file mode 100644 index 00000000..0b04a5ce --- /dev/null +++ b/packages/cultmath/src/CultMath/rect.cs.meta @@ -0,0 +1,2 @@ +fileFormatVersion: 2 +guid: 5aab0d8e1ca94d42b85bba780a7ee449 diff --git a/packages/cultmath/tests.meta b/packages/cultmath/tests.meta new file mode 100644 index 00000000..a35356c4 --- /dev/null +++ b/packages/cultmath/tests.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: 4ca61dedbfb3c5d46b0acaf23202b5c2 +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/packages/cultmath/tests/CultMath.Tests.meta b/packages/cultmath/tests/CultMath.Tests.meta new file mode 100644 index 00000000..3f1451c6 --- /dev/null +++ b/packages/cultmath/tests/CultMath.Tests.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: bcee65f13f85daa45b0cf3fdaa901779 +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/tests/GameCult.Geometry.Planetary.Tests/GameCult.Geometry.Planetary.Tests.csproj b/packages/cultmath/tests/CultMath.Tests/CultMath.Tests.csproj similarity index 77% rename from tests/GameCult.Geometry.Planetary.Tests/GameCult.Geometry.Planetary.Tests.csproj rename to packages/cultmath/tests/CultMath.Tests/CultMath.Tests.csproj index b2511182..d77fd05f 100644 --- a/tests/GameCult.Geometry.Planetary.Tests/GameCult.Geometry.Planetary.Tests.csproj +++ b/packages/cultmath/tests/CultMath.Tests/CultMath.Tests.csproj @@ -1,4 +1,5 @@ + net10.0 enable @@ -7,12 +8,14 @@ - - - - + + + + + + diff --git a/packages/cultmath/tests/CultMath.Tests/CultMath.Tests.csproj.meta b/packages/cultmath/tests/CultMath.Tests/CultMath.Tests.csproj.meta new file mode 100644 index 00000000..709184bb --- /dev/null +++ b/packages/cultmath/tests/CultMath.Tests/CultMath.Tests.csproj.meta @@ -0,0 +1,7 @@ +fileFormatVersion: 2 +guid: 636051db121e19146a0baf92b790415c +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/packages/cultmath/tests/CultMath.Tests/HlslMirrorTests.cs b/packages/cultmath/tests/CultMath.Tests/HlslMirrorTests.cs new file mode 100644 index 00000000..445ad255 --- /dev/null +++ b/packages/cultmath/tests/CultMath.Tests/HlslMirrorTests.cs @@ -0,0 +1,77 @@ +using Xunit; + +namespace CultMath.Tests; + +public sealed class HlslMirrorTests +{ + [Fact] + public void HlslMirrorDoesNotPublishGeometryKernels() + { + var source = File.ReadAllText(Path.Combine(GetCultMathRoot(), "shaders", "CultMath.hlsl")); + Assert.DoesNotContain("AdvancedErosionFilter.hlsl", source); + Assert.DoesNotContain("Planetary.hlsl", source); + Assert.DoesNotContain("spherical_erosion", source); + Assert.DoesNotContain("planetary_radial_refinement", source); + } + + [Fact] + public void HlslMirrorPublishesCultMathPrimitives() + { + var include = File.ReadAllText(Path.Combine(GetCultMathRoot(), "shaders", "CultMath.hlsl")); + var requiredSymbols = new[] + { + "cultmath_radians", + "cultmath_degrees", + "cultmath_frac", + "cultmath_clamp", + "cultmath_saturate", + "cultmath_lerp", + "cultmath_step", + "cultmath_smoothstep", + "cultmath_smootherstep", + "cultmath_lengthsq", + "cultmath_distance", + "cultmath_normalize", + "cultmath_reflect", + "cultmath_rotate", + "cultmath_csum", + "cultmath_decay", + "cultmath_damp", + "cultmath_catmullrom", + "cultmath_quadratic_bezier", + "cultmath_cubic_bezier", + "cultmath_hash", + "cultmath_value_noise", + "cultmath_snoise(float2", + "cultmath_snoise(float3", + "cultmath_value_noise_bicubic", + "cultmath_value_noise_texture", + "cultmath_value_noise_texture_bicubic", + }; + + foreach (var symbol in requiredSymbols) + { + Assert.Contains(symbol, include); + } + + Assert.Contains("CULTMATH_NORMALIZE_EPSILON", include); + Assert.Contains("max(length(value), CULTMATH_NORMALIZE_EPSILON)", include); + } + + private static string GetCultMathRoot() + { + var directory = new DirectoryInfo(AppContext.BaseDirectory); + while (directory is not null) + { + var monorepoPackage = Path.Combine(directory.FullName, "packages", "cultmath"); + if (File.Exists(Path.Combine(monorepoPackage, "shaders", "CultMath.hlsl"))) + { + return monorepoPackage; + } + + directory = directory.Parent; + } + + throw new InvalidOperationException("Could not locate the CultMath package root."); + } +} diff --git a/packages/cultmath/tests/CultMath.Tests/HlslMirrorTests.cs.meta b/packages/cultmath/tests/CultMath.Tests/HlslMirrorTests.cs.meta new file mode 100644 index 00000000..1fcf44b3 --- /dev/null +++ b/packages/cultmath/tests/CultMath.Tests/HlslMirrorTests.cs.meta @@ -0,0 +1,2 @@ +fileFormatVersion: 2 +guid: 3d7eb9df849ea0148a71419bbfe2352c \ No newline at end of file diff --git a/packages/cultmath/tests/CultMath.Tests/HlslSemanticsTests.cs b/packages/cultmath/tests/CultMath.Tests/HlslSemanticsTests.cs new file mode 100644 index 00000000..7b845ea7 --- /dev/null +++ b/packages/cultmath/tests/CultMath.Tests/HlslSemanticsTests.cs @@ -0,0 +1,441 @@ +using System.Text.Json; +using Xunit; + +namespace CultMath.Tests; + +public sealed class HlslSemanticsTests +{ + [Fact] + public void ComponentsAndIndexersAreWritable() + { + var value = new float4(1.0f, 2.0f, 3.0f, 4.0f); + value.w = 9.0f; + value[0] = -1.0f; + + Assert.Equal(new float4(-1.0f, 2.0f, 3.0f, 9.0f), value); + + var cell = new int3(1, 2, 3); + cell[2] = 7; + Assert.Equal(new int3(1, 2, 7), cell); + + ref var component = ref value.y; + component = 20.0f; + Assert.Equal(20.0f, value.y); + } + + [Fact] + public void Float2SwizzlesReadAndWrite() + { + var value = new float2(1.0f, 2.0f); + Assert.Equal(new float2(2.0f, 1.0f), value.yx); + + value.yx = new float2(5.0f, 6.0f); + Assert.Equal(new float2(6.0f, 5.0f), value); + } + + [Fact] + public void Float3SwizzlesReadAndWrite() + { + var source = new float3(1.0f, 2.0f, 3.0f); + Assert.Equal(new float2(1.0f, 2.0f), source.xy); + Assert.Equal(new float2(1.0f, 3.0f), source.xz); + Assert.Equal(new float2(2.0f, 1.0f), source.yx); + Assert.Equal(new float2(2.0f, 3.0f), source.yz); + Assert.Equal(new float2(3.0f, 2.0f), source.zy); + Assert.Equal(new float3(1.0f, 2.0f, 3.0f), source.xyz); + Assert.Equal(new float3(1.0f, 3.0f, 2.0f), source.xzy); + Assert.Equal(new float3(2.0f, 3.0f, 1.0f), source.yzx); + Assert.Equal(new float3(3.0f, 2.0f, 1.0f), source.zyx); + + Assert.Equal(new float3(8.0f, 9.0f, 3.0f), Write(source, (ref float3 v) => v.xy = new float2(8.0f, 9.0f))); + Assert.Equal(new float3(8.0f, 2.0f, 9.0f), Write(source, (ref float3 v) => v.xz = new float2(8.0f, 9.0f))); + Assert.Equal(new float3(9.0f, 8.0f, 3.0f), Write(source, (ref float3 v) => v.yx = new float2(8.0f, 9.0f))); + Assert.Equal(new float3(1.0f, 8.0f, 9.0f), Write(source, (ref float3 v) => v.yz = new float2(8.0f, 9.0f))); + Assert.Equal(new float3(1.0f, 9.0f, 8.0f), Write(source, (ref float3 v) => v.zy = new float2(8.0f, 9.0f))); + Assert.Equal(new float3(7.0f, 8.0f, 9.0f), Write(source, (ref float3 v) => v.xyz = new float3(7.0f, 8.0f, 9.0f))); + Assert.Equal(new float3(7.0f, 9.0f, 8.0f), Write(source, (ref float3 v) => v.xzy = new float3(7.0f, 8.0f, 9.0f))); + Assert.Equal(new float3(9.0f, 7.0f, 8.0f), Write(source, (ref float3 v) => v.yzx = new float3(7.0f, 8.0f, 9.0f))); + Assert.Equal(new float3(9.0f, 8.0f, 7.0f), Write(source, (ref float3 v) => v.zyx = new float3(7.0f, 8.0f, 9.0f))); + } + + [Fact] + public void Float4SwizzlesReadAndWrite() + { + var source = new float4(1.0f, 2.0f, 3.0f, 4.0f); + Assert.Equal(new float2(1.0f, 2.0f), source.xy); + Assert.Equal(new float3(1.0f, 2.0f, 3.0f), source.xyz); + Assert.Equal(new float3(1.0f, 2.0f, 4.0f), source.xyw); + Assert.Equal(new float3(2.0f, 3.0f, 1.0f), source.yzx); + + var a = source; + a.xy = new float2(8.0f, 9.0f); + Assert.Equal(new float4(8.0f, 9.0f, 3.0f, 4.0f), a); + + var b = source; + b.xyz = new float3(7.0f, 8.0f, 9.0f); + Assert.Equal(new float4(7.0f, 8.0f, 9.0f, 4.0f), b); + + var c = source; + c.xyw = new float3(7.0f, 8.0f, 9.0f); + Assert.Equal(new float4(7.0f, 8.0f, 3.0f, 9.0f), c); + + var d = source; + d.yzx = new float3(7.0f, 8.0f, 9.0f); + Assert.Equal(new float4(9.0f, 7.0f, 8.0f, 4.0f), d); + } + + [Fact] + public void GeneratedSwizzlesCoverRepeatsColorNamesAndAllVectorFamilies() + { + var v = new float4(1.0f, 2.0f, 3.0f, 4.0f); + Assert.Equal(new float4(4.0f, 4.0f, 1.0f, 2.0f), v.wwxy); + Assert.Equal(new float3(3.0f, 2.0f, 1.0f), v.bgr); + Assert.Equal(4.0f, v.a); + Assert.Equal(new float4(2.0f, 2.0f, 2.0f, 2.0f), new float2(1.0f, 2.0f).yyyy); + + v.wx = new float2(9.0f, 8.0f); + Assert.Equal(new float4(8.0f, 2.0f, 3.0f, 9.0f), v); + v.r = 0.5f; + v.gb += new float2(1.0f, 1.0f); + Assert.Equal(new float4(0.5f, 3.0f, 4.0f, 9.0f), v); + + Assert.Equal(new int3(3, 1, 1), new int3(1, 2, 3).zxx); + Assert.Equal(new bool2(false, true), new bool4(true, false, false, true).zw); + Assert.Equal(new double2(2.0, 1.0), new double3(1.0, 2.0, 3.0).yx); + + var cell = new int4(1, 2, 3, 4); + cell.xyz = new int3(7, 8, 9); + Assert.Equal(new int4(7, 8, 9, 4), cell); + } + + [Fact] + public void MixedConstructorsMatchHlsl() + { + var xy = new float2(1.0f, 2.0f); + var zw = new float2(3.0f, 4.0f); + var expected = new float4(1.0f, 2.0f, 3.0f, 4.0f); + + Assert.Equal(expected, math.float4(xy, zw)); + Assert.Equal(expected, math.float4(1.0f, new float3(2.0f, 3.0f, 4.0f))); + Assert.Equal(expected, math.float4(new float3(1.0f, 2.0f, 3.0f), 4.0f)); + Assert.Equal(expected, math.float4(xy, 3.0f, 4.0f)); + Assert.Equal(expected, math.float4(1.0f, 2.0f, zw)); + Assert.Equal(expected, math.float4(1.0f, new float2(2.0f, 3.0f), 4.0f)); + Assert.Equal(expected, new float4(xy, zw)); + Assert.Equal(new float3(1.0f, 2.0f, 3.0f), math.float3(1.0f, zw - 1.0f)); + Assert.Equal(new int4(1, 2, 3, 4), math.int4(new int2(1, 2), new int2(3, 4))); + Assert.Equal(new bool3(true, false, true), math.bool3(true, new bool2(false, true))); + Assert.Equal(new double3(1.0, 2.0, 3.0), new double3(new double2(1.0, 2.0), 3.0)); + } + + [Fact] + public void MatrixRowElementsAreWritable() + { + var m = float3x3.identity; + m[1][2] = 5.0f; + m[2].x = 7.0f; + m._m00 = 2.0f; + + Assert.Equal(5.0f, m._m12); + Assert.Equal(7.0f, m._m20); + Assert.Equal(math.float3x3(2.0f, 0.0f, 0.0f, 0.0f, 1.0f, 5.0f, 7.0f, 0.0f, 1.0f), m); + Assert.Equal(float3x3.identity, math.float3x3(1.0f, 0.0f, 0.0f, 0.0f, 1.0f, 0.0f, 0.0f, 0.0f, 1.0f)); + + var r = float2x2.identity; + r[0][1] = 3.0f; + Assert.Equal(math.float2x2(1.0f, 3.0f, 0.0f, 1.0f), r); + Assert.Equal(new float2(0.0f, 1.0f), float2x2.identity[1]); + } + + [Fact] + public void IntrinsicEdgeCasesFollowHlsl() + { + Assert.True(math.any(new float3(0.0f, 0.0f, -2.0f))); + Assert.False(math.any(float4.zero)); + Assert.True(math.all(new int2(1, -1))); + Assert.False(math.all(new int3(1, 0, 1))); + Assert.True(math.any(new float2(float.NaN, 0.0f))); + + Assert.Equal(new int3(1, 2, 3), math.min(new int3(1, 5, 3), new int3(4, 2, 6))); + Assert.Equal(new int2(4, 5), math.max(new int2(1, 5), new int2(4, 2))); + Assert.Equal(new int4(1, 2, 0, 3), math.abs(new int4(-1, 2, 0, -3))); + Assert.Equal(7, math.abs(-7)); + + Assert.Equal(1.0f, math.step(1.0f, float.NaN)); + Assert.Equal(1.0f, math.step(float.NaN, 1.0f)); + Assert.Equal(1.0f, math.step(1.0f, 1.0f)); + + // DXIL FMin/FMax/Saturate: a NaN operand returns the other; max(a, b) is a >= b ? a : b. + Assert.Equal(1.0f, math.min(float.NaN, 1.0f)); + Assert.Equal(1.0f, math.min(1.0f, float.NaN)); + Assert.Equal(1.0f, math.max(float.NaN, 1.0f)); + Assert.Equal(1.0f, math.max(1.0f, float.NaN)); + Assert.True(float.IsNaN(math.max(float.NaN, float.NaN))); + Assert.Equal(2.0f, math.clamp(float.NaN, 2.0f, 3.0f)); + Assert.Equal(new float3(0.0f, 1.0f, 0.0f), math.saturate(new float3(float.NaN, 2.0f, -1.0f))); + Assert.Equal(new float2(-1.0f, 4.0f), math.max(new float2(float.NaN, 4.0f), new float2(-1.0f, float.NaN))); + static int Bits(float value) => BitConverter.SingleToInt32Bits(value); + Assert.Equal(Bits(-0.0f), Bits(math.max(-0.0f, 0.0f))); + Assert.Equal(Bits(0.0f), Bits(math.min(-0.0f, 0.0f))); + Assert.Equal(Bits(0.0f), Bits(math.saturate(-0.0f))); + Assert.Equal(1.0, math.min(double.NaN, 1.0)); + Assert.Equal(1.0, math.min(1.0, double.NaN)); + Assert.Equal(1.0, math.max(double.NaN, 1.0)); + Assert.Equal(1.0, math.max(1.0, double.NaN)); + Assert.True(double.IsNaN(math.max(double.NaN, double.NaN))); + static long DoubleBits(double value) => BitConverter.DoubleToInt64Bits(value); + Assert.Equal(DoubleBits(-0.0), DoubleBits(math.max(-0.0, 0.0))); + Assert.Equal(DoubleBits(0.0), DoubleBits(math.min(-0.0, 0.0))); + + Assert.Equal(0, math.sign(float.NaN)); + Assert.Equal(-1, math.sign(-0.5f)); + Assert.Equal(new int3(-1, 0, 1), math.sign(new float3(-2.0f, 0.0f, 4.0f))); + Assert.Equal(new int2(1, -1), math.sign(new int2(9, -9))); + } + + [Fact] + public void ComparisonOperatorsReturnComponentWiseBoolVectors() + { + var a = new float3(1.0f, 2.0f, 3.0f); + var b = new float3(3.0f, 2.0f, 1.0f); + + Assert.Equal(new bool3(true, false, false), a < b); + Assert.Equal(new bool3(false, false, true), a > b); + Assert.Equal(new bool3(true, true, false), a <= b); + Assert.Equal(new bool3(false, true, true), a >= b); + Assert.Equal(new bool3(false, true, false), a == b); + Assert.Equal(new bool3(true, false, true), a != b); + + Assert.Equal(new bool3(true, false, false), a < 2.0f); + Assert.Equal(new bool3(false, false, true), 2.0f < a); + Assert.Equal(new bool2(true, false), new float2(0.0f, 1.0f) == 0.0f); + Assert.Equal(new bool4(false, true, true, true), new float4(0.0f, 1.0f, 2.0f, 3.0f) >= 1.0f); + Assert.Equal(new bool2(false, true), new int2(1, 2) > new int2(1, 1)); + Assert.Equal(new bool3(true, false, true), new int3(1, 2, 3) != new int3(0, 2, 4)); + Assert.Equal(new bool4(true, true, false, false), new int4(1, 2, 3, 4) <= new int4(1, 2, 2, 2)); + Assert.Equal(new bool2(true, false), new double2(0.5, 1.0) < new double2(1.0, 1.0)); + Assert.Equal(new bool3(false, false, true), new double3(1.0, 1.0, 2.0) > new double3(1.0, 1.0, 1.0)); + } + + [Fact] + public void ScalarOperatorsApplyPerComponentOnBothSides() + { + var value = new float3(1.0f, 2.0f, 4.0f); + + Assert.Equal(new float3(3.0f, 2.0f, 0.0f), 4.0f - value); + Assert.Equal(new float3(0.0f, 1.0f, 3.0f), value - 1.0f); + Assert.Equal(new float3(2.0f, 3.0f, 5.0f), 1.0f + value); + Assert.Equal(new float3(4.0f, 2.0f, 1.0f), 4.0f / value); + Assert.Equal(new float3(0.5f, 1.0f, 2.0f), value / 2.0f); + Assert.Equal(new float3(2.0f, 4.0f, 8.0f), 2.0f * value); + Assert.Equal(new float3(1.0f, 0.0f, 0.0f), value % 2.0f); + Assert.Equal(new float2(1.5f, 2.5f), new float2(1.0f, 2.0f) + 0.5f); + Assert.Equal(new float4(0.0f, 1.0f, 2.0f, 3.0f), new float4(1.0f, 2.0f, 3.0f, 4.0f) - 1.0f); + } + + [Fact] + public void EqualsAndHashCodeStayValueBasedForDictionaryKeys() + { + var map = new Dictionary { [new float3(1.0f, 2.0f, 3.0f)] = "hit" }; + Assert.Equal("hit", map[new float3(1.0f, 2.0f, 3.0f)]); + Assert.False(map.ContainsKey(new float3(1.0f, 2.0f, 3.5f))); + + var cells = new HashSet { new int2(3, 4) }; + Assert.Contains(new int2(3, 4), cells); + Assert.Contains(new int3(1, 2, 3), new HashSet { new int3(1, 2, 3) }); + + var nan = new float2(float.NaN, 0.0f); + var sameNan = nan; + Assert.True(nan.Equals(sameNan)); + Assert.False((nan == sameNan).x); + + Assert.True(new quaternion(0.0f, 0.0f, 0.0f, 1.0f) == quaternion.identity); + Assert.Equal(float2x2.identity, math.float2x2(1.0f, 0.0f, 0.0f, 1.0f)); + Assert.Equal("float3(1, 2.5, -3)", new float3(1.0f, 2.5f, -3.0f).ToString()); + } + + [Fact] + public void AnyAllAndSelectFollowHlsl() + { + Assert.True(math.any(new bool2(false, true))); + Assert.False(math.any(bool3.@false)); + Assert.True(math.all(bool4.@true)); + Assert.False(math.all(new bool3(true, false, true))); + + var mask = new bool3(true, false, true); + Assert.Equal(new float3(1.0f, 20.0f, 3.0f), math.select(mask, new float3(1.0f, 2.0f, 3.0f), new float3(10.0f, 20.0f, 30.0f))); + Assert.Equal(7.0f, math.select(true, 7.0f, 8.0f)); + Assert.Equal(new float2(0.0f, 5.0f), math.select(new bool2(false, true), math.float2(5.0f), math.float2(0.0f))); + Assert.Equal(new float4(1.0f, 0.0f, 1.0f, 0.0f), math.select(new bool4(true, false, true, false), 1.0f, 0.0f)); + + Assert.Equal(new bool2(false, true), !new bool2(true, false)); + Assert.Equal(new bool3(true, false, false), new bool3(true, true, false) & new bool3(true, false, true)); + Assert.Equal(new bool3(true, true, true), new bool3(true, true, false) | new bool3(true, false, true)); + } + + [Fact] + public void AddedIntrinsicsAreComponentWise() + { + Assert.Equal(MathF.Log(3.0f), math.log(3.0f)); + Assert.Equal(new float3(0.0f, MathF.Log(2.0f), MathF.Log(4.0f)), math.log(new float3(1.0f, 2.0f, 4.0f))); + Assert.Equal(new float2(8.0f, 9.0f), math.pow(new float2(2.0f, 3.0f), new float2(3.0f, 2.0f))); + Assert.Equal(new float4(1.0f, 2.0f, 3.0f, 4.0f), math.sqrt(new float4(1.0f, 4.0f, 9.0f, 16.0f))); + var sines = math.sin(new float3(0.0f, math.HALF_PI, math.PI)); + Assert.Equal(0.0f, sines.x, precision: 5); + Assert.Equal(1.0f, sines.y, precision: 5); + Assert.Equal(0.0f, sines.z, precision: 5); + var cosines = math.cos(new float2(0.0f, math.PI)); + Assert.Equal(1.0f, cosines.x, precision: 5); + Assert.Equal(-1.0f, cosines.y, precision: 5); + var arccosines = math.acos(new float3(1.0f, 0.0f, -1.0f)); + Assert.Equal(0.0f, arccosines.x, precision: 5); + Assert.Equal(math.HALF_PI, arccosines.y, precision: 5); + Assert.Equal(math.PI, arccosines.z, precision: 5); + Assert.Equal(math.atan2(1.0f, -1.0f), math.atan2(new float2(1.0f, 0.0f), new float2(-1.0f, 1.0f)).x); + Assert.Equal(0.0f, math.atan2(new float4(1.0f, 0.0f, 1.0f, 0.0f), new float4(1.0f, 1.0f, 1.0f, 1.0f)).y); + Assert.Equal(2.5, math.min(2.5, 3.5)); + Assert.Equal(3.5, math.max(2.5, 3.5)); + } + + [Fact] + public void IntVectorsPromoteToFloatVectors() + { + float3 promoted3 = new int3(1, 2, 3); + float4 promoted4 = new int4(1, 2, 3, 4); + + Assert.Equal(new float3(1.0f, 2.0f, 3.0f), promoted3); + Assert.Equal(new float4(1.0f, 2.0f, 3.0f, 4.0f), promoted4); + Assert.Equal(new int3(2, 4, 6), new int3(1, 2, 3) * 2); + Assert.Equal(new int4(1, 0, 1, 0), new int4(3, 4, 5, 6) % 2); + Assert.Equal(new int3(1, 2, 3), math.int3(1, 2, 3)); + Assert.Equal(new int4(5, 5, 5, 5), math.int4(5)); + } + + [Fact] + public void MatrixMulFollowsHlslRowMajorConvention() + { + var m = math.float2x2(1.0f, 2.0f, 3.0f, 4.0f); + var v = new float2(5.0f, 6.0f); + + Assert.Equal(new float2(1.0f, 2.0f), m[0]); + Assert.Equal(new float2(17.0f, 39.0f), math.mul(m, v)); + Assert.Equal(new float2(23.0f, 34.0f), math.mul(v, m)); + Assert.Equal(math.float2x2(7.0f, 10.0f, 15.0f, 22.0f), math.mul(m, m)); + + var m3 = math.float3x3(new float3(1.0f, 2.0f, 3.0f), new float3(4.0f, 5.0f, 6.0f), new float3(7.0f, 8.0f, 9.0f)); + var v3 = new float3(1.0f, 0.0f, -1.0f); + Assert.Equal(new float3(-2.0f, -2.0f, -2.0f), math.mul(m3, v3)); + Assert.Equal(new float3(-6.0f, -6.0f, -6.0f), math.mul(v3, m3)); + Assert.Equal(math.float3x3(30.0f, 36.0f, 42.0f, 66.0f, 81.0f, 96.0f, 102.0f, 126.0f, 150.0f), math.mul(m3, m3)); + Assert.Equal(m3, math.mul(m3, float3x3.identity)); + + m3[1] = new float3(0.0f, 1.0f, 0.0f); + Assert.Equal(1.0f, m3._m11); + Assert.Equal(12.0f, math.mul(3.0f, 4.0f)); + } + + [Fact] + public void Float2x2RotateMatchesRotateAndUnityMathematics() + { + var rotation = float2x2.Rotate(0.5f); + var column = math.mul(rotation, new float2(1.0f, 0.0f)); + var row = math.mul(new float2(1.0f, 0.0f), rotation); + var reference = math.rotate(new float2(1.0f, 0.0f), 0.5f); + + Assert.Equal(reference.x, column.x, precision: 6); + Assert.Equal(reference.y, column.y, precision: 6); + + // Unity.Mathematics 1.x reference values: mul(float2(1, 0), float2x2.Rotate(0.5f)). + Assert.Equal(0.87758255f, row.x, precision: 6); + Assert.Equal(-0.47942555f, row.y, precision: 6); + } + + [Fact] + public void Float3x3EulerMatchesUnityMathematicsReference() + { + // Unity.Mathematics 1.x reference values captured from float3x3.Euler. + AssertRows( + float3x3.Euler(new float3(0.4f, -0.7f, 1.3f), math.RotationOrder.YXZ), + 0.44632244f, -0.8874959f, 0.114662126f, + 0.6698625f, 0.24638279f, -0.700414f, + 0.59336376f, 0.38941833f, 0.70446634f); + AssertRows( + float3x3.Euler(new float3(0.4f, -0.7f, 1.3f)), + -0.037133574f, -0.8040775f, -0.59336376f, + 0.8874959f, 0.24638279f, -0.38941833f, + 0.45931715f, -0.5410684f, 0.70446634f); + + // Aetheria ActionGameManager view direction: mul(float3(0, 0, 1), Euler(float3(yawPitch.yx, 0), YXZ)). + var yawPitch = new float2(0.3f, -1.1f); + var view = math.mul(new float3(0.0f, 0.0f, 1.0f), float3x3.Euler(new float3(yawPitch.yx, 0.0f), math.RotationOrder.YXZ)); + Assert.Equal(-0.13404681f, view.x, precision: 5); + Assert.Equal(-0.8912074f, view.y, precision: 5); + Assert.Equal(0.4333369f, view.z, precision: 5); + + var composed = math.mul(float3x3.RotateZ(0.2f), math.mul(float3x3.RotateY(0.3f), float3x3.RotateX(0.1f))); + AssertRows(float3x3.Euler(0.1f, 0.3f, 0.2f, math.RotationOrder.XYZ), + composed._m00, composed._m01, composed._m02, + composed._m10, composed._m11, composed._m12, + composed._m20, composed._m21, composed._m22); + } + + [Theory] + [InlineData(0.0f, 0.0f, 0.0f, -0.41219875f)] + [InlineData(0.25f, -0.5f, 1.75f, 0.5250269f)] + [InlineData(12.25f, -4.5f, 3.125f, 0.008141451f)] + [InlineData(-7.3f, 2.9f, 101.4f, 0.578015f)] + public void Snoise3MatchesUnityMathematicsReference(float x, float y, float z, float expected) + { + Assert.Equal(expected, math.snoise(new float3(x, y, z)), precision: 5); + } + + [Fact] + public void RandomFloat3OverloadsShareOneDrawOrder() + { + var a = new Random(99); + var b = new Random(99); + var c = new Random(99); + + var unit = a.NextFloat3(); + Assert.Equal(new float3(b.NextFloat(), b.NextFloat(), b.NextFloat()), unit); + Assert.Equal(unit * new float3(2.0f, 4.0f, 8.0f), c.NextFloat3(new float3(2.0f, 4.0f, 8.0f))); + + var d = new Random(99); + Assert.Equal(math.float3(-1.0f) + unit * 2.0f, d.NextFloat3(math.float3(-1.0f), math.float3(1.0f))); + } + + [Fact] + public void CoreAssemblyStaysEngineFree() + { + var references = typeof(math).Assembly.GetReferencedAssemblies(); + Assert.DoesNotContain(references, reference => reference.Name!.StartsWith("UnityEngine", StringComparison.Ordinal)); + + var root = new DirectoryInfo(AppContext.BaseDirectory); + while (root is not null && !File.Exists(Path.Combine(root.FullName, "packages", "cultmath", "src", "CultMath", "CultMath.asmdef"))) + { + root = root.Parent; + } + + Assert.NotNull(root); + var asmdefPath = Path.Combine(root!.FullName, "packages", "cultmath", "src", "CultMath", "CultMath.asmdef"); + using var asmdef = JsonDocument.Parse(File.ReadAllText(asmdefPath)); + Assert.True(asmdef.RootElement.GetProperty("noEngineReferences").GetBoolean()); + } + + private static float3 Write(float3 value, WriteAction action) + { + action(ref value); + return value; + } + + private delegate void WriteAction(ref float3 value); + + private static void AssertRows(float3x3 m, params float[] expected) + { + var actual = new[] { m._m00, m._m01, m._m02, m._m10, m._m11, m._m12, m._m20, m._m21, m._m22 }; + for (var index = 0; index < expected.Length; index++) + { + Assert.Equal(expected[index], actual[index], precision: 5); + } + } +} diff --git a/packages/cultmath/tests/CultMath.Tests/HlslSemanticsTests.cs.meta b/packages/cultmath/tests/CultMath.Tests/HlslSemanticsTests.cs.meta new file mode 100644 index 00000000..8158a66d --- /dev/null +++ b/packages/cultmath/tests/CultMath.Tests/HlslSemanticsTests.cs.meta @@ -0,0 +1,2 @@ +fileFormatVersion: 2 +guid: 7164d00a023e43578a31897347d8f106 diff --git a/packages/cultmath/tests/CultMath.Tests/HlslSourceCompatibilityTests.cs b/packages/cultmath/tests/CultMath.Tests/HlslSourceCompatibilityTests.cs new file mode 100644 index 00000000..ca4584c7 --- /dev/null +++ b/packages/cultmath/tests/CultMath.Tests/HlslSourceCompatibilityTests.cs @@ -0,0 +1,203 @@ +using System.Reflection; +using System.Text.RegularExpressions; +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp; +using Xunit; +using static CultMath.math; + +namespace CultMath.Tests; + +/// +/// The real shader mirror, shaders/CultMath.hlsl, compiles as C# against CultMath under +/// using static CultMath.math; after exactly the source transformations listed in +/// docs/design.md (HLSL Target: Source Transformations). is +/// that list in code; if a body needs anything else, this test fails. +/// +public sealed class HlslSourceCompatibilityTests +{ + [Fact] + public void ShaderMirrorBodiesCompileAsCSharp() + { + var (assembly, errors) = CompileShaderMirror(); + Assert.True(assembly is not null, "HLSL mirror did not compile as C#:" + Environment.NewLine + string.Join(Environment.NewLine, errors)); + + var shader = assembly!.GetType("CultMathHlsl.HlslShader")!; + Assert.NotNull(ShaderFunction(shader, "cultmath_snoise", typeof(float3))); + Assert.NotNull(ShaderFunction(shader, "cultmath_snoise", typeof(float2))); + Assert.NotNull(ShaderFunction(shader, "cultmath_value_noise_bicubic", typeof(float2))); + } + + // HLSL functions carry no access modifier, so they compile as private instance methods. + private static MethodInfo? ShaderFunction(Type shader, string name, params Type[] parameters) => + shader.GetMethod(name, BindingFlags.Instance | BindingFlags.NonPublic, null, parameters, null); + + // Mirror internals with no public C# math counterpart. Any other cultmath_* function must match one. + private static readonly string[] MirrorOnly = { "cultmath_snoise_mod289", "cultmath_snoise_permute" }; + + private static readonly float[] Specials = { 0.0f, -0.0f, 1.0f, -1.0f, 0.5f, -2.75f, 3.0f, 1.0e-7f, 1.0e4f, -1.0e4f, 1.0e7f, float.NaN }; + + /// + /// Every mirror function with a C# math counterpart returns the same bits (any NaN equals any NaN; + /// -0 differs from 0) over random inputs, every special value in every argument (ties, zeros, + /// negatives, NaN, large values, integer lattice points), special pairs, and vectors whose first two + /// components tie (x0.x == x0.y in snoise). This proves the text of CultMath.hlsl computes the same + /// float32 results as C# math on the CPU. It does not prove GPU agreement: driver sin precision alone + /// makes cultmath_hash and value noise differ bit for bit on hardware. The mirror's intrinsics are C# + /// math itself (it compiles under using static CultMath.math), so this proves the composition in the + /// text, not the intrinsic rules; HlslSemanticsTests alone pins those. + /// + [Fact] + public void EveryMirrorFunctionMatchesCSharpMath() + { + var (assembly, errors) = CompileShaderMirror(); + Assert.True(assembly is not null, string.Join(Environment.NewLine, errors)); + var shaderType = assembly!.GetType("CultMathHlsl.HlslShader")!; + var shader = Activator.CreateInstance(shaderType); + + var random = new System.Random(0x5EED); + float Next() => random.NextSingle() * 200.0f - 100.0f; + var cases = new List>(); + foreach (var s in Specials) cases.Add(_ => s); + for (var k = 0; k < Specials.Length; k++) { var o = k; cases.Add(i => Specials[(i + o + 1) % Specials.Length]); } + for (var k = 0; k < 64; k++) { var values = Enumerable.Range(0, 32).Select(_ => Next()).ToArray(); cases.Add(i => values[i]); } + for (var k = 0; k < 16; k++) { var values = Enumerable.Range(0, 32).Select(_ => Next()).ToArray(); cases.Add(i => values[i & ~1]); } + + var mismatches = new List(); + var compared = new HashSet(); + foreach (var mirror in shaderType.GetMethods(BindingFlags.Instance | BindingFlags.NonPublic).Where(m => m.Name.StartsWith("cultmath_"))) + { + var types = mirror.GetParameters().Select(p => p.ParameterType).ToArray(); + var counterpart = typeof(math).GetMethod(mirror.Name["cultmath_".Length..], BindingFlags.Public | BindingFlags.Static, null, types, null); + if (counterpart is null) + { + Assert.Contains(mirror.Name, MirrorOnly); + continue; + } + + compared.Add(mirror.Name); + foreach (var input in cases) + { + var slot = 0; + var args = types.Select(t => t == typeof(float) ? (object)input(slot++) + : Activator.CreateInstance(t, Enumerable.Range(0, Components(t)).Select(_ => (object)input(slot++)).ToArray())!).ToArray(); + var expected = Floats(counterpart.Invoke(null, args)!); + var actual = Floats(mirror.Invoke(shader, args)!); + if (expected.Length != actual.Length || expected.Zip(actual).Any(p => !(float.IsNaN(p.First) && float.IsNaN(p.Second)) + && BitConverter.SingleToInt32Bits(p.First) != BitConverter.SingleToInt32Bits(p.Second))) + { + mismatches.Add($"{mirror.Name}({string.Join(", ", args.Select(a => string.Join(" ", Floats(a))))}): C# {string.Join(" ", expected)}, mirror {string.Join(" ", actual)}"); + } + } + } + + Assert.True(mismatches.Count == 0, $"{mismatches.Count} mismatches:{Environment.NewLine}{string.Join(Environment.NewLine, mismatches.Take(12))}"); + Assert.Equal(26, compared.Count); + } + + private static int Components(Type t) => t.GetFields().Count(f => f.FieldType == typeof(float) && !f.IsStatic); + + private static float[] Floats(object value) => value is float f ? new[] { f } + : value.GetType().GetFields().Where(field => field.FieldType == typeof(float) && !field.IsStatic).Select(field => (float)field.GetValue(value)!).ToArray(); + + /// The documented HLSL-to-C# transformations, and nothing else. + internal static string TransformHlslToCSharp(string hlsl) + { + // 1. Include guards (#ifndef/#define/#endif) are dropped; C# has no textual include. + var source = Regex.Replace(hlsl, @"(?m)^\s*#(ifndef|define|endif)\b.*$", string.Empty); + + // 2. Functions taking Texture2D/SamplerState resources are dropped; GPU resources have + // no CultMath analog. + source = Regex.Replace(source, @"(?ms)^\w+ \w+\([^)]*\b(Texture2D|SamplerState)\b[^)]*\)\s*\{.*?^\}[ \t]*\r?$", string.Empty); + + // 3. File-scope `static const` becomes `const` (C# consts are implicitly static). + source = Regex.Replace(source, @"(?m)^static const ", "const "); + + // 4. Floating literals gain the `f` suffix; an unsuffixed C# literal is a double. + source = Regex.Replace(source, @"(? Errors) CompileShaderMirror() + { + var hlsl = File.ReadAllText(Path.Combine(FindCultMathRoot(), "shaders", "CultMath.hlsl")); + var tree = CSharpSyntaxTree.ParseText(TransformHlslToCSharp(hlsl)); + var references = ((string)AppContext.GetData("TRUSTED_PLATFORM_ASSEMBLIES")!) + .Split(Path.PathSeparator) + .Append(typeof(math).Assembly.Location) + .Distinct(StringComparer.OrdinalIgnoreCase) + .Select(path => MetadataReference.CreateFromFile(path)); + var compilation = CSharpCompilation.Create( + "CultMathHlslMirror", + new[] { tree }, + references, + new CSharpCompilationOptions(OutputKind.DynamicallyLinkedLibrary)); + + using var stream = new MemoryStream(); + var result = compilation.Emit(stream); + var errors = result.Diagnostics + .Where(diagnostic => diagnostic.Severity == DiagnosticSeverity.Error) + .Select(diagnostic => diagnostic.ToString()) + .ToList(); + return (result.Success ? Assembly.Load(stream.ToArray()) : null, errors); + } + + private static string FindCultMathRoot() + { + var directory = new DirectoryInfo(AppContext.BaseDirectory); + while (directory is not null && !File.Exists(Path.Combine(directory.FullName, "packages", "cultmath", "shaders", "CultMath.hlsl"))) + { + directory = directory.Parent; + } + + return Path.Combine(directory?.FullName ?? throw new InvalidOperationException("CultMath package root not found."), "packages", "cultmath"); + } + + // ---- Hand-written HLSL-shaped host code: swizzles, mixed constructors, matrix rows. ---- + static float3 shade(float3 position, float3 normal, float2 wind, float time) + { + float3 p = position; + p.xz = p.xz + wind * time; + p.y = saturate(p.y); + + float3x3 basis = float3x3(float3(1, 0, 0), float3(0, 1, 0), float3(0, 0, 1)); + basis[1][1] = 1.0f; + float3 n = normalize(mul(basis, normal)); + float ndotl = saturate(dot(n, normalize(float3(0.3f, 1, 0.2f)))); + float4 color = float4(lerp(float3(0.1f, 0.1f, 0.2f), float3(1, 0.9f, 0.7f), ndotl), 1); + + if (any(p < 0.0f)) + color.rgb *= 0.5f; + if (all(abs(n) <= 1.0f)) + color.bgr = color.bgr * float3(1, 1, 1); + + color.rgb = select(p > 10.0f, float3(0), color.rgb); + float2 uv = frac(p.xz * 0.25f); + color.rg = lerp(color.rg, uv, 0.1f); + return float4(color.xy, color.zw).xyz; + } + + static float2 steer(float2 direction, float torque) + { + float2x2 turn = float2x2(cos(torque), -sin(torque), sin(torque), cos(torque)); + float2 turned = mul(direction, turn); + return normalize(turned) * pow(length(turned), 1.0f); + } + + [Fact] + public void HandWrittenShaderStyleMathEvaluates() + { + var color = shade(float3(1.0f, 0.5f, 2.0f), float3(0.0f, 1.0f, 0.0f), float2(1.0f, 0.0f), 0.0f); + Assert.True(color.x > 0.1f && color.x <= 1.0f); + Assert.True(float.IsFinite(color.y) && float.IsFinite(color.z)); + + var far = shade(float3(20.0f, 20.0f, 20.0f), float3(0.0f, 1.0f, 0.0f), float2(0.0f), 0.0f); + Assert.Equal(0.0f, far.z); + + var steered = steer(float2(1.0f, 0.0f), 0.5f); + var viaRotate = mul(float2(1.0f, 0.0f), CultMath.float2x2.Rotate(0.5f)); + Assert.Equal(viaRotate.x, steered.x, precision: 5); + Assert.Equal(viaRotate.y, steered.y, precision: 5); + } +} diff --git a/packages/cultmath/tests/CultMath.Tests/HlslSourceCompatibilityTests.cs.meta b/packages/cultmath/tests/CultMath.Tests/HlslSourceCompatibilityTests.cs.meta new file mode 100644 index 00000000..0d673e1e --- /dev/null +++ b/packages/cultmath/tests/CultMath.Tests/HlslSourceCompatibilityTests.cs.meta @@ -0,0 +1,2 @@ +fileFormatVersion: 2 +guid: 942fbcb9c28a4e559e5baf435dc24cf1 diff --git a/packages/cultmath/tests/CultMath.Tests/MathTests.cs b/packages/cultmath/tests/CultMath.Tests/MathTests.cs new file mode 100644 index 00000000..de1bf996 --- /dev/null +++ b/packages/cultmath/tests/CultMath.Tests/MathTests.cs @@ -0,0 +1,335 @@ +using CultMath; +using Xunit; + +namespace CultMath.Tests; + +public sealed class MathTests +{ + [Fact] + public void FracMatchesShaderStylePositiveAndNegativeValues() + { + Assert.Equal(0.25f, math.frac(1.25f)); + Assert.Equal(0.75f, math.frac(-1.25f)); + } + + [Fact] + public void VectorOperationsAreComponentWise() + { + var value = new float3(1.0f, 2.0f, 3.0f) * new float3(4.0f, 5.0f, 6.0f); + + Assert.Equal(new float3(4.0f, 10.0f, 18.0f), value); + } + + [Fact] + public void VectorConstructorsAcceptLowerDimensionVectors() + { + Assert.Equal(new int2(1, 2), math.int2(1, 2)); + Assert.Equal(new float2(1.0f, 2.0f), math.float2(1.0f, 2.0f)); + Assert.Equal(new float3(1.0f, 1.0f, 1.0f), math.float3(1.0f)); + Assert.Equal(new float4(1.0f, 2.0f, 3.0f, 4.0f), math.float4(new float3(1.0f, 2.0f, 3.0f), 4.0f)); + Assert.Equal(new float3(1.0f, 2.0f, 3.0f), new float3(new float2(1.0f, 2.0f), 3.0f)); + Assert.Equal(new float4(1.0f, 2.0f, 3.0f, 4.0f), new float4(new float2(1.0f, 2.0f), 3.0f, 4.0f)); + Assert.Equal(new float4(1.0f, 2.0f, 3.0f, 4.0f), new float4(new float3(1.0f, 2.0f, 3.0f), 4.0f)); + } + + [Fact] + public void VectorsExposeCommonSwizzles() + { + Assert.Equal(new float3(2.0f, 3.0f, 1.0f), new float3(1.0f, 2.0f, 3.0f).yzx); + Assert.Equal(new float3(3.0f, 2.0f, 1.0f), new float3(1.0f, 2.0f, 3.0f).zyx); + Assert.Equal(new float2(3.0f, 2.0f), new float3(1.0f, 2.0f, 3.0f).zy); + Assert.Equal(new float3(1.0f, 2.0f, 4.0f), new float4(1.0f, 2.0f, 3.0f, 4.0f).xyw); + Assert.Equal(new float3(2.0f, 3.0f, 1.0f), new float4(1.0f, 2.0f, 3.0f, 4.0f).yzx); + Assert.Equal(new float4(1.0f, 2.0f, 3.0f, 4.0f), new float4(1.0f, new float3(2.0f, 3.0f, 4.0f))); + Assert.Equal(new float4(1.0f, 2.0f, 3.0f, 4.0f), math.float4(1.0f, new float3(2.0f, 3.0f, 4.0f))); + } + + [Fact] + public void VectorsSupportUnityStyleFieldAndSwizzleMutation() + { + var position = new float3(1.0f, 2.0f, 3.0f); + position.y = 5.0f; + position.xz = new float2(8.0f, 13.0f); + + Assert.Equal(new float3(8.0f, 5.0f, 13.0f), position); + + var bounds = new float4(1.0f, 2.0f, 3.0f, 4.0f); + bounds.xy = new float2(-1.0f, -2.0f); + + Assert.Equal(new float4(-1.0f, -2.0f, 3.0f, 4.0f), bounds); + } + + [Fact] + public void DoubleVectorsSupportDampedMath() + { + var value = new double3(1.0, 2.0, 3.0); + value.xy = new double2(5.0, 8.0); + + Assert.Equal(new double3(5.0, 8.0, 3.0), value); + Assert.Equal(new double2(Math.Exp(-2.0), Math.Exp(-4.0)), math.exp(new double2(-2.0, -4.0))); + Assert.Equal(new double3(2.0, 4.0, 6.0), math.lerp(new double3(0.0, 0.0, 0.0), new double3(4.0, 8.0, 12.0), new double3(0.5, 0.5, 0.5))); + } + + [Fact] + public void Int2SupportsGridArithmeticAndFloatPromotion() + { + var cell = new int2(2, 3); + + Assert.Equal(new int2(3, 5), cell + new int2(1, 2)); + Assert.Equal(new int2(4, 6), cell * new int2(2, 2)); + Assert.Equal(3, cell[1]); + + float2 promoted = cell; + Assert.Equal(new float2(2.0f, 3.0f), promoted); + } + + [Fact] + public void Bool2SupportsHullGridStyleStorage() + { + bool2 conductivity = true; + + conductivity.y = false; + + Assert.True(conductivity.x); + Assert.False(conductivity[1]); + Assert.Equal(new bool2(false, true), math.bool2(false, true)); + } + + [Fact] + public void QuaternionLookRotationProducesUnitOrientation() + { + var rotation = quaternion.LookRotation(new float3(0.0f, 0.0f, 1.0f), new float3(0.0f, 1.0f, 0.0f)); + + Assert.Equal(quaternion.identity, rotation); + Assert.Equal(1.0f, math.length((float4)quaternion.LookRotation(new float3(1.0f, 0.0f, 0.0f), new float3(0.0f, 1.0f, 0.0f))), precision: 5); + } + + [Fact] + public void RandomUsesStableXorShift32Sequence() + { + var a = new Random(1234); + var b = new Random(1234); + + Assert.Equal(332584831u, a.NextUInt()); + Assert.Equal(1855942593u, a.NextUInt()); + Assert.Equal(4018585650u, a.NextUInt()); + Assert.Equal(3348358578u, a.NextUInt()); + Assert.Equal(332584831u, b.NextUInt()); + } + + [Fact] + public void RandomProvidesDeterministicPortableSampling() + { + var a = new Random(1234); + var b = new Random(1234); + + Assert.Equal(a.NextUInt(), b.NextUInt()); + Assert.Equal(a.NextFloat(), b.NextFloat()); + + var bounded = a.NextFloat(-2.0f, 4.0f); + Assert.True(bounded >= -2.0f && bounded < 4.0f); + + var direction = a.NextFloat2Direction(); + Assert.Equal(1.0f, math.length(direction), precision: 5); + + var point = a.NextFloat3(new float3(-1.0f, -2.0f, -3.0f), new float3(1.0f, 2.0f, 3.0f)); + Assert.True(point.x >= -1.0f && point.x < 1.0f); + Assert.True(point.y >= -2.0f && point.y < 2.0f); + Assert.True(point.z >= -3.0f && point.z < 3.0f); + } + + [Fact] + public void NormalizeProducesUnitLength() + { + var value = math.normalize(new float3(3.0f, 4.0f, 0.0f)); + + Assert.Equal(1.0f, math.length(value), precision: 5); + } + + [Fact] + public void SmoothstepUsesHermiteRamp() + { + Assert.Equal(0.0f, math.smoothstep(0.0f, 1.0f, -1.0f)); + Assert.Equal(0.5f, math.smoothstep(0.0f, 1.0f, 0.5f), precision: 5); + Assert.Equal(1.0f, math.smoothstep(0.0f, 1.0f, 2.0f)); + Assert.Equal(0.5f, math.smootherstep(0.5f), precision: 5); + } + + [Fact] + public void TrigIntrinsicsUseRadians() + { + Assert.Equal(1.0f, math.sin(math.HALF_PI), precision: 5); + Assert.Equal(-1.0f, math.cos(math.PI), precision: 5); + Assert.Equal(1.0f, math.tan(math.PI * 0.25f), precision: 5); + Assert.Equal(math.HALF_PI, math.atan2(1.0f, 0.0f), precision: 5); + Assert.Equal(2, math.min(2, 5)); + Assert.Equal(5, math.max(2, 5)); + Assert.True(math.isinf(float.PositiveInfinity)); + } + + [Fact] + public void SignIsComponentWiseForSimulationVectors() + { + Assert.Equal(-1, math.sign(-12.0f)); + Assert.Equal(0, math.sign(0.0f)); + Assert.Equal(1, math.sign(12.0f)); + Assert.Equal(new int3(-1, 0, 1), math.sign(new float3(-2.0f, 0.0f, 4.0f))); + } + + [Fact] + public void RotateTurnsFloat2CounterClockwiseInRadians() + { + var rotated = math.rotate(new float2(1.0f, 0.0f), math.HALF_PI); + + Assert.Equal(0.0f, rotated.x, precision: 5); + Assert.Equal(1.0f, rotated.y, precision: 5); + + var rotatedDegrees = math.rotate_degrees(new float2(0.0f, 1.0f), 90.0f); + Assert.Equal(-1.0f, rotatedDegrees.x, precision: 5); + Assert.Equal(0.0f, rotatedDegrees.y, precision: 5); + } + + [Fact] + public void AetheriaPrimitivesRemainSmallAndDeterministic() + { + Assert.Equal(6.0f, math.csum(new float3(1.0f, 2.0f, 3.0f))); + Assert.Equal(0.25f, math.unlerp(10.0f, 30.0f, 15.0f)); + Assert.Equal(new float2(0.25f, 0.75f), math.unlerp(new float2(10.0f, 10.0f), new float2(30.0f, 30.0f), new float2(15.0f, 25.0f))); + Assert.Equal(MathF.Exp(-2.0f), math.decay(1.0f, 2.0f, 1.0f), precision: 5); + Assert.Equal(1.0f - MathF.Exp(-2.0f), math.damp(0.0f, 1.0f, 2.0f, 1.0f), precision: 5); + + Assert.Equal(15.0f, math.catmullrom(0.0f, 10.0f, 20.0f, 30.0f, 0.5f), precision: 5); + Assert.Equal(0.25f, math.quadratic_bezier(0.0f, 0.0f, 1.0f, 0.5f), precision: 5); + Assert.Equal(0.5f, math.cubic_bezier(0.0f, 0.0f, 1.0f, 1.0f, 0.5f), precision: 5); + } + + [Fact] + public void SegmentDistanceReportsClosestPoint() + { + var distance = math.distance_to_segment(new float2(2.0f, 3.0f), new float2(0.0f, 0.0f), new float2(4.0f, 0.0f), out var closest); + + Assert.Equal(3.0f, distance, precision: 5); + Assert.Equal(new float2(2.0f, 0.0f), closest); + } + + [Fact] + public void RectNormalizesAndDerivesFromMinMax() + { + var bounds = new rect(8.0f, 6.0f, 2.0f, -2.0f); + + Assert.Equal(new float2(2.0f, -2.0f), bounds.min); + Assert.Equal(new float2(8.0f, 6.0f), bounds.max); + Assert.Equal(new float2(6.0f, 8.0f), bounds.size); + Assert.Equal(new float2(5.0f, 2.0f), bounds.center); + Assert.Equal(48.0f, bounds.area); + } + + [Fact] + public void RectContainmentIncludesBoundary() + { + var bounds = new rect(-2.0f, -1.0f, 4.0f, 3.0f); + + Assert.True(bounds.Contains(new float2(-2.0f, -1.0f))); + Assert.True(bounds.Contains(new float2(4.0f, 3.0f))); + Assert.True(bounds.Contains(new rect(-1.0f, 0.0f, 1.0f, 2.0f))); + Assert.False(bounds.Contains(new float2(4.001f, 3.0f))); + } + + [Fact] + public void RectIntersectionTreatsTouchingEdgesAsContact() + { + var left = new rect(0.0f, 0.0f, 4.0f, 4.0f); + var touching = new rect(4.0f, 1.0f, 8.0f, 3.0f); + var separate = new rect(4.001f, 1.0f, 8.0f, 3.0f); + + Assert.True(left.Intersects(touching)); + Assert.Equal(0.0f, left.Intersection(touching).area); + Assert.False(left.Intersects(separate)); + } + + [Fact] + public void FirstOrderInterceptSolvesSimpleClosingTarget() + { + var time = math.first_order_intercept_time( + 2.0f, + new float3(10.0f, 0.0f, 0.0f), + new float3(-1.0f, 0.0f, 0.0f)); + + Assert.Equal(10.0f / 3.0f, time, precision: 5); + } + + [Fact] + public void ValueNoiseIsDeterministicAndSmoothAtCellCorners() + { + var position = new float2(12.25f, -4.5f); + + Assert.Equal(math.value_noise(position), math.value_noise(position)); + Assert.Equal(math.hash(new float2(3.0f, 5.0f)), math.value_noise(new float2(3.0f, 5.0f)), precision: 5); + + var bicubic = math.value_noise_bicubic(position); + Assert.True(bicubic > -0.25f && bicubic < 1.25f); + } + + [Theory] + [InlineData(0.0f, 0.0f, 0.0f)] + [InlineData(0.25f, -0.5f, -0.425866783f)] + [InlineData(12.25f, -4.5f, 0.258312f)] + [InlineData(536.5106f, 536.5106f, -0.6901103f)] + public void SimplexNoiseMatchesUnityMathematics(float x, float y, float expected) + { + Assert.Equal(expected, math.snoise(new float2(x, y)), precision: 6); + } + + [Fact] + public void VoronoiBatchProducesToneMappedColors() + { + var xs = new[] { 0.0f, 24.0f, 96.0f }; + var ys = new[] { 0.0f, 36.0f, 144.0f }; + var tones = new[] { CultMathTone.Background, CultMathTone.Header, CultMathTone.Body }; + var spans = new[] { 1920.0f, 8.0f, 8.0f }; + var colors = new Color32[xs.Length]; + + Voronoi.SampleTones(xs, ys, tones, spans, 1080.0f, 12, colors); + + Assert.NotEqual(new Color32(), colors[0]); + Assert.True(colors[1].r > colors[1].g); + Assert.True(colors[2].r > 120 && colors[2].g > 120 && colors[2].b > 120); + } + + [Fact] + public void BatchRadialFalloffAccelerationMatchesScalarContract() + { + var xs = new[] { 0.0f, 5.0f, 10.0f }; + var ys = new[] { 0.0f, 0.0f, 0.0f }; + var ax = new float[xs.Length]; + var ay = new float[xs.Length]; + + BatchMath.AddRadialFalloffAcceleration2D(xs, ys, 10.0f, 0.0f, 10.0f, 20.0f, ax, ay); + + Assert.True(BatchMath.LaneCount >= 4); + Assert.Equal(5.0f, ax[0], precision: 5); + Assert.Equal(7.5f, ax[1], precision: 5); + Assert.Equal(0.0f, ax[2], precision: 5); + Assert.Equal(0.0f, ay[0], precision: 5); + } + + [Fact] + public void BatchEulerIntegrationHonorsDynamicMask() + { + var dynamicMask = new[] { 1.0f, 0.0f }; + var px = new[] { 0.0f, 0.0f }; + var py = new[] { 0.0f, 0.0f }; + var vx = new[] { 1.0f, 1.0f }; + var vy = new[] { 0.0f, 0.0f }; + var ax = new[] { 2.0f, 2.0f }; + var ay = new[] { 0.0f, 4.0f }; + + BatchMath.IntegrateSemiImplicitEuler2D(0.5f, dynamicMask, px, py, vx, vy, ax, ay); + + Assert.Equal(2.0f, vx[0], precision: 5); + Assert.Equal(1.0f, px[0], precision: 5); + Assert.Equal(1.0f, vx[1], precision: 5); + Assert.Equal(0.0f, px[1], precision: 5); + Assert.Equal(0.0f, py[1], precision: 5); + } +} diff --git a/packages/cultmath/tests/CultMath.Tests/MathTests.cs.meta b/packages/cultmath/tests/CultMath.Tests/MathTests.cs.meta new file mode 100644 index 00000000..277df5a3 --- /dev/null +++ b/packages/cultmath/tests/CultMath.Tests/MathTests.cs.meta @@ -0,0 +1,2 @@ +fileFormatVersion: 2 +guid: c7a709e5345da214aa650e8c796fcaff \ No newline at end of file diff --git a/packages/cultmath/tools.meta b/packages/cultmath/tools.meta new file mode 100644 index 00000000..3594fa6a --- /dev/null +++ b/packages/cultmath/tools.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: 697162910ffd9614ba9655d31af9afa4 +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/packages/cultmath/tools/compile-hlsl-spirv.ps1 b/packages/cultmath/tools/compile-hlsl-spirv.ps1 new file mode 100644 index 00000000..280e08d5 --- /dev/null +++ b/packages/cultmath/tools/compile-hlsl-spirv.ps1 @@ -0,0 +1,69 @@ +param( + [Parameter(Mandatory = $true)] + [string]$ShaderPath, + + [Parameter(Mandatory = $true)] + [string]$OutputPath, + + [string]$EntryPoint = "main", + [string]$Profile = "cs_6_0", + [string[]]$IncludePath = @(), + [string]$TargetEnv = "", + [int]$VulkanCBufferShift = 0, + [int]$VulkanTextureShift = 10, + [int]$VulkanUavShift = 20, + [int]$VulkanSamplerShift = 30, + [string]$DxcPath = "" +) + +$ErrorActionPreference = "Stop" + +$repoRoot = Resolve-Path (Join-Path $PSScriptRoot "..") +if (-not $DxcPath) { + $DxcPath = Join-Path $repoRoot ".tools\dxc\current\bin\x64\dxc.exe" +} + +if (-not (Test-Path $DxcPath)) { + throw "DXC not found at '$DxcPath'. Run tools\get-dxc.ps1 first or pass -DxcPath." +} + +$resolvedShader = Resolve-Path $ShaderPath +$resolvedIncludes = @((Join-Path $repoRoot "shaders")) +foreach ($path in $IncludePath) { + $resolvedIncludes += (Resolve-Path $path) +} + +$outputDirectory = Split-Path -Parent $OutputPath +if ($outputDirectory) { + New-Item -ItemType Directory -Force -Path $outputDirectory | Out-Null +} + +$arguments = @( + "-T", $Profile, + "-E", $EntryPoint, + "-spirv" +) + +if ($TargetEnv) { + $arguments += "-fspv-target-env=$TargetEnv" +} + +$arguments += @( + "-fvk-b-shift", $VulkanCBufferShift.ToString(), "0", + "-fvk-t-shift", $VulkanTextureShift.ToString(), "0", + "-fvk-u-shift", $VulkanUavShift.ToString(), "0", + "-fvk-s-shift", $VulkanSamplerShift.ToString(), "0" +) + +foreach ($include in $resolvedIncludes) { + $arguments += @("-I", $include) +} + +$arguments += @($resolvedShader, "-Fo", $OutputPath) + +& $DxcPath @arguments +if ($LASTEXITCODE -ne 0) { + throw "dxc.exe failed with exit code $LASTEXITCODE." +} + +Get-Item -LiteralPath $OutputPath diff --git a/packages/cultmath/tools/compile-hlsl-spirv.ps1.meta b/packages/cultmath/tools/compile-hlsl-spirv.ps1.meta new file mode 100644 index 00000000..31e7948a --- /dev/null +++ b/packages/cultmath/tools/compile-hlsl-spirv.ps1.meta @@ -0,0 +1,7 @@ +fileFormatVersion: 2 +guid: b47e7016a99a5d148bc44c399c13c4d3 +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/packages/cultmath/tools/compile-spirv-metal.ps1 b/packages/cultmath/tools/compile-spirv-metal.ps1 new file mode 100644 index 00000000..490e6331 --- /dev/null +++ b/packages/cultmath/tools/compile-spirv-metal.ps1 @@ -0,0 +1,34 @@ +param( + [Parameter(Mandatory = $true)] + [string]$SpirvPath, + + [Parameter(Mandatory = $true)] + [string]$OutputPath, + + [int]$MslVersion = 23000, + [string]$SpirvCrossPath = "" +) + +$ErrorActionPreference = "Stop" + +$repoRoot = Resolve-Path (Join-Path $PSScriptRoot "..") +if (-not $SpirvCrossPath) { + $SpirvCrossPath = Join-Path $repoRoot ".tools\spirv-cross\build\spirv-cross.exe" +} + +if (-not (Test-Path $SpirvCrossPath)) { + throw "SPIRV-Cross not found at '$SpirvCrossPath'. Run tools\get-spirv-cross.ps1 first or pass -SpirvCrossPath." +} + +$resolvedSpirv = Resolve-Path $SpirvPath +$outputDirectory = Split-Path -Parent $OutputPath +if ($outputDirectory) { + New-Item -ItemType Directory -Force -Path $outputDirectory | Out-Null +} + +& $SpirvCrossPath $resolvedSpirv --msl --msl-version $MslVersion --output $OutputPath +if ($LASTEXITCODE -ne 0) { + throw "spirv-cross failed with exit code $LASTEXITCODE." +} + +Get-Item -LiteralPath $OutputPath diff --git a/packages/cultmath/tools/compile-spirv-metal.ps1.meta b/packages/cultmath/tools/compile-spirv-metal.ps1.meta new file mode 100644 index 00000000..2e7e74c2 --- /dev/null +++ b/packages/cultmath/tools/compile-spirv-metal.ps1.meta @@ -0,0 +1,7 @@ +fileFormatVersion: 2 +guid: 6150c8f461e471b4c8641fd32d1984ef +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/packages/cultmath/tools/generate-swizzles.ps1 b/packages/cultmath/tools/generate-swizzles.ps1 new file mode 100644 index 00000000..9030f2be --- /dev/null +++ b/packages/cultmath/tools/generate-swizzles.ps1 @@ -0,0 +1,132 @@ +param( + [string]$OutputPath = "" +) + +# Writes src\CultMath\Swizzles.g.cs: HLSL swizzles and mixed constructors for +# CultMath's vector types. The output is checked in; rerun after changing the +# family table below and commit both files together. + +$ErrorActionPreference = "Stop" + +if (-not $OutputPath) { + $OutputPath = Join-Path $PSScriptRoot "..\src\CultMath\Swizzles.g.cs" +} + +# Functions: emit math.(...) constructor functions. double has none because +# a math.double3 method group would hide the double3 type name under +# `using static CultMath.math;` and HLSL shader code does not construct doubles. +$families = @( + @{ Scalar = "float"; Sizes = @(2, 3, 4); Functions = $true }, + @{ Scalar = "int"; Sizes = @(2, 3, 4); Functions = $true }, + @{ Scalar = "bool"; Sizes = @(2, 3, 4); Functions = $true }, + @{ Scalar = "double"; Sizes = @(2, 3); Functions = $false } +) +$componentFields = @("x", "y", "z", "w") +$namings = @(@("x", "y", "z", "w"), @("r", "g", "b", "a")) + +function Get-Compositions([int]$total) { + # Ordered part sizes summing to $total, excluding the all-scalar form that the + # hand-written base constructor already owns. + $results = [System.Collections.Generic.List[object]]::new() + function Expand([int]$remaining, [int[]]$prefix) { + if ($remaining -eq 0) { + if (@($prefix | Where-Object { $_ -ne 1 }).Count -gt 0) { + $results.Add($prefix) + } + return + } + for ($part = 1; $part -le $remaining; $part++) { + Expand ($remaining - $part) ($prefix + $part) + } + } + Expand $total @() + return , $results +} + +$builder = [System.Text.StringBuilder]::new() +function Line([string]$text = "") { [void]$builder.Append($text).Append("`n") } + +Line "// " +Line "// Generated by tools/generate-swizzles.ps1. Do not edit by hand." +Line "// " +Line "" +Line "namespace CultMath;" + +$functionLines = [System.Collections.Generic.List[string]]::new() + +foreach ($family in $families) { + $scalar = $family.Scalar + $maxSize = ($family.Sizes | Measure-Object -Maximum).Maximum + foreach ($size in $family.Sizes) { + $type = "$scalar$size" + Line "" + Line "public partial struct $type" + Line "{" + + foreach ($composition in (Get-Compositions $size)) { + $parameters = @() + $arguments = @() + $offset = 0 + foreach ($part in $composition) { + $name = -join $componentFields[$offset..($offset + $part - 1)] + if ($part -eq 1) { + $parameters += "$scalar $name" + $arguments += $name + } else { + $parameters += "$scalar$part $name" + for ($i = 0; $i -lt $part; $i++) { $arguments += "$name.$($componentFields[$i])" } + } + $offset += $part + } + $parameterList = $parameters -join ", " + Line " public $type($parameterList) : this($($arguments -join ', ')) { }" + if ($family.Functions) { + $names = @($parameters | ForEach-Object { ($_ -split " ")[1] }) + $functionLines.Add(" public static $type $type($parameterList) => new($($names -join ', '));") + } + } + + # Single-component color aliases; x/y/z/w are the fields themselves. + $letters = $namings[1] + for ($i = 0; $i -lt $size; $i++) { + Line " public $scalar $($letters[$i]) { readonly get => $($componentFields[$i]); set => $($componentFields[$i]) = value; }" + } + + foreach ($letters in $namings) { + for ($length = 2; $length -le $maxSize; $length++) { + $count = [math]::Pow($size, $length) + for ($index = 0; $index -lt $count; $index++) { + [int[]]$digits = @() + [int]$remainder = $index + for ($d = 0; $d -lt $length; $d++) { + $digits = @([int]($remainder % $size)) + $digits + $remainder = [int][math]::Floor($remainder / $size) + } + $name = -join ($digits | ForEach-Object { $letters[$_] }) + $getter = ($digits | ForEach-Object { $componentFields[$_] }) -join ", " + $resultType = "$scalar$length" + $unique = @($digits | Select-Object -Unique).Count -eq $length + if ($unique) { + $assignments = for ($d = 0; $d -lt $length; $d++) { "$($componentFields[$digits[$d]]) = value.$($componentFields[$d]);" } + Line " public $resultType $name { readonly get => new($getter); set { $($assignments -join ' ') } }" + } else { + Line " public readonly $resultType $name => new($getter);" + } + } + } + } + + Line "}" + } +} + +Line "" +Line "public static partial class math" +Line "{" +foreach ($functionLine in $functionLines) { Line $functionLine } +Line "}" + +$fullPath = [System.IO.Path]::GetFullPath($OutputPath) +$content = $builder.ToString() -replace "`n", "`r`n" +[System.IO.File]::WriteAllText($fullPath, $content, [System.Text.UTF8Encoding]::new($false)) +Write-Host "Wrote $fullPath ($((Get-Item -LiteralPath $fullPath).Length) bytes)" diff --git a/packages/cultmath/tools/generate-swizzles.ps1.meta b/packages/cultmath/tools/generate-swizzles.ps1.meta new file mode 100644 index 00000000..74842405 --- /dev/null +++ b/packages/cultmath/tools/generate-swizzles.ps1.meta @@ -0,0 +1,7 @@ +fileFormatVersion: 2 +guid: d7a796fd2d87489c9f055fd6e73513b5 +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/packages/cultmath/tools/get-dxc.ps1 b/packages/cultmath/tools/get-dxc.ps1 new file mode 100644 index 00000000..d9b12593 --- /dev/null +++ b/packages/cultmath/tools/get-dxc.ps1 @@ -0,0 +1,43 @@ +param( + [string]$InstallRoot = ".tools\dxc" +) + +$ErrorActionPreference = "Stop" + +$repoRoot = Resolve-Path (Join-Path $PSScriptRoot "..") +$installPath = Join-Path $repoRoot $InstallRoot +New-Item -ItemType Directory -Force -Path $installPath | Out-Null + +# Pinned: parity means this compiler's lowering (dxcompiler 1.9.0.5402). The hash is the SHA256 +# of dxc_2026_07_29.zip downloaded from Microsoft's GitHub release v1.9.2607, and matches the +# digest GitHub publishes for that asset. +$zipName = "dxc_2026_07_29.zip" +$zipUrl = "https://github.com/microsoft/DirectXShaderCompiler/releases/download/v1.9.2607/$zipName" +$zipSha256 = "A1DFB116BA3EEAE6A1582291B53A8E7BF65AD760676BD3194685C8F7367CD241" + +$zipPath = Join-Path $installPath $zipName +Remove-Item -LiteralPath $zipPath -Force -ErrorAction SilentlyContinue + +& curl.exe -L --retry 5 --retry-delay 2 --fail -o $zipPath $zipUrl +if ($LASTEXITCODE -ne 0) { + throw "curl.exe failed with exit code $LASTEXITCODE." +} + +$actualSha256 = (Get-FileHash -LiteralPath $zipPath -Algorithm SHA256).Hash +if ($actualSha256 -ne $zipSha256) { + Remove-Item -LiteralPath $zipPath -Force + throw "$zipName SHA256 is $actualSha256, expected $zipSha256." +} + +$currentPath = Join-Path $installPath "current" +Remove-Item -LiteralPath $currentPath -Recurse -Force -ErrorAction SilentlyContinue +New-Item -ItemType Directory -Force -Path $currentPath | Out-Null +Expand-Archive -LiteralPath $zipPath -DestinationPath $currentPath -Force + +$dxcPath = Join-Path $currentPath "bin\x64\dxc.exe" +if (-not (Test-Path $dxcPath)) { + throw "DXC x64 executable was not found after extraction." +} + +& $dxcPath --version +Write-Host "DXC installed at $dxcPath" diff --git a/packages/cultmath/tools/get-dxc.ps1.meta b/packages/cultmath/tools/get-dxc.ps1.meta new file mode 100644 index 00000000..c9ab0d82 --- /dev/null +++ b/packages/cultmath/tools/get-dxc.ps1.meta @@ -0,0 +1,7 @@ +fileFormatVersion: 2 +guid: 86d0b9a1006042745b7fff523831049c +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/packages/cultmath/tools/get-spirv-cross.ps1 b/packages/cultmath/tools/get-spirv-cross.ps1 new file mode 100644 index 00000000..f5105b9a --- /dev/null +++ b/packages/cultmath/tools/get-spirv-cross.ps1 @@ -0,0 +1,34 @@ +param( + [string]$InstallRoot = ".tools\spirv-cross" +) + +$ErrorActionPreference = "Stop" + +$repoRoot = Resolve-Path (Join-Path $PSScriptRoot "..") +$installPath = Join-Path $repoRoot $InstallRoot +$sourcePath = Join-Path $installPath "src" +$buildPath = Join-Path $installPath "build" + +New-Item -ItemType Directory -Force -Path $installPath | Out-Null + +if (-not (Test-Path $sourcePath)) { + git clone --depth 1 https://github.com/KhronosGroup/SPIRV-Cross.git $sourcePath +} + +cmake ` + -S $sourcePath ` + -B $buildPath ` + -G Ninja ` + -DSPIRV_CROSS_CLI=ON ` + -DSPIRV_CROSS_ENABLE_TESTS=OFF ` + -DSPIRV_CROSS_ENABLE_C_API=OFF + +cmake --build $buildPath --target spirv-cross --config Release + +$spirvCrossPath = Join-Path $buildPath "spirv-cross.exe" +if (-not (Test-Path $spirvCrossPath)) { + throw "SPIRV-Cross executable was not found after build." +} + +& $spirvCrossPath --version +Write-Host "SPIRV-Cross installed at $spirvCrossPath" diff --git a/packages/cultmath/tools/get-spirv-cross.ps1.meta b/packages/cultmath/tools/get-spirv-cross.ps1.meta new file mode 100644 index 00000000..12ccdd8e --- /dev/null +++ b/packages/cultmath/tools/get-spirv-cross.ps1.meta @@ -0,0 +1,7 @@ +fileFormatVersion: 2 +guid: dcf41da07502f4b439f8fc5ad25d16c9 +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/packages/cultmath/unity.meta b/packages/cultmath/unity.meta new file mode 100644 index 00000000..de47a618 --- /dev/null +++ b/packages/cultmath/unity.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: 366efeaef9f68224b80229332c7ecfc6 +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/packages/cultmath/unity/org.gamecult.cultmath/LICENSE.md b/packages/cultmath/unity/org.gamecult.cultmath/LICENSE.md new file mode 100644 index 00000000..5e50f2d6 --- /dev/null +++ b/packages/cultmath/unity/org.gamecult.cultmath/LICENSE.md @@ -0,0 +1,10 @@ +# CultMath License + +CultMath is licensed under the Mozilla Public License 2.0 (`MPL-2.0`). + +The complete license text is available from the Mozilla Foundation at +. Source and modification history for the +precompiled assembly in this package are available in the GameCult/CultMath +repository at the corresponding package tag. + +Third-party code retains its original license. See `THIRD-PARTY-NOTICES.md`. diff --git a/packages/cultmath/unity/org.gamecult.cultmath/LICENSE.md.meta b/packages/cultmath/unity/org.gamecult.cultmath/LICENSE.md.meta new file mode 100644 index 00000000..c234cd28 --- /dev/null +++ b/packages/cultmath/unity/org.gamecult.cultmath/LICENSE.md.meta @@ -0,0 +1,2 @@ +fileFormatVersion: 2 +guid: 7172bce28532878409e57f5fbbb12719 \ No newline at end of file diff --git a/packages/cultmath/unity/org.gamecult.cultmath/README.md b/packages/cultmath/unity/org.gamecult.cultmath/README.md new file mode 100644 index 00000000..20e1840c --- /dev/null +++ b/packages/cultmath/unity/org.gamecult.cultmath/README.md @@ -0,0 +1,36 @@ +# CultMath for Unity + +This is CultMath's Unity 2021.3-compatible Git-UPM surface. It contains the +precompiled `CultMath.dll`, portable symbols, the numeric +`Shaders/CultMath.hlsl` mirror, and the `CultMath.UnityBridge` assembly. + +`CultMath.UnityBridge` is the only C# source Unity compiles here. It holds the +UnityEngine conversions the engine-free core cannot declare: `ToCultMath()` on +`Vector2/3/4`, `Vector2Int/3Int`, `Quaternion`, and `Color`; `ToUnity()` on +`float2/3/4`, `int2/3`, and `quaternion`; `ToColor()` on `float4`. C# only +admits conversion operators on the source or target type, so these are +extension methods rather than implicit casts. Import `CultMath.UnityBridge` to +use them. + +Unity does not compile CultMath's repository source tree. That source uses the +current C# language and remains owned by the normal .NET project. The package +imports only the tracked auto-referenced precompiled assembly, so consumers do +not need `csc.rsp` language overrides or an asmdef propagation shim. + +Build and inspect the package from the repository root: + +```powershell +.\scripts\build-unity-package.ps1 +``` + +The builder fresh-builds CultMath for `netstandard2.1`, verifies the tracked +DLL/PDB and stable Unity metadata, and stages an inspectable package under +`artifacts/unity/org.gamecult.cultmath`. + +Consume the repository package with: + +```json +"org.gamecult.cultmath": "https://github.com/GameCult/CultLib.git?path=/packages/cultmath/unity/org.gamecult.cultmath" +``` + +Production consumers should pin that URL to a commit or release tag. diff --git a/packages/cultmath/unity/org.gamecult.cultmath/README.md.meta b/packages/cultmath/unity/org.gamecult.cultmath/README.md.meta new file mode 100644 index 00000000..0ea270c2 --- /dev/null +++ b/packages/cultmath/unity/org.gamecult.cultmath/README.md.meta @@ -0,0 +1,2 @@ +fileFormatVersion: 2 +guid: b7429934026d747489a3876fd15576e5 \ No newline at end of file diff --git a/packages/cultmath/unity/org.gamecult.cultmath/Runtime.meta b/packages/cultmath/unity/org.gamecult.cultmath/Runtime.meta new file mode 100644 index 00000000..b0e76599 --- /dev/null +++ b/packages/cultmath/unity/org.gamecult.cultmath/Runtime.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: a31bf2bb35e54dd1ad1c765337682769 +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/unity/org.gamecult.geometry/Runtime/Plugins.meta b/packages/cultmath/unity/org.gamecult.cultmath/Runtime/Plugins.meta similarity index 76% rename from unity/org.gamecult.geometry/Runtime/Plugins.meta rename to packages/cultmath/unity/org.gamecult.cultmath/Runtime/Plugins.meta index 98acd60d..fe8acafa 100644 --- a/unity/org.gamecult.geometry/Runtime/Plugins.meta +++ b/packages/cultmath/unity/org.gamecult.cultmath/Runtime/Plugins.meta @@ -1,5 +1,5 @@ fileFormatVersion: 2 -guid: 7f8484deafdf444abe3b1dedb9d09d16 +guid: f84156013e814c14b73f668b6610b9e0 folderAsset: yes DefaultImporter: externalObjects: {} diff --git a/packages/cultmath/unity/org.gamecult.cultmath/Runtime/Plugins/CultMath.dll b/packages/cultmath/unity/org.gamecult.cultmath/Runtime/Plugins/CultMath.dll new file mode 100644 index 00000000..bf712012 Binary files /dev/null and b/packages/cultmath/unity/org.gamecult.cultmath/Runtime/Plugins/CultMath.dll differ diff --git a/unity/org.gamecult.geometry/Runtime/Plugins/GameCult.Geometry.dll.meta b/packages/cultmath/unity/org.gamecult.cultmath/Runtime/Plugins/CultMath.dll.meta similarity index 90% rename from unity/org.gamecult.geometry/Runtime/Plugins/GameCult.Geometry.dll.meta rename to packages/cultmath/unity/org.gamecult.cultmath/Runtime/Plugins/CultMath.dll.meta index 4253270b..917e3849 100644 --- a/unity/org.gamecult.geometry/Runtime/Plugins/GameCult.Geometry.dll.meta +++ b/packages/cultmath/unity/org.gamecult.cultmath/Runtime/Plugins/CultMath.dll.meta @@ -1,5 +1,5 @@ fileFormatVersion: 2 -guid: 798aad4558fb450cb48da955d4a9b973 +guid: 063b8a850f0f474aa7e89c4167f97a19 PluginImporter: externalObjects: {} serializedVersion: 2 diff --git a/packages/cultmath/unity/org.gamecult.cultmath/Runtime/Plugins/CultMath.pdb b/packages/cultmath/unity/org.gamecult.cultmath/Runtime/Plugins/CultMath.pdb new file mode 100644 index 00000000..1eaadd23 Binary files /dev/null and b/packages/cultmath/unity/org.gamecult.cultmath/Runtime/Plugins/CultMath.pdb differ diff --git a/packages/cultmath/unity/org.gamecult.cultmath/Runtime/Plugins/CultMath.pdb.meta b/packages/cultmath/unity/org.gamecult.cultmath/Runtime/Plugins/CultMath.pdb.meta new file mode 100644 index 00000000..e18fedbe --- /dev/null +++ b/packages/cultmath/unity/org.gamecult.cultmath/Runtime/Plugins/CultMath.pdb.meta @@ -0,0 +1,7 @@ +fileFormatVersion: 2 +guid: a8cc555f643a4d7eb4dd340025b208f6 +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/packages/cultmath/unity/org.gamecult.cultmath/Runtime/UnityBridge.meta b/packages/cultmath/unity/org.gamecult.cultmath/Runtime/UnityBridge.meta new file mode 100644 index 00000000..d34f2ef0 --- /dev/null +++ b/packages/cultmath/unity/org.gamecult.cultmath/Runtime/UnityBridge.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: 01f9f472b0a645b1ab392567aa38911a +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/unity/org.gamecult.geometry/Runtime/GameCult.Geometry.asmdef b/packages/cultmath/unity/org.gamecult.cultmath/Runtime/UnityBridge/CultMath.UnityBridge.asmdef similarity index 54% rename from unity/org.gamecult.geometry/Runtime/GameCult.Geometry.asmdef rename to packages/cultmath/unity/org.gamecult.cultmath/Runtime/UnityBridge/CultMath.UnityBridge.asmdef index ea9a8c9f..c54da7da 100644 --- a/unity/org.gamecult.geometry/Runtime/GameCult.Geometry.asmdef +++ b/packages/cultmath/unity/org.gamecult.cultmath/Runtime/UnityBridge/CultMath.UnityBridge.asmdef @@ -1,19 +1,16 @@ { - "name": "GameCult.Geometry", - "rootNamespace": "GameCult.Geometry", - "references": [ - "CultMath", - "GameCult.CultLib" - ], + "name": "CultMath.UnityBridge", + "rootNamespace": "CultMath.UnityBridge", + "references": [], "includePlatforms": [], "excludePlatforms": [], "allowUnsafeCode": false, "overrideReferences": true, "precompiledReferences": [ - "GameCult.Geometry.dll" + "CultMath.dll" ], "autoReferenced": true, "defineConstraints": [], "versionDefines": [], - "noEngineReferences": true + "noEngineReferences": false } diff --git a/packages/cultmath/unity/org.gamecult.cultmath/Runtime/UnityBridge/CultMath.UnityBridge.asmdef.meta b/packages/cultmath/unity/org.gamecult.cultmath/Runtime/UnityBridge/CultMath.UnityBridge.asmdef.meta new file mode 100644 index 00000000..6875b788 --- /dev/null +++ b/packages/cultmath/unity/org.gamecult.cultmath/Runtime/UnityBridge/CultMath.UnityBridge.asmdef.meta @@ -0,0 +1,7 @@ +fileFormatVersion: 2 +guid: 77e50a05974842bc99d285ef6d5d04f7 +AssemblyDefinitionImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/packages/cultmath/unity/org.gamecult.cultmath/Runtime/UnityBridge/UnityConversions.cs b/packages/cultmath/unity/org.gamecult.cultmath/Runtime/UnityBridge/UnityConversions.cs new file mode 100644 index 00000000..c8289a77 --- /dev/null +++ b/packages/cultmath/unity/org.gamecult.cultmath/Runtime/UnityBridge/UnityConversions.cs @@ -0,0 +1,28 @@ +using UnityEngine; + +namespace CultMath.UnityBridge +{ + /// + /// Explicit conversions between UnityEngine and CultMath value types. + /// C# only admits user-defined conversion operators declared on the source or target + /// type, and CultMath's core types stay engine-free, so the bridge is extension methods. + /// + public static class UnityConversions + { + public static float2 ToCultMath(this Vector2 value) => new float2(value.x, value.y); + public static float3 ToCultMath(this Vector3 value) => new float3(value.x, value.y, value.z); + public static float4 ToCultMath(this Vector4 value) => new float4(value.x, value.y, value.z, value.w); + public static float4 ToCultMath(this Color value) => new float4(value.r, value.g, value.b, value.a); + public static quaternion ToCultMath(this Quaternion value) => new quaternion(value.x, value.y, value.z, value.w); + public static int2 ToCultMath(this Vector2Int value) => new int2(value.x, value.y); + public static int3 ToCultMath(this Vector3Int value) => new int3(value.x, value.y, value.z); + + public static Vector2 ToUnity(this float2 value) => new Vector2(value.x, value.y); + public static Vector3 ToUnity(this float3 value) => new Vector3(value.x, value.y, value.z); + public static Vector4 ToUnity(this float4 value) => new Vector4(value.x, value.y, value.z, value.w); + public static Quaternion ToUnity(this quaternion value) => new Quaternion(value.x, value.y, value.z, value.w); + public static Vector2Int ToUnity(this int2 value) => new Vector2Int(value.x, value.y); + public static Vector3Int ToUnity(this int3 value) => new Vector3Int(value.x, value.y, value.z); + public static Color ToColor(this float4 value) => new Color(value.x, value.y, value.z, value.w); + } +} diff --git a/unity/org.gamecult.geometry/Samples~/PlanetaryViewer/Editor/GameCultGeometryPlanetaryViewerMenu.cs.meta b/packages/cultmath/unity/org.gamecult.cultmath/Runtime/UnityBridge/UnityConversions.cs.meta similarity index 60% rename from unity/org.gamecult.geometry/Samples~/PlanetaryViewer/Editor/GameCultGeometryPlanetaryViewerMenu.cs.meta rename to packages/cultmath/unity/org.gamecult.cultmath/Runtime/UnityBridge/UnityConversions.cs.meta index 8e952599..ccbd3f10 100644 --- a/unity/org.gamecult.geometry/Samples~/PlanetaryViewer/Editor/GameCultGeometryPlanetaryViewerMenu.cs.meta +++ b/packages/cultmath/unity/org.gamecult.cultmath/Runtime/UnityBridge/UnityConversions.cs.meta @@ -1,11 +1,11 @@ fileFormatVersion: 2 -guid: 4cbba35a250d4e2aadc125df07f85a54 +guid: 8fc742177b3f4715b204afe0e69a400c MonoImporter: externalObjects: {} serializedVersion: 2 defaultReferences: [] executionOrder: 0 icon: {instanceID: 0} - userData: - assetBundleName: - assetBundleVariant: + userData: + assetBundleName: + assetBundleVariant: diff --git a/unity/org.gamecult.geometry/Shaders.meta b/packages/cultmath/unity/org.gamecult.cultmath/Shaders.meta similarity index 76% rename from unity/org.gamecult.geometry/Shaders.meta rename to packages/cultmath/unity/org.gamecult.cultmath/Shaders.meta index 3f517308..1e7e7110 100644 --- a/unity/org.gamecult.geometry/Shaders.meta +++ b/packages/cultmath/unity/org.gamecult.cultmath/Shaders.meta @@ -1,5 +1,5 @@ fileFormatVersion: 2 -guid: 1695895ad6a5467f8136d33a1f160133 +guid: ca5d773061ed49b3a173526048d40cb3 folderAsset: yes DefaultImporter: externalObjects: {} diff --git a/packages/cultmath/unity/org.gamecult.cultmath/Shaders/CultMath.hlsl b/packages/cultmath/unity/org.gamecult.cultmath/Shaders/CultMath.hlsl new file mode 100644 index 00000000..365705f5 --- /dev/null +++ b/packages/cultmath/unity/org.gamecult.cultmath/Shaders/CultMath.hlsl @@ -0,0 +1,356 @@ +#ifndef CULTMATH_HLSL +#define CULTMATH_HLSL + +// CultMath HLSL mirror. +// +// HLSL already owns the float2/float3/float4 type names and most primitive +// intrinsics. This include names the CultMath semantic surface explicitly so +// shader code, C# wrappers, and Rust/native parity fixtures can share the same +// small vocabulary without local helper drift. + +static const float CULTMATH_PI = 3.14159265358979323846; +static const float CULTMATH_TAU = 6.28318530717958647692; +static const float CULTMATH_HALF_PI = 1.57079632679489661923; +static const float CULTMATH_NORMALIZE_EPSILON = 1.0e-20; + +float cultmath_radians(float degrees_value) { return degrees_value * (CULTMATH_PI / 180.0); } +float cultmath_degrees(float radians_value) { return radians_value * (180.0 / CULTMATH_PI); } + +float cultmath_frac(float value) { return value - floor(value); } +float2 cultmath_frac(float2 value) { return value - floor(value); } +float3 cultmath_frac(float3 value) { return value - floor(value); } +float4 cultmath_frac(float4 value) { return value - floor(value); } + +float cultmath_clamp(float value, float minimum, float maximum) { return min(max(value, minimum), maximum); } +float2 cultmath_clamp(float2 value, float2 minimum, float2 maximum) { return min(max(value, minimum), maximum); } +float3 cultmath_clamp(float3 value, float3 minimum, float3 maximum) { return min(max(value, minimum), maximum); } +float4 cultmath_clamp(float4 value, float4 minimum, float4 maximum) { return min(max(value, minimum), maximum); } + +// DXIL Saturate is FMin(1, FMax(0, x)); dxc lowers min/max to the same FMin/FMax, NaN rules included. +float cultmath_saturate(float value) { return min(1.0, max(0.0, value)); } +float2 cultmath_saturate(float2 value) { return min(float2(1.0, 1.0), max(float2(0.0, 0.0), value)); } +float3 cultmath_saturate(float3 value) { return min(float3(1.0, 1.0, 1.0), max(float3(0.0, 0.0, 0.0), value)); } +float4 cultmath_saturate(float4 value) { return min(float4(1.0, 1.0, 1.0, 1.0), max(float4(0.0, 0.0, 0.0, 0.0), value)); } + +float cultmath_lerp(float start, float end, float amount) { return start + (end - start) * amount; } +float2 cultmath_lerp(float2 start, float2 end, float2 amount) { return start + (end - start) * amount; } +float3 cultmath_lerp(float3 start, float3 end, float3 amount) { return start + (end - start) * amount; } +float4 cultmath_lerp(float4 start, float4 end, float4 amount) { return start + (end - start) * amount; } + +float cultmath_step(float edge, float value) { return value < edge ? 0.0 : 1.0; } +float2 cultmath_step(float2 edge, float2 value) { return float2(cultmath_step(edge.x, value.x), cultmath_step(edge.y, value.y)); } +float3 cultmath_step(float3 edge, float3 value) { return float3(cultmath_step(edge.x, value.x), cultmath_step(edge.y, value.y), cultmath_step(edge.z, value.z)); } +float4 cultmath_step(float4 edge, float4 value) { return float4(cultmath_step(edge.x, value.x), cultmath_step(edge.y, value.y), cultmath_step(edge.z, value.z), cultmath_step(edge.w, value.w)); } + +float cultmath_smoothstep(float minimum, float maximum, float value) +{ + float t = cultmath_saturate((value - minimum) / (maximum - minimum)); + return t * t * (3.0 - 2.0 * t); +} + +float2 cultmath_smoothstep(float2 minimum, float2 maximum, float2 value) +{ + float2 t = cultmath_saturate((value - minimum) / (maximum - minimum)); + return t * t * (float2(3.0, 3.0) - float2(2.0, 2.0) * t); +} + +float3 cultmath_smoothstep(float3 minimum, float3 maximum, float3 value) +{ + float3 t = cultmath_saturate((value - minimum) / (maximum - minimum)); + return t * t * (float3(3.0, 3.0, 3.0) - float3(2.0, 2.0, 2.0) * t); +} + +float4 cultmath_smoothstep(float4 minimum, float4 maximum, float4 value) +{ + float4 t = cultmath_saturate((value - minimum) / (maximum - minimum)); + return t * t * (float4(3.0, 3.0, 3.0, 3.0) - float4(2.0, 2.0, 2.0, 2.0) * t); +} + +float cultmath_smoothstep01(float value) { return cultmath_smoothstep(0.0, 1.0, value); } + +float cultmath_smootherstep(float value) +{ + value = cultmath_saturate(value); + return value * value * value * (value * (value * 6.0 - 15.0) + 10.0); +} + +float2 cultmath_snoise_mod289(float2 value) { return value - floor(value * (1.0 / 289.0)) * 289.0; } +float3 cultmath_snoise_mod289(float3 value) { return value - floor(value * (1.0 / 289.0)) * 289.0; } +float4 cultmath_snoise_mod289(float4 value) { return value - floor(value * (1.0 / 289.0)) * 289.0; } +float3 cultmath_snoise_permute(float3 value) { return cultmath_snoise_mod289(((value * 34.0) + 1.0) * value); } +float4 cultmath_snoise_permute(float4 value) { return cultmath_snoise_mod289(((value * 34.0) + 1.0) * value); } + +// Ashima Arts / Ian McEwan 3D simplex noise (MIT), same float32 evaluation +// order as the C# math.snoise(float3) mirror. +float cultmath_snoise(float3 value) +{ + float2 c = float2(1.0 / 6.0, 1.0 / 3.0); + float3 i = floor(value + dot(value, c.yyy)); + float3 x0 = value - i + dot(i, c.xxx); + float3 g = step(x0.yzx, x0.xyz); + float3 l = 1.0 - g; + float3 i1 = min(g.xyz, l.zxy); + float3 i2 = max(g.xyz, l.zxy); + float3 x1 = x0 - i1 + c.xxx; + float3 x2 = x0 - i2 + c.yyy; + float3 x3 = x0 - 0.5; + + i = cultmath_snoise_mod289(i); + float4 p = cultmath_snoise_permute(cultmath_snoise_permute(cultmath_snoise_permute( + i.z + float4(0.0, i1.z, i2.z, 1.0)) + i.y + float4(0.0, i1.y, i2.y, 1.0)) + i.x + float4(0.0, i1.x, i2.x, 1.0)); + + const float n_ = 0.142857142857; + float3 ns = float3(2.0 * n_, 0.5 * n_ - 1.0, n_); + float4 j = p - 49.0 * floor(p * ns.z * ns.z); + float4 x_ = floor(j * ns.z); + float4 y_ = floor(j - 7.0 * x_); + float4 x = x_ * ns.x + ns.yyyy; + float4 y = y_ * ns.x + ns.yyyy; + float4 h = 1.0 - abs(x) - abs(y); + + float4 b0 = float4(x.xy, y.xy); + float4 b1 = float4(x.zw, y.zw); + float4 s0 = floor(b0) * 2.0 + 1.0; + float4 s1 = floor(b1) * 2.0 + 1.0; + float4 sh = -step(h, 0.0); + float4 a0 = b0.xzyw + s0.xzyw * sh.xxyy; + float4 a1 = b1.xzyw + s1.xzyw * sh.zzww; + + float3 p0 = float3(a0.xy, h.x); + float3 p1 = float3(a0.zw, h.y); + float3 p2 = float3(a1.xy, h.z); + float3 p3 = float3(a1.zw, h.w); + float4 norm = 1.79284291400159 - 0.85373472095314 * float4(dot(p0, p0), dot(p1, p1), dot(p2, p2), dot(p3, p3)); + p0 *= norm.x; + p1 *= norm.y; + p2 *= norm.z; + p3 *= norm.w; + + float4 m = max(0.6 - float4(dot(x0, x0), dot(x1, x1), dot(x2, x2), dot(x3, x3)), 0.0); + m = m * m; + return 42.0 * dot(m * m, float4(dot(p0, x0), dot(p1, x1), dot(p2, x2), dot(p3, x3))); +} + +float cultmath_snoise(float2 value) +{ + float4 c = float4( + 0.211324865405187, + 0.366025403784439, + -0.577350269189626, + 0.024390243902439); + float2 i = floor(value + dot(value, c.yy)); + float2 x0 = value - i + dot(i, c.xx); + float2 i1 = x0.x > x0.y ? float2(1.0, 0.0) : float2(0.0, 1.0); + float4 x12 = float4(x0.x + c.x - i1.x, x0.y + c.x - i1.y, x0.x + c.z, x0.y + c.z); + + i = cultmath_snoise_mod289(i); + float3 p = cultmath_snoise_permute( + cultmath_snoise_permute(i.y + float3(0.0, i1.y, 1.0)) + i.x + float3(0.0, i1.x, 1.0)); + float3 m = max(0.5 - float3(dot(x0, x0), dot(x12.xy, x12.xy), dot(x12.zw, x12.zw)), 0.0); + m *= m; + m *= m; + + float3 x = 2.0 * frac(p * c.w) - 1.0; + float3 h = abs(x) - 0.5; + float3 ox = floor(x + 0.5); + float3 a0 = x - ox; + m *= 1.79284291400159 - 0.85373472095314 * (a0 * a0 + h * h); + + float3 g = float3( + a0.x * x0.x + h.x * x0.y, + a0.y * x12.x + h.y * x12.y, + a0.z * x12.z + h.z * x12.w); + return 130.0 * dot(m, g); +} + +float cultmath_lengthsq(float2 value) { return dot(value, value); } +float cultmath_lengthsq(float3 value) { return dot(value, value); } +float cultmath_lengthsq(float4 value) { return dot(value, value); } + +float cultmath_distance(float2 left, float2 right) { return length(left - right); } +float cultmath_distance(float3 left, float3 right) { return length(left - right); } +float cultmath_distance(float4 left, float4 right) { return length(left - right); } + +float2 cultmath_normalize(float2 value) { return value / max(length(value), CULTMATH_NORMALIZE_EPSILON); } +float3 cultmath_normalize(float3 value) { return value / max(length(value), CULTMATH_NORMALIZE_EPSILON); } +float4 cultmath_normalize(float4 value) { return value / max(length(value), CULTMATH_NORMALIZE_EPSILON); } + +float2 cultmath_reflect(float2 incident, float2 normal_value) +{ + return incident - 2.0 * dot(normal_value, incident) * normal_value; +} + +float3 cultmath_reflect(float3 incident, float3 normal_value) +{ + return incident - 2.0 * dot(normal_value, incident) * normal_value; +} + +float2 cultmath_rotate(float2 value, float radians_value) +{ + float s = sin(radians_value); + float c = cos(radians_value); + return float2(value.x * c - value.y * s, value.x * s + value.y * c); +} + +float2 cultmath_rotate_degrees(float2 value, float degrees_value) +{ + return cultmath_rotate(value, cultmath_radians(degrees_value)); +} + +float cultmath_csum(float2 value) { return value.x + value.y; } +float cultmath_csum(float3 value) { return value.x + value.y + value.z; } +float cultmath_csum(float4 value) { return value.x + value.y + value.z + value.w; } + +float cultmath_decay(float source, float lambda_value, float dt) { return source * exp(-lambda_value * dt); } +float2 cultmath_decay(float2 source, float lambda_value, float dt) { return source * exp(-lambda_value * dt); } +float3 cultmath_decay(float3 source, float lambda_value, float dt) { return source * exp(-lambda_value * dt); } +float4 cultmath_decay(float4 source, float lambda_value, float dt) { return source * exp(-lambda_value * dt); } + +float cultmath_damp(float start, float end, float lambda_value, float dt) +{ + return cultmath_lerp(start, end, 1.0 - exp(-lambda_value * dt)); +} + +float2 cultmath_damp(float2 start, float2 end, float lambda_value, float dt) +{ + return cultmath_lerp(start, end, 1.0 - exp(-lambda_value * dt)); +} + +float3 cultmath_damp(float3 start, float3 end, float lambda_value, float dt) +{ + return cultmath_lerp(start, end, 1.0 - exp(-lambda_value * dt)); +} + +float4 cultmath_damp(float4 start, float4 end, float lambda_value, float dt) +{ + return cultmath_lerp(start, end, 1.0 - exp(-lambda_value * dt)); +} + +float cultmath_catmullrom(float p0, float p1, float p2, float p3, float t) +{ + float t2 = t * t; + float t3 = t2 * t; + return 0.5 * ((2.0 * p1) + (p2 - p0) * t + (2.0 * p0 - 5.0 * p1 + 4.0 * p2 - p3) * t2 + (-p0 + 3.0 * p1 - 3.0 * p2 + p3) * t3); +} + +float2 cultmath_catmullrom(float2 p0, float2 p1, float2 p2, float2 p3, float t) +{ + float t2 = t * t; + float t3 = t2 * t; + return 0.5 * ((2.0 * p1) + (p2 - p0) * t + (2.0 * p0 - 5.0 * p1 + 4.0 * p2 - p3) * t2 + (-p0 + 3.0 * p1 - 3.0 * p2 + p3) * t3); +} + +float3 cultmath_catmullrom(float3 p0, float3 p1, float3 p2, float3 p3, float t) +{ + float t2 = t * t; + float t3 = t2 * t; + return 0.5 * ((2.0 * p1) + (p2 - p0) * t + (2.0 * p0 - 5.0 * p1 + 4.0 * p2 - p3) * t2 + (-p0 + 3.0 * p1 - 3.0 * p2 + p3) * t3); +} + +float4 cultmath_catmullrom(float4 p0, float4 p1, float4 p2, float4 p3, float t) +{ + float t2 = t * t; + float t3 = t2 * t; + return 0.5 * ((2.0 * p1) + (p2 - p0) * t + (2.0 * p0 - 5.0 * p1 + 4.0 * p2 - p3) * t2 + (-p0 + 3.0 * p1 - 3.0 * p2 + p3) * t3); +} + +float cultmath_quadratic_bezier(float p0, float p1, float p2, float t) +{ + t = cultmath_saturate(t); + float inv = 1.0 - t; + return inv * inv * p0 + 2.0 * inv * t * p1 + t * t * p2; +} + +float3 cultmath_quadratic_bezier(float3 p0, float3 p1, float3 p2, float t) +{ + t = cultmath_saturate(t); + float inv = 1.0 - t; + return inv * inv * p0 + 2.0 * inv * t * p1 + t * t * p2; +} + +float cultmath_cubic_bezier(float p0, float p1, float p2, float p3, float t) +{ + t = cultmath_saturate(t); + float inv = 1.0 - t; + return inv * inv * inv * p0 + 3.0 * inv * inv * t * p1 + 3.0 * inv * t * t * p2 + t * t * t * p3; +} + +float3 cultmath_cubic_bezier(float3 p0, float3 p1, float3 p2, float3 p3, float t) +{ + t = cultmath_saturate(t); + float inv = 1.0 - t; + return inv * inv * inv * p0 + 3.0 * inv * inv * t * p1 + 3.0 * inv * t * t * p2 + t * t * t * p3; +} + +float cultmath_hash(float value) { return cultmath_frac(sin(value) * 43758.5453); } +float cultmath_hash(float2 value) { return cultmath_hash(dot(value, float2(127.1, 311.7))); } +float cultmath_hash(float3 value) { return cultmath_hash(dot(value, float3(127.1, 311.7, 74.7))); } + +float cultmath_value_noise(float2 position) +{ + float2 cell = floor(position); + float2 local = cultmath_frac(position); + float2 u = local * local * (float2(3.0, 3.0) - float2(2.0, 2.0) * local); + + float a = cultmath_hash(cell); + float b = cultmath_hash(cell + float2(1.0, 0.0)); + float c = cultmath_hash(cell + float2(0.0, 1.0)); + float d = cultmath_hash(cell + float2(1.0, 1.0)); + + return cultmath_lerp(cultmath_lerp(a, b, u.x), cultmath_lerp(c, d, u.x), u.y); +} + +float cultmath_value_noise_bicubic(float2 position) +{ + float2 cell = floor(position); + float2 local = cultmath_frac(position); + + float y0 = cultmath_catmullrom(cultmath_hash(cell + float2(-1.0, -1.0)), cultmath_hash(cell + float2(0.0, -1.0)), cultmath_hash(cell + float2(1.0, -1.0)), cultmath_hash(cell + float2(2.0, -1.0)), local.x); + float y1 = cultmath_catmullrom(cultmath_hash(cell + float2(-1.0, 0.0)), cultmath_hash(cell + float2(0.0, 0.0)), cultmath_hash(cell + float2(1.0, 0.0)), cultmath_hash(cell + float2(2.0, 0.0)), local.x); + float y2 = cultmath_catmullrom(cultmath_hash(cell + float2(-1.0, 1.0)), cultmath_hash(cell + float2(0.0, 1.0)), cultmath_hash(cell + float2(1.0, 1.0)), cultmath_hash(cell + float2(2.0, 1.0)), local.x); + float y3 = cultmath_catmullrom(cultmath_hash(cell + float2(-1.0, 2.0)), cultmath_hash(cell + float2(0.0, 2.0)), cultmath_hash(cell + float2(1.0, 2.0)), cultmath_hash(cell + float2(2.0, 2.0)), local.x); + + return cultmath_catmullrom(y0, y1, y2, y3, local.y); +} + +float cultmath_value_noise_texture(Texture2D noise_texture, SamplerState noise_sampler, float2 uv, float scale) +{ + return noise_texture.SampleLevel(noise_sampler, uv * scale, 0.0).r; +} + +float cultmath_value_noise_texture_bicubic(Texture2D noise_texture, SamplerState noise_sampler, float2 uv, float scale, float2 texel_size) +{ + float2 position = uv * scale; + float2 cell = floor(position); + float2 local = cultmath_frac(position); + float2 base_uv = cell / scale; + + float row0 = cultmath_catmullrom( + noise_texture.SampleLevel(noise_sampler, base_uv + texel_size * float2(-1.0, -1.0), 0.0).r, + noise_texture.SampleLevel(noise_sampler, base_uv + texel_size * float2(0.0, -1.0), 0.0).r, + noise_texture.SampleLevel(noise_sampler, base_uv + texel_size * float2(1.0, -1.0), 0.0).r, + noise_texture.SampleLevel(noise_sampler, base_uv + texel_size * float2(2.0, -1.0), 0.0).r, + local.x); + float row1 = cultmath_catmullrom( + noise_texture.SampleLevel(noise_sampler, base_uv + texel_size * float2(-1.0, 0.0), 0.0).r, + noise_texture.SampleLevel(noise_sampler, base_uv + texel_size * float2(0.0, 0.0), 0.0).r, + noise_texture.SampleLevel(noise_sampler, base_uv + texel_size * float2(1.0, 0.0), 0.0).r, + noise_texture.SampleLevel(noise_sampler, base_uv + texel_size * float2(2.0, 0.0), 0.0).r, + local.x); + float row2 = cultmath_catmullrom( + noise_texture.SampleLevel(noise_sampler, base_uv + texel_size * float2(-1.0, 1.0), 0.0).r, + noise_texture.SampleLevel(noise_sampler, base_uv + texel_size * float2(0.0, 1.0), 0.0).r, + noise_texture.SampleLevel(noise_sampler, base_uv + texel_size * float2(1.0, 1.0), 0.0).r, + noise_texture.SampleLevel(noise_sampler, base_uv + texel_size * float2(2.0, 1.0), 0.0).r, + local.x); + float row3 = cultmath_catmullrom( + noise_texture.SampleLevel(noise_sampler, base_uv + texel_size * float2(-1.0, 2.0), 0.0).r, + noise_texture.SampleLevel(noise_sampler, base_uv + texel_size * float2(0.0, 2.0), 0.0).r, + noise_texture.SampleLevel(noise_sampler, base_uv + texel_size * float2(1.0, 2.0), 0.0).r, + noise_texture.SampleLevel(noise_sampler, base_uv + texel_size * float2(2.0, 2.0), 0.0).r, + local.x); + + return cultmath_catmullrom(row0, row1, row2, row3, local.y); +} + +#endif diff --git a/src/GameCult.Geometry/Shaders/AdvancedErosionFilter.hlsl.meta b/packages/cultmath/unity/org.gamecult.cultmath/Shaders/CultMath.hlsl.meta similarity index 75% rename from src/GameCult.Geometry/Shaders/AdvancedErosionFilter.hlsl.meta rename to packages/cultmath/unity/org.gamecult.cultmath/Shaders/CultMath.hlsl.meta index 015c51c2..d3b6d2b3 100644 --- a/src/GameCult.Geometry/Shaders/AdvancedErosionFilter.hlsl.meta +++ b/packages/cultmath/unity/org.gamecult.cultmath/Shaders/CultMath.hlsl.meta @@ -1,5 +1,5 @@ fileFormatVersion: 2 -guid: 3252ac02e8853ef4b9d1fbc5204b5c6b +guid: a46136cb4acb4ff2bc47b6a4f786e4b3 ShaderIncludeImporter: externalObjects: {} userData: diff --git a/packages/cultmath/unity/org.gamecult.cultmath/THIRD-PARTY-NOTICES.md b/packages/cultmath/unity/org.gamecult.cultmath/THIRD-PARTY-NOTICES.md new file mode 100644 index 00000000..06ebb6c3 --- /dev/null +++ b/packages/cultmath/unity/org.gamecult.cultmath/THIRD-PARTY-NOTICES.md @@ -0,0 +1,11 @@ +# Third-Party Notices + +## 2D and 3D Simplex Noise + +`math.snoise` and `cultmath_snoise` port the textureless 2D and 3D +simplex-noise functions by Ian McEwan and Ashima Arts, as distributed by Stefan +Gustavson's `webgl-noise` project and Unity Mathematics. + +- Copyright 2011 Ashima Arts. +- Upstream: +- License: MIT. diff --git a/packages/cultmath/unity/org.gamecult.cultmath/THIRD-PARTY-NOTICES.md.meta b/packages/cultmath/unity/org.gamecult.cultmath/THIRD-PARTY-NOTICES.md.meta new file mode 100644 index 00000000..a53f08d8 --- /dev/null +++ b/packages/cultmath/unity/org.gamecult.cultmath/THIRD-PARTY-NOTICES.md.meta @@ -0,0 +1,2 @@ +fileFormatVersion: 2 +guid: 0c88669e4d1db604588be2a726b7d231 \ No newline at end of file diff --git a/packages/cultmath/unity/org.gamecult.cultmath/package.json b/packages/cultmath/unity/org.gamecult.cultmath/package.json new file mode 100644 index 00000000..c09bf6bb --- /dev/null +++ b/packages/cultmath/unity/org.gamecult.cultmath/package.json @@ -0,0 +1,17 @@ +{ + "name": "org.gamecult.cultmath", + "displayName": "CultMath", + "version": "0.2.0", + "unity": "2021.3", + "description": "Precompiled CultMath numeric primitives and matching HLSL semantics for GameCult Unity runtimes.", + "license": "MPL-2.0", + "author": { + "name": "GameCult" + }, + "keywords": [ + "GameCult", + "Math", + "HLSL", + "Simulation" + ] +} diff --git a/packages/cultmath/unity/org.gamecult.cultmath/package.json.meta b/packages/cultmath/unity/org.gamecult.cultmath/package.json.meta new file mode 100644 index 00000000..337ba716 --- /dev/null +++ b/packages/cultmath/unity/org.gamecult.cultmath/package.json.meta @@ -0,0 +1,2 @@ +fileFormatVersion: 2 +guid: 33d8d4af49bf0f946ad00a375f26b1d1 \ No newline at end of file diff --git a/packages/cultmath/web.meta b/packages/cultmath/web.meta new file mode 100644 index 00000000..d65ff3e6 --- /dev/null +++ b/packages/cultmath/web.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: daf45471f8b5f4f44a2cb86e77e010db +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/packages/cultmesh-browser/README.md b/packages/cultmesh-browser/README.md new file mode 100644 index 00000000..59a44016 --- /dev/null +++ b/packages/cultmesh-browser/README.md @@ -0,0 +1,61 @@ +# CultMesh Browser + +`cultmesh-browser` is the browser-safe CultMesh client boundary. It owns one +WebSocket session for a stable Verse/authority-runtime target, retains explicit raw +document leases across route replacement, and correlates typed CultNet +operations with provider receipts. + +It does not own application state and it does not expose direct document +writes. A renderer watches provider-owned records and invokes provider-owned +operations. + +```ts +const odin = new CultMeshBrowserOdinRendezvous({ + endpoints: ["wss://odin.example/verse"], + runtimeId: "counter-browser.discovery", +}); + +const mesh = await CultMeshBrowserClient.connect({ + verseId: "sample.counter", + authorityRuntimeId: "sample.counter-daemon", + runtimeId: "counter-browser", + rendezvous: odin, +}); + +const counter = await mesh.leaseRawDocument({ + schemaId: "sample.counter_state.v1", + recordKey: "counter:main", +}); + +counter.watch(record => renderCounter(decodeCultNetPayload(record))); +``` + +`invoke(...)` returns the correlated provider response. Routing, schema, and +envelope failures reject immediately with `CultMeshBrowserOperationError`; +callers can branch on its stable `status` and `code` without parsing prose or +waiting for the request timeout. Domain rejections remain typed provider +responses because the domain still owns their receipt schema. + +`CultMeshBrowserOdinRendezvous` speaks the canonical CultNet Verse-catalog +messages to one or more configured Odin WebSocket endpoints. Application code +supplies the Verse and authority-runtime identity; it does not own endpoint +selection, reconnect, or resubscription. This is deliberately distinct from an +Eve surface's `providerId`, which identifies the UI provider inside the Verse. +The physical socket is a replaceable route, not either stable identity. + +Odin returns exact authority-route bindings. The browser selects only routes +bound to the requested authority and `cultmesh.documents.v1`, then performs a +session-open handshake. It does not report `connected` until the peer proves +the exact Verse, authority runtime, protocol, and route generation. Operation +responses must also name that authority. Legacy catalogs are accepted only +when one authority runtime makes the endpoint binding unambiguous. + +The browser handshake carries authentication through the host application's +ordinary secure cookie or equivalent upgrade credential. The C# host adapter +rejects endpoints without an authorization predicate unless anonymous local +development is explicitly enabled. + +Run `scripts/verify-eve-browser-network.mjs --eve-root ` for the +real Chromium + C# headless chronology. The witness publishes a lower-priority +decoy endpoint owned by another runtime and proves that neither client connects +to it before and after route rotation. diff --git a/packages/cultmesh-browser/package.json b/packages/cultmesh-browser/package.json new file mode 100644 index 00000000..c49062b0 --- /dev/null +++ b/packages/cultmesh-browser/package.json @@ -0,0 +1,50 @@ +{ + "name": "cultmesh-browser", + "version": "0.3.0", + "description": "Browser-safe CultMesh document leases and typed operations over CultNet WebSocket transport.", + "type": "module", + "main": "dist/index.js", + "types": "dist/index.d.ts", + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js", + "default": "./dist/index.js" + } + }, + "files": [ + "dist", + "README.md" + ], + "scripts": { + "build": "node ../../node_modules/typescript/bin/tsc -p tsconfig.json", + "prepack": "node ../../node_modules/typescript/bin/tsc -p tsconfig.json", + "typecheck": "node ../../node_modules/typescript/bin/tsc -p tsconfig.json --pretty false", + "test": "node ../../node_modules/typescript/bin/tsc -p tsconfig.json && node ../../node_modules/typescript/bin/tsc -p tsconfig.test.json && node --test dist-test/test/*.test.js" + }, + "keywords": [ + "cultmesh", + "cultnet", + "browser", + "websocket", + "gamecult" + ], + "author": "metacratic", + "license": "MIT", + "repository": { + "type": "git", + "url": "git+https://github.com/GameCult/CultLib.git", + "directory": "packages/cultmesh-browser" + }, + "engines": { + "node": ">=18" + }, + "dependencies": { + "@msgpack/msgpack": "^3.1.3", + "cultnet-ts": "^0.2.0" + }, + "devDependencies": { + "@types/node": "^24.0.0", + "typescript": "^5.9.0" + } +} diff --git a/packages/cultmesh-browser/src/index.ts b/packages/cultmesh-browser/src/index.ts new file mode 100644 index 00000000..4dddfa7d --- /dev/null +++ b/packages/cultmesh-browser/src/index.ts @@ -0,0 +1,1076 @@ +import { decode, encode } from "@msgpack/msgpack"; +import { + encodeCultNetMessageForWire, + parseCultNetMessage, + type CultNetDatabaseChangeRawMessage, + type CultNetDatabaseSubscribeMessage, + type CultNetDatabaseUnsubscribeMessage, + type CultNetMessage, + type CultNetOperationRequestMessage, + type CultNetOperationResponseMessage, + type CultNetRawDocumentRecord, + type CultNetSnapshotResponseRawMessage, + type CultMeshVerseCatalogRequestMessage, + type CultMeshVerseCatalogResponseMessage, + type CultMeshSessionAcceptedMessage, + type CultMeshSessionOpenMessage, +} from "cultnet-ts/contracts"; + +export interface CultMeshBrowserIdentity { + verseId: string; + authorityRuntimeId: string; +} + +export interface CultMeshBrowserRoute extends CultMeshBrowserIdentity { + endpoint: string; + protocolId?: string; + protocolIds?: readonly string[]; + priority?: number; + generation: string; + certificate?: CultMeshBrowserRouteCertificate; +} + +export interface CultMeshBrowserP256PublicKey { + keyId: string; + x: string; + y: string; +} + +export interface CultMeshBrowserRouteCertificate { + providerKey: CultMeshBrowserP256PublicKey; + odinKeyId: string; + issuedAtUnixMilliseconds: number; + expiresAtUnixMilliseconds: number; + signature: string; +} + +export type CultMeshBrowserAuthorityTrustMode = "authenticated-remote" | "local-development"; + +export interface CultMeshBrowserAuthorityTrustPolicy { + mode: CultMeshBrowserAuthorityTrustMode; + odinRoots?: readonly CultMeshBrowserP256PublicKey[]; + now?: () => number; +} + +export interface CultMeshBrowserRendezvous { + resolve( + identity: CultMeshBrowserIdentity, + trust?: CultMeshBrowserAuthorityTrustPolicy, + ): Promise; +} + +export interface CultMeshBrowserOdinRendezvousOptions { + endpoints: readonly string[]; + runtimeId: string; + transportVersion?: string; + timeoutMs?: number; + maxFrameBytes?: number; + createId?: () => string; + socketFactory?: (endpoint: string) => CultMeshBrowserSocket; +} + +export interface CultMeshBrowserSocket { + binaryType: BinaryType; + readonly readyState: number; + onopen: ((event: Event) => void) | null; + onmessage: ((event: MessageEvent) => void) | null; + onerror: ((event: Event) => void) | null; + onclose: ((event: CloseEvent) => void) | null; + send(data: ArrayBuffer | ArrayBufferView): void; + close(code?: number, reason?: string): void; +} + +/** Resolves a stable Verse/authority-runtime target through Odin's canonical CultNet Verse catalog. */ +export class CultMeshBrowserOdinRendezvous implements CultMeshBrowserRendezvous { + #options: Required> & { + transportVersion?: string; + socketFactory: (endpoint: string) => CultMeshBrowserSocket; + }; + + constructor(options: CultMeshBrowserOdinRendezvousOptions) { + requireText(options.runtimeId, "runtimeId"); + const endpoints = [...new Set(options.endpoints.map(value => value.trim()).filter(Boolean))]; + if (endpoints.length === 0) throw new Error("At least one Odin WebSocket endpoint is required."); + for (const endpoint of endpoints) requireWebSocketEndpoint(endpoint, "Odin rendezvous endpoint"); + this.#options = { + ...options, + endpoints, + timeoutMs: options.timeoutMs ?? 10_000, + maxFrameBytes: options.maxFrameBytes ?? 4 * 1024 * 1024, + createId: options.createId ?? (() => crypto.randomUUID()), + socketFactory: options.socketFactory ?? (endpoint => new WebSocket(endpoint)), + }; + if (this.#options.timeoutMs <= 0) throw new Error("Odin rendezvous timeoutMs must be positive."); + if (this.#options.maxFrameBytes <= 0) throw new Error("Odin rendezvous maxFrameBytes must be positive."); + } + + async resolve( + identity: CultMeshBrowserIdentity, + trust: CultMeshBrowserAuthorityTrustPolicy = { mode: "authenticated-remote", odinRoots: [] }, + ): Promise { + requireText(identity.verseId, "verseId"); + requireText(identity.authorityRuntimeId, "authorityRuntimeId"); + const failures: Error[] = []; + for (const endpoint of this.#options.endpoints) { + try { + return await this.resolveFrom(endpoint, identity, trust); + } catch (error) { + failures.push(error instanceof Error ? error : new Error(String(error))); + } + } + throw new AggregateError( + failures, + `Odin could not resolve Verse '${identity.verseId}' authority runtime '${identity.authorityRuntimeId}'.`, + ); + } + + private resolveFrom( + endpoint: string, + identity: CultMeshBrowserIdentity, + trust: CultMeshBrowserAuthorityTrustPolicy, + ): Promise { + const socket = this.#options.socketFactory(endpoint); + socket.binaryType = "arraybuffer"; + const messageId = this.#options.createId(); + const request: CultMeshVerseCatalogRequestMessage = { + schemaVersion: "cultmesh.verse_catalog_request.v0", + messageId, + verseIds: [identity.verseId], + ...(this.#options.transportVersion ? { transportVersion: this.#options.transportVersion } : {}), + }; + return new Promise((resolve, reject) => { + let settled = false; + const finish = (error?: Error, route?: CultMeshBrowserRoute) => { + if (settled) return; + settled = true; + clearTimeout(timer); + socket.onopen = null; + socket.onmessage = null; + socket.onerror = null; + socket.onclose = null; + socket.close(1000, error ? "Odin resolution failed" : "Odin resolution complete"); + if (error) reject(error); + else resolve(route!); + }; + const timer = setTimeout( + () => finish(new Error(`Odin resolution timed out at '${endpoint}'.`)), + this.#options.timeoutMs, + ); + socket.onopen = () => { + try { + sendSocketMessage(socket, { + schemaVersion: "cultnet.hello.v0", + runtimeId: this.#options.runtimeId, + runtimeKind: "browser", + supportedMessageVersions: [ + "cultmesh.verse_catalog_request.v0", + "cultmesh.verse_catalog_response.v0", + ], + }, this.#options.maxFrameBytes); + sendSocketMessage(socket, request, this.#options.maxFrameBytes); + } catch (error) { + finish(error instanceof Error ? error : new Error(String(error))); + } + }; + socket.onmessage = event => { + void decodeSocketMessage(event, this.#options.maxFrameBytes).then(message => { + if (message.schemaVersion === "cultnet.error.v0") { + finish(new Error(message.error)); + return; + } + if (message.schemaVersion !== "cultmesh.verse_catalog_response.v0" || message.messageId !== messageId) return; + void selectOdinRoute(message, identity, trust) + .then(route => finish(undefined, route)) + .catch(error => finish(error instanceof Error ? error : new Error(String(error)))); + }).catch(error => finish(error instanceof Error ? error : new Error(String(error)))); + }; + socket.onerror = () => finish(new Error(`Odin WebSocket could not open '${endpoint}'.`)); + socket.onclose = () => finish(new Error(`Odin WebSocket closed before resolving '${identity.verseId}'.`)); + }); + } +} + +export interface CultMeshBrowserClientOptions extends CultMeshBrowserIdentity { + runtimeId: string; + rendezvous: CultMeshBrowserRendezvous; + socketFactory?: (endpoint: string) => CultMeshBrowserSocket; + reconnectDelayMs?: number; + connectTimeoutMs?: number; + requestTimeoutMs?: number; + maxFrameBytes?: number; + createId?: () => string; + /** Consumer-owned authority roots. Remote sessions fail closed when omitted. */ + trust?: CultMeshBrowserAuthorityTrustPolicy; +} + +export interface CultMeshRawDocumentLeaseOptions { + schemaId: string; + recordKey: string; + subscriptionId?: string; +} + +export interface CultMeshOperationOptions { + serviceId: string; + operation: string; + payloadSchema: string; + payload: unknown; + idempotencyKey?: string; + /** @deprecated The connected authority is the operation target. A different value is rejected. */ + targetRuntimeId?: string; +} + +/** Portable schema returned when routing or envelope validation fails before a domain reply exists. */ +export const cultNetOperationFailureSchema = "gamecult.cultnet.operation_failure.v1"; + +export interface CultNetOperationFailure { + code: string; + message: string; +} + +/** Correlated framework-level failure returned by a CultMesh operation provider. */ +export class CultMeshBrowserOperationError extends Error { + readonly status: string; + readonly code: string; + readonly diagnostics: readonly string[]; + + constructor(response: CultNetOperationResponseMessage, failure: CultNetOperationFailure) { + super(failure.message || "CultMesh operation failed."); + this.name = "CultMeshBrowserOperationError"; + this.status = response.status || "error"; + this.code = failure.code || "operation-error"; + this.diagnostics = response.diagnostics ?? []; + } +} + +export type CultMeshConnectionState = "connecting" | "connected" | "reconnecting" | "disposed"; + +type DocumentWatcher = (record: CultNetRawDocumentRecord | undefined) => void; + +interface PendingSnapshot { + lease: CultMeshRawDocumentLease; + resolve: () => void; + reject: (error: Error) => void; + timer: ReturnType; +} + +interface PendingOperation { + request: CultNetOperationRequestMessage; + resolve: (response: CultNetOperationResponseMessage) => void; + reject: (error: Error) => void; + timer: ReturnType; +} + +export class CultMeshRawDocumentLease implements Disposable { + readonly schemaId: string; + readonly recordKey: string; + readonly subscriptionId: string; + #client: CultMeshBrowserClient | undefined; + #current: CultNetRawDocumentRecord | undefined; + #watchers = new Set(); + + constructor( + client: CultMeshBrowserClient, + options: Required, + ) { + this.#client = client; + this.schemaId = options.schemaId; + this.recordKey = options.recordKey; + this.subscriptionId = options.subscriptionId; + } + + get current(): CultNetRawDocumentRecord | undefined { + return this.#current; + } + + watch(watcher: DocumentWatcher): () => void { + if (!this.#client) throw new Error("CultMesh document lease is disposed."); + this.#watchers.add(watcher); + watcher(this.#current); + return () => this.#watchers.delete(watcher); + } + + dispose(): void { + this.#client?.releaseLease(this); + this.#client = undefined; + this.#watchers.clear(); + } + + [Symbol.dispose](): void { + this.dispose(); + } + + apply(record: CultNetRawDocumentRecord | undefined): void { + this.#current = record; + for (const watcher of this.#watchers) watcher(record); + } +} + +export class CultMeshBrowserClient implements AsyncDisposable { + readonly identity: CultMeshBrowserIdentity; + readonly runtimeId: string; + #options: Required> & { + socketFactory: (endpoint: string) => CultMeshBrowserSocket; + }; + #socket: CultMeshBrowserSocket | undefined; + #socketGeneration = 0; + #state: CultMeshConnectionState = "connecting"; + #disposed = false; + #reconnectTimer: ReturnType | undefined; + #connectPromise: Promise | undefined; + #leases = new Map(); + #openingLeases = new Map(); + #pendingSnapshots = new Map(); + #pendingOperations = new Map(); + #stateWatchers = new Set<(state: CultMeshConnectionState) => void>(); + + private constructor(options: CultMeshBrowserClientOptions) { + requireText(options.verseId, "verseId"); + requireText(options.authorityRuntimeId, "authorityRuntimeId"); + requireText(options.runtimeId, "runtimeId"); + this.identity = { verseId: options.verseId, authorityRuntimeId: options.authorityRuntimeId }; + this.runtimeId = options.runtimeId; + this.#options = { + ...options, + reconnectDelayMs: options.reconnectDelayMs ?? 250, + connectTimeoutMs: options.connectTimeoutMs ?? 10_000, + requestTimeoutMs: options.requestTimeoutMs ?? 10_000, + maxFrameBytes: options.maxFrameBytes ?? 4 * 1024 * 1024, + createId: options.createId ?? (() => crypto.randomUUID()), + socketFactory: options.socketFactory ?? (endpoint => new WebSocket(endpoint)), + trust: options.trust ?? { mode: "authenticated-remote", odinRoots: [] }, + }; + } + + static async connect(options: CultMeshBrowserClientOptions): Promise { + const client = new CultMeshBrowserClient(options); + await client.connectSocket(); + return client; + } + + get state(): CultMeshConnectionState { + return this.#state; + } + + watchState(watcher: (state: CultMeshConnectionState) => void): () => void { + this.#stateWatchers.add(watcher); + watcher(this.#state); + return () => this.#stateWatchers.delete(watcher); + } + + async leaseRawDocument(options: CultMeshRawDocumentLeaseOptions): Promise { + this.throwIfDisposed(); + requireText(options.schemaId, "schemaId"); + requireText(options.recordKey, "recordKey"); + const subscriptionId = options.subscriptionId ?? `lease:${this.#options.createId()}`; + if (this.#leases.has(subscriptionId) || this.#openingLeases.has(subscriptionId)) { + throw new Error(`CultMesh subscription '${subscriptionId}' is already leased.`); + } + const lease = new CultMeshRawDocumentLease(this, { ...options, subscriptionId }); + this.#openingLeases.set(subscriptionId, lease); + try { + await this.subscribe(lease); + this.#openingLeases.delete(subscriptionId); + this.#leases.set(subscriptionId, lease); + return lease; + } catch (error) { + this.#openingLeases.delete(subscriptionId); + throw error; + } + } + + async invoke(options: CultMeshOperationOptions): Promise { + this.throwIfDisposed(); + requireText(options.serviceId, "serviceId"); + requireText(options.operation, "operation"); + requireText(options.payloadSchema, "payloadSchema"); + if (options.targetRuntimeId && options.targetRuntimeId !== this.identity.authorityRuntimeId) { + throw new Error( + `CultMesh operation target '${options.targetRuntimeId}' does not match connected authority '${this.identity.authorityRuntimeId}'.`, + ); + } + await this.ensureConnected(); + const messageId = options.idempotencyKey ?? this.#options.createId(); + requireText(messageId, "operation idempotencyKey"); + if (this.#pendingOperations.has(messageId)) { + throw new Error(`CultMesh operation '${messageId}' is already in flight.`); + } + const request: CultNetOperationRequestMessage = { + schemaVersion: "cultnet.operation_request.v0", + messageId, + serviceId: options.serviceId, + operation: options.operation, + payloadSchema: options.payloadSchema, + payloadEncoding: "messagepack-base64", + payload: bytesToBase64(encode(options.payload)), + sourceRuntimeId: this.runtimeId, + targetRuntimeId: this.identity.authorityRuntimeId, + }; + const response = new Promise((resolve, reject) => { + const timer = setTimeout(() => { + if (!this.#pendingOperations.delete(messageId)) return; + reject(new Error(`CultMesh operation '${messageId}' timed out.`)); + }, this.#options.requestTimeoutMs); + this.#pendingOperations.set(messageId, { request, resolve, reject, timer }); + }); + try { + this.send(request); + } catch (error) { + this.scheduleReconnect(); + } + return response; + } + + async refreshRoute(): Promise { + this.throwIfDisposed(); + this.#socketGeneration++; + const socket = this.#socket; + this.#socket = undefined; + socket?.close(1012, "CultMesh route refresh"); + await this.connectSocket(); + } + + releaseLease(lease: CultMeshRawDocumentLease): void { + if (this.#leases.get(lease.subscriptionId) !== lease) return; + this.#leases.delete(lease.subscriptionId); + for (const [messageId, pending] of this.#pendingSnapshots) { + if (pending.lease !== lease) continue; + this.#pendingSnapshots.delete(messageId); + clearTimeout(pending.timer); + pending.reject(new Error("CultMesh document lease was disposed before its snapshot arrived.")); + } + if (this.#state === "connected") { + const message: CultNetDatabaseUnsubscribeMessage = { + schemaVersion: "cultnet.database_unsubscribe.v0", + messageId: this.#options.createId(), + subscriptionId: lease.subscriptionId, + }; + this.send(message); + } + } + + async dispose(): Promise { + if (this.#disposed) return; + this.#disposed = true; + this.setState("disposed"); + if (this.#reconnectTimer) clearTimeout(this.#reconnectTimer); + this.#reconnectTimer = undefined; + const error = new Error("CultMesh browser client was disposed."); + for (const pending of this.#pendingSnapshots.values()) { + clearTimeout(pending.timer); + pending.reject(error); + } + for (const pending of this.#pendingOperations.values()) { + clearTimeout(pending.timer); + pending.reject(error); + } + this.#pendingSnapshots.clear(); + this.#pendingOperations.clear(); + this.#leases.clear(); + this.#openingLeases.clear(); + const socket = this.#socket; + this.#socket = undefined; + socket?.close(1000, "CultMesh client disposed"); + } + + async [Symbol.asyncDispose](): Promise { + await this.dispose(); + } + + private async subscribe(lease: CultMeshRawDocumentLease): Promise { + await this.ensureConnected(); + const messageId = this.#options.createId(); + const ready = new Promise((resolve, reject) => { + const timer = setTimeout(() => { + if (!this.#pendingSnapshots.delete(messageId)) return; + reject(new Error(`CultMesh subscription '${lease.subscriptionId}' timed out.`)); + }, this.#options.requestTimeoutMs); + this.#pendingSnapshots.set(messageId, { lease, resolve, reject, timer }); + }); + const request: CultNetDatabaseSubscribeMessage = { + schemaVersion: "cultnet.database_subscribe.v0", + messageId, + subscriptionId: lease.subscriptionId, + schemaIds: [lease.schemaId], + recordKeys: [lease.recordKey], + includeSnapshot: true, + consumerRuntimeId: this.runtimeId, + }; + try { + this.send(request); + } catch (error) { + const pending = this.#pendingSnapshots.get(messageId); + if (pending) clearTimeout(pending.timer); + this.#pendingSnapshots.delete(messageId); + throw error; + } + return ready; + } + + private ensureConnected(): Promise { + if (this.#state === "connected" && this.#socket?.readyState === 1) return Promise.resolve(); + return this.connectSocket(); + } + + private connectSocket(): Promise { + this.throwIfDisposed(); + if (this.#state === "connected" && this.#socket?.readyState === 1) return Promise.resolve(); + if (this.#reconnectTimer) { + clearTimeout(this.#reconnectTimer); + this.#reconnectTimer = undefined; + } + if (this.#connectPromise) return this.#connectPromise; + const generation = ++this.#socketGeneration; + this.setState(this.#socket ? "reconnecting" : "connecting"); + this.#connectPromise = this.openSocket(generation).finally(() => { + this.#connectPromise = undefined; + }); + return this.#connectPromise; + } + + private async openSocket(generation: number): Promise { + const route = await this.#options.rendezvous.resolve(this.identity, this.#options.trust); + if (route.verseId !== this.identity.verseId || + route.authorityRuntimeId !== this.identity.authorityRuntimeId) { + throw new Error("CultMesh rendezvous returned a route for the wrong stable identity."); + } + await verifyAuthorityRoute(route, this.#options.trust); + const endpoint = new URL(route.endpoint); + if (endpoint.protocol !== "ws:" && endpoint.protocol !== "wss:") { + throw new Error(`CultMesh browser route must use ws:// or wss://, got '${route.endpoint}'.`); + } + const socket = this.#options.socketFactory(route.endpoint); + socket.binaryType = "arraybuffer"; + try { + await new Promise((resolve, reject) => { + let opened = false; + let settled = false; + const handshakeId = this.#options.createId(); + const clientNonce = randomNonce(); + const sessionRequest: CultMeshSessionOpenMessage = { + schemaVersion: "cultmesh.session_open.v2", + messageId: handshakeId, + sourceRuntimeId: this.runtimeId, + verseId: this.identity.verseId, + authorityRuntimeId: this.identity.authorityRuntimeId, + protocolId: route.protocolId ?? "cultmesh.documents.v1", + routeGeneration: route.generation, + clientNonce, + }; + const finish = (error?: Error) => { + if (settled) return; + settled = true; + clearTimeout(timer); + if (error) reject(error); + else resolve(); + }; + const timer = setTimeout( + () => finish(new Error( + opened + ? `CultMesh authority handshake timed out for '${route.endpoint}'.` + : `CultMesh WebSocket open timed out for '${route.endpoint}'.`, + )), + this.#options.connectTimeoutMs, + ); + socket.onopen = () => { + opened = true; + try { + sendSocketMessage(socket, { + schemaVersion: "cultnet.hello.v0", + runtimeId: this.runtimeId, + runtimeKind: "browser", + supportedMessageVersions: [ + "cultmesh.session_open.v2", + "cultmesh.session_accepted.v2", + "cultnet.database_subscribe.v0", + "cultnet.database_unsubscribe.v0", + "cultnet.database_change_raw.v0", + "cultnet.operation_request.v0", + "cultnet.operation_response.v0", + ], + }, this.#options.maxFrameBytes); + sendSocketMessage(socket, sessionRequest, this.#options.maxFrameBytes); + } catch (error) { + finish(error instanceof Error ? error : new Error(String(error))); + } + }; + socket.onmessage = event => { + void decodeSocketMessage(event, this.#options.maxFrameBytes).then(message => { + if (message.schemaVersion === "cultnet.error.v0") { + finish(new Error(message.error)); + return; + } + if (message.schemaVersion !== "cultmesh.session_accepted.v2" || message.messageId !== handshakeId) return; + void validateSessionAcceptance( + message, + sessionRequest, + route, + this.#options.trust, + ).then(finish).catch(error => finish(error instanceof Error ? error : new Error(String(error)))); + }).catch(error => finish(error instanceof Error ? error : new Error(String(error)))); + }; + socket.onerror = () => finish(new Error(`CultMesh WebSocket could not open '${route.endpoint}'.`)); + socket.onclose = () => finish(new Error(`CultMesh WebSocket closed before proving authority at '${route.endpoint}'.`)); + }); + } catch (error) { + socket.close(1000, "CultMesh route open failed"); + throw error; + } + if (this.#disposed || generation !== this.#socketGeneration) { + socket.close(1000, "Stale CultMesh route"); + return; + } + this.#socket = socket; + socket.onmessage = event => { void this.receive(event, generation); }; + socket.onerror = () => undefined; + socket.onclose = () => this.handleClosed(generation); + this.setState("connected"); + try { + await Promise.all([...this.#leases.values()].map(lease => this.subscribe(lease))); + for (const pending of this.#pendingOperations.values()) this.send(pending.request); + } catch (error) { + if (this.#socket === socket) this.#socket = undefined; + socket.onclose = null; + socket.close(1000, "CultMesh session initialization failed"); + throw error; + } + } + + private handleClosed(generation: number): void { + if (this.#disposed || generation !== this.#socketGeneration) return; + this.#socket = undefined; + const error = new Error("CultMesh route closed before the provider response arrived."); + for (const pending of this.#pendingSnapshots.values()) { + clearTimeout(pending.timer); + pending.reject(error); + } + this.#pendingSnapshots.clear(); + this.setState("reconnecting"); + this.#reconnectTimer = setTimeout(() => { + this.#reconnectTimer = undefined; + void this.connectSocket().catch(() => this.handleClosed(this.#socketGeneration)); + }, this.#options.reconnectDelayMs); + } + + private async receive(event: MessageEvent, generation: number): Promise { + if (this.#disposed || generation !== this.#socketGeneration) return; + let message: CultNetMessage; + try { + message = await decodeSocketMessage(event, this.#options.maxFrameBytes); + } catch (error) { + const oversized = error instanceof Error && error.message.includes("exceeds"); + this.#socket?.close( + oversized ? 1009 : 1003, + oversized ? "CultNet frame exceeds configured maximum" : "CultNet requires binary WebSocket frames", + ); + return; + } + switch (message.schemaVersion) { + case "cultnet.snapshot_response_raw.v0": + this.applySnapshot(message); + return; + case "cultnet.database_change_raw.v0": + this.applyChange(message); + return; + case "cultnet.operation_response.v0": { + const pending = this.#pendingOperations.get(message.messageId); + if (!pending) return; + if (message.sourceRuntimeId !== this.identity.authorityRuntimeId) { + this.#pendingOperations.delete(message.messageId); + clearTimeout(pending.timer); + pending.reject(new Error( + `CultMesh operation response came from '${message.sourceRuntimeId ?? "unknown"}', expected '${this.identity.authorityRuntimeId}'.`, + )); + this.#socket?.close(1008, "CultMesh authority identity changed"); + return; + } + this.#pendingOperations.delete(message.messageId); + clearTimeout(pending.timer); + if (message.payloadSchema === cultNetOperationFailureSchema) { + try { + pending.reject(new CultMeshBrowserOperationError( + message, + decodeCultNetOperationPayload(message), + )); + } catch (error) { + pending.reject(error instanceof Error + ? error + : new Error("CultMesh operation failure payload was malformed.")); + } + return; + } + pending.resolve(message); + return; + } + case "cultnet.error.v0": + this.rejectCorrelated(message.error); + return; + default: + return; + } + } + + private applySnapshot(message: CultNetSnapshotResponseRawMessage): void { + const pending = this.#pendingSnapshots.get(message.messageId); + if (!pending) return; + this.#pendingSnapshots.delete(message.messageId); + clearTimeout(pending.timer); + const matches = message.documents.filter(record => + record.recordKey === pending.lease.recordKey && recordMatchesSchema(record, pending.lease.schemaId)); + if (matches.length > 1) { + pending.reject(new Error(`CultMesh snapshot returned duplicate record '${pending.lease.recordKey}'.`)); + return; + } + pending.lease.apply(matches[0]); + pending.resolve(); + } + + private applyChange(message: CultNetDatabaseChangeRawMessage): void { + const lease = this.#leases.get(message.subscriptionId) ?? this.#openingLeases.get(message.subscriptionId); + if (!lease) return; + if (message.changeKind === "removed") { + if (message.recordKey === lease.recordKey) lease.apply(undefined); + return; + } + const record = message.document; + if (record?.recordKey === lease.recordKey && recordMatchesSchema(record, lease.schemaId)) lease.apply(record); + } + + private rejectCorrelated(reason: string): void { + if (this.#pendingOperations.size === 1) { + const [messageId, pending] = this.#pendingOperations.entries().next().value!; + this.#pendingOperations.delete(messageId); + clearTimeout(pending.timer); + pending.reject(new Error(reason)); + } + } + + private send(message: CultNetMessage): void { + const socket = this.#socket; + if (!socket || socket.readyState !== 1) throw new Error("CultMesh browser route is not connected."); + const payload = encode(encodeCultNetMessageForWire(message)); + if (payload.byteLength > this.#options.maxFrameBytes) { + throw new Error(`CultMesh message exceeds the ${this.#options.maxFrameBytes}-byte limit.`); + } + socket.send(payload); + } + + private setState(state: CultMeshConnectionState): void { + if (this.#state === state) return; + this.#state = state; + for (const watcher of this.#stateWatchers) watcher(state); + } + + private scheduleReconnect(): void { + if (this.#disposed || this.#reconnectTimer) return; + this.setState("reconnecting"); + this.#reconnectTimer = setTimeout(() => { + this.#reconnectTimer = undefined; + void this.connectSocket().catch(() => this.scheduleReconnect()); + }, this.#options.reconnectDelayMs); + } + + private throwIfDisposed(): void { + if (this.#disposed) throw new Error("CultMesh browser client is disposed."); + } +} + +export function decodeCultNetPayload(record: CultNetRawDocumentRecord): T { + if (record.payloadEncoding !== "messagepack") { + throw new Error(`Unsupported CultNet document payload '${record.payloadEncoding}'.`); + } + return decode(record.payload) as T; +} + +function recordMatchesSchema(record: CultNetRawDocumentRecord, requestedSchema: string): boolean { + return record.schemaId === requestedSchema || + record.schemaName === requestedSchema || + record.schemaVersion === requestedSchema; +} + +export function decodeCultNetOperationPayload(response: CultNetOperationResponseMessage): T { + if (response.payloadEncoding !== "messagepack-base64") { + throw new Error(`Unsupported CultNet operation payload '${response.payloadEncoding ?? "unspecified"}'.`); + } + return decode(base64ToBytes(response.payload)) as T; +} + +function bytesToBase64(bytes: Uint8Array): string { + let binary = ""; + for (let offset = 0; offset < bytes.length; offset += 0x8000) { + binary += String.fromCharCode(...bytes.subarray(offset, offset + 0x8000)); + } + return btoa(binary); +} + +function base64ToBytes(value: string): Uint8Array { + const binary = atob(value); + return Uint8Array.from(binary, character => character.charCodeAt(0)); +} + +function requireWebSocketEndpoint(value: string, field: string): URL { + let endpoint: URL; + try { + endpoint = new URL(value); + } catch { + throw new Error(`${field} '${value}' is not a valid URL.`); + } + if (endpoint.protocol !== "ws:" && endpoint.protocol !== "wss:") { + throw new Error(`${field} must use ws:// or wss://, got '${value}'.`); + } + return endpoint; +} + +function sendSocketMessage(socket: CultMeshBrowserSocket, message: CultNetMessage, maxFrameBytes: number): void { + const payload = encode(encodeCultNetMessageForWire(message)); + if (payload.byteLength > maxFrameBytes) { + throw new Error(`CultNet message exceeds the ${maxFrameBytes}-byte limit.`); + } + socket.send(payload); +} + +async function decodeSocketMessage(event: MessageEvent, maxFrameBytes: number): Promise { + let payload: Uint8Array; + if (event.data instanceof ArrayBuffer) { + payload = new Uint8Array(event.data); + } else if (ArrayBuffer.isView(event.data)) { + payload = new Uint8Array(event.data.buffer, event.data.byteOffset, event.data.byteLength); + } else if (event.data instanceof Blob) { + payload = new Uint8Array(await event.data.arrayBuffer()); + } else { + throw new Error("CultNet requires binary WebSocket frames."); + } + if (payload.byteLength > maxFrameBytes) { + throw new Error(`CultNet frame exceeds the ${maxFrameBytes}-byte limit.`); + } + return parseCultNetMessage(decode(payload)); +} + +async function selectOdinRoute( + response: CultMeshVerseCatalogResponseMessage, + identity: CultMeshBrowserIdentity, + trust: CultMeshBrowserAuthorityTrustPolicy, +): Promise { + const matchingVerses = response.verses.filter(verse => verse.verseId === identity.verseId); + if (matchingVerses.length !== 1) { + throw new Error( + `Odin returned ${matchingVerses.length} routes for Verse '${identity.verseId}' authority runtime '${identity.authorityRuntimeId}'.`, + ); + } + const verse = matchingVerses[0]; + if (verse.authorityRoutes == null && verse.authorityRuntimeIds.length !== 1) { + throw new Error( + `Odin returned ambiguous legacy routes for Verse '${identity.verseId}'. Explicit authority route bindings are required.`, + ); + } + if (verse.authorityRoutes == null && verse.authorityRuntimeIds[0] !== identity.authorityRuntimeId) { + throw new Error( + `Odin advertised no route for authority runtime '${identity.authorityRuntimeId}'.`, + ); + } + const boundRoutes = verse.authorityRoutes ?? verse.discoveryEndpoints.map(endpoint => ({ + authorityRuntimeId: verse.authorityRuntimeIds[0], + endpoint, + protocolIds: ["cultmesh.documents.v1"], + priority: 0, + generation: [verse.compatibility.transportVersion, verse.compatibility.rulesHash, endpoint].join(":"), + certificate: undefined, + })); + const routes = boundRoutes.filter(route => + route.authorityRuntimeId === identity.authorityRuntimeId && + route.protocolIds.includes("cultmesh.documents.v1") && + (() => { + try { + requireWebSocketEndpoint(route.endpoint, "Odin provider route"); + return true; + } catch { + return false; + } + })()).sort((left, right) => left.priority - right.priority || left.endpoint.localeCompare(right.endpoint)); + if (routes.length === 0) { + throw new Error(`Odin advertised no browser-compatible route for authority runtime '${identity.authorityRuntimeId}'.`); + } + const failures: Error[] = []; + for (const route of routes) { + const candidate: CultMeshBrowserRoute = { + ...identity, + endpoint: route.endpoint, + protocolId: "cultmesh.documents.v1", + protocolIds: route.protocolIds, + priority: route.priority, + generation: route.generation, + ...(route.certificate ? { certificate: { + providerKey: { + keyId: route.certificate.providerKeyId, + x: route.certificate.providerPublicKeyX, + y: route.certificate.providerPublicKeyY, + }, + odinKeyId: route.certificate.odinKeyId, + issuedAtUnixMilliseconds: route.certificate.issuedAtUnixMilliseconds, + expiresAtUnixMilliseconds: route.certificate.expiresAtUnixMilliseconds, + signature: route.certificate.signature, + } } : {}), + }; + try { + await verifyAuthorityRoute(candidate, trust); + return candidate; + } catch (error) { + failures.push(error instanceof Error ? error : new Error(String(error))); + } + } + throw new AggregateError( + failures, + `Odin advertised no trusted browser route for authority runtime '${identity.authorityRuntimeId}'.`, + ); +} + +async function validateSessionAcceptance( + message: CultMeshSessionAcceptedMessage, + request: CultMeshSessionOpenMessage, + route: CultMeshBrowserRoute, + trust: CultMeshBrowserAuthorityTrustPolicy, +): Promise { + if (!message.accepted) return new Error(message.error ?? "CultMesh authority rejected the session."); + const protocolId = route.protocolId ?? "cultmesh.documents.v1"; + if (message.verseId !== request.verseId || + message.authorityRuntimeId !== request.authorityRuntimeId || + message.protocolId !== protocolId || + message.routeGeneration !== route.generation || + message.clientNonce !== request.clientNonce) { + return new Error( + `CultMesh route proved '${message.verseId}/${message.authorityRuntimeId}/${message.protocolId}/${message.routeGeneration}', ` + + `expected '${request.verseId}/${request.authorityRuntimeId}/${protocolId}/${route.generation}'.`, + ); + } + if (trust.mode === "local-development" && isLoopbackEndpoint(route.endpoint) && !route.certificate) return undefined; + const providerKey = route.certificate?.providerKey; + if (!providerKey || message.providerKeyId !== providerKey.keyId || !message.providerSignature) { + return new Error("CultMesh authority did not prove possession of the Odin-certified provider key."); + } + if (!await verifyP256(providerKey, canonicalSession(request, route.endpoint), message.providerSignature)) { + return new Error("CultMesh provider session proof is invalid."); + } + return undefined; +} + +async function verifyAuthorityRoute( + route: CultMeshBrowserRoute, + trust: CultMeshBrowserAuthorityTrustPolicy, +): Promise { + const certificate = route.certificate; + if (!certificate) { + if (trust.mode === "local-development" && isLoopbackEndpoint(route.endpoint)) return; + throw new Error("Remote CultMesh routes require an Odin-signed authority certificate."); + } + const endpoint = new URL(route.endpoint); + if (endpoint.protocol !== "wss:" && !(trust.mode === "local-development" && isLoopbackEndpoint(route.endpoint))) { + throw new Error("Authenticated remote CultMesh browser routes require wss:// channel protection."); + } + const now = trust.now?.() ?? Date.now(); + if (now < certificate.issuedAtUnixMilliseconds || now >= certificate.expiresAtUnixMilliseconds) { + throw new Error("The Odin route certificate is not currently valid."); + } + const root = trust.odinRoots?.find(candidate => candidate.keyId === certificate.odinKeyId); + if (!root) throw new Error(`Odin key '${certificate.odinKeyId}' is not trusted by this consumer.`); + if (!await verifyP256(root, canonicalRoute(route), certificate.signature)) { + throw new Error("The Odin route certificate signature is invalid."); + } +} + +function canonicalRoute(route: CultMeshBrowserRoute): Uint8Array { + const certificate = route.certificate!; + return canonicalFields( + "gamecult.cultmesh.route-certificate.v1", + route.verseId, + route.authorityRuntimeId, + route.endpoint, + [...(route.protocolIds ?? [route.protocolId ?? "cultmesh.documents.v1"])].sort().join("\u001f"), + String(route.priority ?? 0), + route.generation, + certificate.providerKey.keyId, + certificate.providerKey.x, + certificate.providerKey.y, + certificate.odinKeyId, + String(certificate.issuedAtUnixMilliseconds), + String(certificate.expiresAtUnixMilliseconds), + ); +} + +function canonicalSession(request: CultMeshSessionOpenMessage, endpoint: string): Uint8Array { + return canonicalFields( + "gamecult.cultmesh.session-proof.v1", + request.clientNonce, + request.messageId, + request.sourceRuntimeId, + request.verseId, + request.authorityRuntimeId, + request.protocolId, + endpoint, + request.routeGeneration, + ); +} + +function canonicalFields(...values: string[]): Uint8Array { + const encoder = new TextEncoder(); + const encoded = values.map(value => encoder.encode(value)); + const total = encoded.reduce((sum, value) => sum + 4 + value.byteLength, 0); + const result = new Uint8Array(total); + const view = new DataView(result.buffer); + let offset = 0; + for (const value of encoded) { + view.setUint32(offset, value.byteLength, false); + offset += 4; + result.set(value, offset); + offset += value.byteLength; + } + return result; +} + +async function verifyP256( + key: CultMeshBrowserP256PublicKey, + payload: Uint8Array, + signatureBase64: string, +): Promise { + try { + const x = base64ToBytes(key.x); + const y = base64ToBytes(key.y); + const signature = base64ToBytes(signatureBase64); + if (x.byteLength !== 32 || y.byteLength !== 32 || signature.byteLength !== 64) return false; + const raw = new Uint8Array(65); + raw[0] = 4; + raw.set(x, 1); + raw.set(y, 33); + const publicKey = await crypto.subtle.importKey( + "raw", + raw.slice().buffer as ArrayBuffer, + { name: "ECDSA", namedCurve: "P-256" }, + false, + ["verify"], + ); + return await crypto.subtle.verify( + { name: "ECDSA", hash: "SHA-256" }, + publicKey, + signature.slice().buffer as ArrayBuffer, + payload.slice().buffer as ArrayBuffer, + ); + } catch { + return false; + } +} + +function randomNonce(): string { + const bytes = crypto.getRandomValues(new Uint8Array(32)); + return bytesToBase64(bytes); +} + +function isLoopbackEndpoint(value: string): boolean { + try { + const host = new URL(value).hostname.replace(/^\[|\]$/g, "").toLowerCase(); + return host === "localhost" || host === "127.0.0.1" || host === "::1"; + } catch { + return false; + } +} + +function requireText(value: string, field: string): void { + if (!value || value.trim().length === 0) throw new Error(`CultMesh browser ${field} is required.`); +} diff --git a/packages/cultmesh-browser/test/cultmesh-browser.test.ts b/packages/cultmesh-browser/test/cultmesh-browser.test.ts new file mode 100644 index 00000000..6c169d93 --- /dev/null +++ b/packages/cultmesh-browser/test/cultmesh-browser.test.ts @@ -0,0 +1,929 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { decode, encode } from "@msgpack/msgpack"; +import { + encodeCultNetMessageForWire, + parseCultNetMessage, + type CultNetDatabaseSubscribeMessage, + type CultNetMessage, + type CultNetOperationRequestMessage, + type CultNetRawDocumentRecord, + type CultMeshVerseCatalogResponseMessage, + type CultMeshSessionOpenMessage, +} from "cultnet-ts/contracts"; + +import { + CultMeshBrowserClient, + CultMeshBrowserOperationError, + CultMeshBrowserOdinRendezvous, + decodeCultNetOperationPayload, + decodeCultNetPayload, + type CultMeshBrowserRoute, + type CultMeshBrowserP256PublicKey, + type CultMeshBrowserSocket, +} from "../src/index.js"; + +const localTrust = { mode: "local-development" as const }; + +test("Odin rendezvous resolves stable authority-runtime identity and observes route rotation", async () => { + const odin = new FakeOdin("ws://127.0.0.1:4050/mesh"); + let id = 0; + const rendezvous = new CultMeshBrowserOdinRendezvous({ + endpoints: ["ws://127.0.0.1:4040/odin"], + runtimeId: "browser-odin-test", + createId: () => `odin-${++id}`, + socketFactory: () => odin.open(), + }); + + assert.equal((await rendezvous.resolve({ + verseId: "sample.counter", + authorityRuntimeId: "sample.counter-daemon", + }, localTrust)).endpoint, "ws://127.0.0.1:4050/mesh"); + + odin.providerEndpoint = "ws://127.0.0.1:4060/mesh"; + assert.equal((await rendezvous.resolve({ + verseId: "sample.counter", + authorityRuntimeId: "sample.counter-daemon", + }, localTrust)).endpoint, "ws://127.0.0.1:4060/mesh"); + + await assert.rejects( + rendezvous.resolve({ verseId: "sample.counter", authorityRuntimeId: "other-runtime" }, localTrust), + /could not resolve/, + ); +}); + +test("Odin rendezvous never selects another authority's faster endpoint", async () => { + const odin = new FakeOdin("ws://127.0.0.1:4050/mesh"); + odin.decoyEndpoint = "ws://127.0.0.1:4049/decoy"; + const rendezvous = new CultMeshBrowserOdinRendezvous({ + endpoints: ["ws://127.0.0.1:4040/odin"], + runtimeId: "browser-route-binding-test", + socketFactory: () => odin.open(), + }); + + const route = await rendezvous.resolve({ + verseId: "sample.counter", + authorityRuntimeId: "sample.counter-daemon", + }, localTrust); + assert.equal(route.endpoint, "ws://127.0.0.1:4050/mesh"); +}); + +test("Odin rendezvous accepts one-authority legacy routes and rejects ambiguous legacy catalogs", async () => { + const odin = new FakeOdin("ws://127.0.0.1:4070/mesh"); + odin.omitAuthorityRoutes = true; + const rendezvous = new CultMeshBrowserOdinRendezvous({ + endpoints: ["ws://127.0.0.1:4040/odin"], + runtimeId: "browser-legacy-route-test", + socketFactory: () => odin.open(), + }); + + const route = await rendezvous.resolve({ + verseId: "sample.counter", + authorityRuntimeId: "sample.counter-daemon", + }, localTrust); + assert.equal(route.endpoint, "ws://127.0.0.1:4070/mesh"); + + odin.decoyEndpoint = "ws://127.0.0.1:4069/decoy"; + await assert.rejects( + rendezvous.resolve({ verseId: "sample.counter", authorityRuntimeId: "sample.counter-daemon" }, localTrust), + (error: unknown) => + error instanceof AggregateError && + error.errors.some(inner => inner instanceof Error && /ambiguous legacy routes/.test(inner.message)), + ); +}); + +test("Odin rendezvous rejects a poisoned route without vetoing a certified fallback", async () => { + const signed = await createSignedRoute("wss://provider.example/mesh"); + const odin = new FakeOdin(signed.route.endpoint); + odin.authorityRoutesOverride = [ + { + authorityRuntimeId: signed.route.authorityRuntimeId, + endpoint: "wss://poison.example/mesh", + protocolIds: ["cultmesh.documents.v1"], + priority: 0, + generation: "poisoned", + }, + routeForCatalog(signed.route), + ]; + const rendezvous = new CultMeshBrowserOdinRendezvous({ + endpoints: ["ws://127.0.0.1:4040/odin"], + runtimeId: "browser-certified-fallback-test", + socketFactory: () => odin.open(), + }); + + const selected = await rendezvous.resolve( + { verseId: signed.route.verseId, authorityRuntimeId: signed.route.authorityRuntimeId }, + { mode: "authenticated-remote", odinRoots: [signed.odinPublic] }, + ); + + assert.equal(selected.endpoint, signed.route.endpoint); +}); + +test("browser client leases provider state, invokes an operation, and follows route replacement", async () => { + const first = new FakeProvider("ws://127.0.0.1:4101/mesh", 4); + const second = new FakeProvider("ws://127.0.0.1:4102/mesh", 9); + let route = first.route; + let id = 0; + const client = await CultMeshBrowserClient.connect({ + verseId: route.verseId, + authorityRuntimeId: route.authorityRuntimeId, + runtimeId: "browser-test", + trust: localTrust, + rendezvous: { resolve: async () => route }, + createId: () => `id-${++id}`, + socketFactory: endpoint => endpoint === first.route.endpoint ? first.open() : second.open(), + }); + + const lease = await client.leaseRawDocument({ + schemaId: "sample.counter_state.v1", + recordKey: "counter:main", + }); + assert.deepEqual(decodeCultNetPayload<{ count: number }>(lease.current!).count, 4); + + const observed: number[] = []; + const unwatch = lease.watch(record => { + if (record) observed.push(decodeCultNetPayload<{ count: number }>(record).count); + }); + const response = await client.invoke({ + serviceId: "sample.counter", + operation: "increment", + payloadSchema: "sample.increment.v1", + payload: { amount: 2 }, + }); + assert.equal(response.status, "accepted"); + assert.deepEqual(decodeCultNetOperationPayload<{ count: number }>(response), { count: 6 }); + assert.deepEqual(observed, [4, 6]); + + route = second.route; + await client.refreshRoute(); + assert.equal(client.state, "connected"); + assert.deepEqual(decodeCultNetPayload<{ count: number }>(lease.current!).count, 9); + assert.deepEqual(observed, [4, 6, 9]); + assert.equal(first.activeSubscriptionCount, 0); + assert.equal(second.activeSubscriptionCount, 1); + + unwatch(); + lease.dispose(); + assert.equal(second.activeSubscriptionCount, 0); + await client.dispose(); +}); + +test("browser client rejects wrong rendezvous identity and has no document write API", async () => { + const provider = new FakeProvider("ws://127.0.0.1:4201/mesh", 1); + await assert.rejects( + CultMeshBrowserClient.connect({ + verseId: "sample.counter", + authorityRuntimeId: "sample.counter-daemon", + runtimeId: "browser-test", + trust: localTrust, + rendezvous: { + resolve: async () => ({ + ...provider.route, + authorityRuntimeId: "intruder-runtime", + }), + }, + socketFactory: () => provider.open(), + }), + /wrong stable identity/, + ); + assert.equal("putDocument" in CultMeshBrowserClient.prototype, false); + assert.equal("setDocument" in CultMeshBrowserClient.prototype, false); +}); + +test("browser client rejects a route whose peer proves the wrong authority", async () => { + const provider = new FakeProvider("ws://127.0.0.1:4251/mesh", 1, false, "intruder-runtime"); + await assert.rejects( + CultMeshBrowserClient.connect({ + verseId: "sample.counter", + authorityRuntimeId: "sample.counter-daemon", + runtimeId: "browser-peer-proof-test", + trust: localTrust, + rendezvous: { resolve: async () => ({ ...provider.route, authorityRuntimeId: "sample.counter-daemon" }) }, + socketFactory: () => provider.open(), + }), + /proved .*intruder-runtime.*expected .*sample.counter-daemon/, + ); +}); + +test("browser client accepts an Odin-certified route only after provider nonce proof", async () => { + const fixture = await createSignedRoute("wss://provider.example/mesh"); + const client = await CultMeshBrowserClient.connect({ + ...fixture.route, + runtimeId: "browser-authenticated-test", + rendezvous: { resolve: async () => fixture.route }, + trust: { mode: "authenticated-remote", odinRoots: [fixture.odinPublic] }, + socketFactory: () => signedHandshakeSocket(fixture.route, fixture.providerPrivate), + }); + + assert.equal(client.state, "connected"); + await client.dispose(); +}); + +test("browser client rejects a credential-free authority impersonator", async () => { + const fixture = await createSignedRoute("wss://provider.example/mesh"); + const impersonator = new FakeProvider("wss://provider.example/mesh", 1); + + await assert.rejects( + CultMeshBrowserClient.connect({ + ...fixture.route, + runtimeId: "browser-impersonation-test", + rendezvous: { resolve: async () => fixture.route }, + trust: { mode: "authenticated-remote", odinRoots: [fixture.odinPublic] }, + socketFactory: () => impersonator.open(), + }), + /did not prove possession/, + ); +}); + +test("browser client rejects a mutated or expired signed route before opening a provider socket", async () => { + const fixture = await createSignedRoute("wss://provider.example/mesh"); + let opened = false; + const connect = (route: CultMeshBrowserRoute, now?: () => number) => CultMeshBrowserClient.connect({ + ...route, + runtimeId: "browser-route-certificate-test", + rendezvous: { resolve: async () => route }, + trust: { mode: "authenticated-remote", odinRoots: [fixture.odinPublic], now }, + socketFactory: () => { + opened = true; + return signedHandshakeSocket(route, fixture.providerPrivate); + }, + }); + + await assert.rejects(connect({ ...fixture.route, endpoint: "wss://evil.example/mesh" }), /signature is invalid/); + assert.equal(opened, false); + await assert.rejects(connect(fixture.route, () => fixture.expiresAt), /not currently valid/); + assert.equal(opened, false); +}); + +test("browser client bounds unanswered provider operations", async () => { + const route = { + verseId: "sample.counter", + authorityRuntimeId: "sample.counter-daemon", + endpoint: "ws://127.0.0.1:4301/mesh", + generation: "route-4301", + }; + const client = await CultMeshBrowserClient.connect({ + ...route, + runtimeId: "browser-timeout-test", + trust: localTrust, + rendezvous: { resolve: async () => route }, + requestTimeoutMs: 5, + socketFactory: () => handshakeOnlySocket(route), + }); + await assert.rejects( + client.invoke({ + serviceId: "sample.counter", + operation: "increment", + payloadSchema: "sample.increment.v1", + payload: { amount: 1 }, + idempotencyKey: "timeout-1", + }), + /timed out/, + ); + await client.dispose(); +}); + +test("browser client rejects a correlated framework failure without waiting for timeout", async () => { + const provider = new FakeProvider("ws://127.0.0.1:4351/mesh", 1, true); + const client = await CultMeshBrowserClient.connect({ + ...provider.route, + runtimeId: "browser-failure-test", + trust: localTrust, + rendezvous: { resolve: async () => provider.route }, + requestTimeoutMs: 60_000, + socketFactory: () => provider.open(), + }); + + await assert.rejects( + client.invoke({ + serviceId: "sample.counter", + operation: "increment", + payloadSchema: "wrong.request.v1", + payload: { amount: 1 }, + idempotencyKey: "failure-1", + }), + (error: unknown) => { + assert.ok(error instanceof CultMeshBrowserOperationError); + assert.equal(error.status, "invalid"); + assert.equal(error.code, "request-schema-mismatch"); + assert.match(error.message, /expected payload schema/i); + return true; + }, + ); + await client.dispose(); +}); + +test("browser client replays one idempotent operation after route loss without duplicating its effect", async () => { + const firstRoute = localRoute("ws://127.0.0.1:4311/mesh", "generation-1"); + const secondRoute = localRoute("ws://127.0.0.1:4312/mesh", "generation-2"); + let route = firstRoute; + let effectCount = 0; + let dropFirstResponse = true; + const results = new Map(); + const open = () => { + let socket: FakeSocket; + socket = new FakeSocket(wire => { + const message = parseCultNetMessage(decode(wire)); + if (message.schemaVersion === "cultmesh.session_open.v2") { + socket.deliver({ + schemaVersion: "cultmesh.session_accepted.v2", + messageId: message.messageId, + accepted: true, + verseId: message.verseId, + authorityRuntimeId: message.authorityRuntimeId, + protocolId: message.protocolId, + routeGeneration: message.routeGeneration, + clientNonce: message.clientNonce, + providerKeyId: "", + providerSignature: "", + }); + return; + } + if (message.schemaVersion !== "cultnet.operation_request.v0") return; + let response = results.get(message.messageId); + if (!response) { + effectCount++; + response = { + schemaVersion: "cultnet.operation_response.v0", + messageId: message.messageId, + serviceId: message.serviceId, + operation: message.operation, + status: "accepted", + payloadSchema: "sample.increment_receipt.v1", + payloadEncoding: "messagepack-base64", + payload: Buffer.from(encode({ count: effectCount })).toString("base64"), + sourceRuntimeId: "sample.counter-daemon", + }; + results.set(message.messageId, response!); + } + if (dropFirstResponse) { + dropFirstResponse = false; + route = secondRoute; + socket.close(1012, "route lost after commit"); + return; + } + socket.deliver(response!); + }, () => undefined); + return socket; + }; + const client = await CultMeshBrowserClient.connect({ + ...firstRoute, + runtimeId: "browser-operation-replay-test", + rendezvous: { resolve: async () => route }, + trust: localTrust, + reconnectDelayMs: 1, + requestTimeoutMs: 2_000, + socketFactory: () => open(), + }); + + const response = await client.invoke({ + serviceId: "sample.counter", + operation: "increment", + payloadSchema: "sample.increment.v1", + payload: { amount: 1 }, + idempotencyKey: "replay-once", + }); + + assert.deepEqual(decodeCultNetOperationPayload<{ count: number }>(response), { count: 1 }); + assert.equal(effectCount, 1); + await client.dispose(); +}); + +test("an early caller reconnect cancels the scheduled reconnect and keeps one live socket", async () => { + const provider = new FakeProvider("ws://127.0.0.1:4361/mesh", 2); + const client = await CultMeshBrowserClient.connect({ + ...provider.route, + runtimeId: "browser-single-reconnect-test", + rendezvous: { resolve: async () => provider.route }, + trust: localTrust, + reconnectDelayMs: 40, + socketFactory: () => provider.open(), + }); + const lease = await client.leaseRawDocument({ + schemaId: "sample.counter_state.v1", + recordKey: "counter:main", + }); + + provider.dropConnections(); + await client.invoke({ + serviceId: "sample.counter", + operation: "increment", + payloadSchema: "sample.increment.v1", + payload: { amount: 1 }, + }); + await new Promise(resolve => setTimeout(resolve, 80)); + + assert.equal(provider.openCount, 2); + assert.equal(provider.liveSocketCount, 1); + assert.equal(provider.activeSubscriptionCount, 1); + lease.dispose(); + await client.dispose(); +}); + +test("browser client rejects an operation response from another runtime", async () => { + const provider = new FakeProvider( + "ws://127.0.0.1:4371/mesh", + 1, + false, + "sample.counter-daemon", + "intruder-runtime", + ); + const client = await CultMeshBrowserClient.connect({ + ...provider.route, + runtimeId: "browser-response-authority-test", + trust: localTrust, + rendezvous: { resolve: async () => provider.route }, + socketFactory: () => provider.open(), + }); + + await assert.rejects( + client.invoke({ + serviceId: "sample.counter", + operation: "increment", + payloadSchema: "sample.increment.v1", + payload: { amount: 1 }, + }), + /response came from 'intruder-runtime'/, + ); + await client.dispose(); +}); + +test("browser client rejects oversized outbound schema messages", async () => { + const route = { + verseId: "sample.counter", + authorityRuntimeId: "sample.counter-daemon", + endpoint: "ws://127.0.0.1:4401/mesh", + generation: "route-4401", + }; + await assert.rejects( + CultMeshBrowserClient.connect({ + ...route, + runtimeId: "browser-size-test", + trust: localTrust, + rendezvous: { resolve: async () => route }, + maxFrameBytes: 8, + socketFactory: () => new FakeSocket(() => undefined, () => undefined), + }), + /exceeds the 8-byte limit/, + ); +}); + +test("browser client reserves subscription identity while the initial lease opens", async () => { + const provider = new FakeProvider("ws://127.0.0.1:4501/mesh", 3); + const client = await CultMeshBrowserClient.connect({ + ...provider.route, + runtimeId: "browser-lease-race-test", + trust: localTrust, + rendezvous: { resolve: async () => provider.route }, + socketFactory: () => provider.open(), + }); + const first = client.leaseRawDocument({ + schemaId: "sample.counter_state.v1", + recordKey: "counter:main", + subscriptionId: "one-owner", + }); + await assert.rejects( + client.leaseRawDocument({ + schemaId: "sample.counter_state.v1", + recordKey: "counter:main", + subscriptionId: "one-owner", + }), + /already leased/, + ); + (await first).dispose(); + await client.dispose(); +}); + +class FakeProvider { + readonly route: CultMeshBrowserRoute; + #count: number; + #subscriptions = new Map>(); + #rejectOperations: boolean; + #runtimeId: string; + #responseRuntimeId: string; + #openCount = 0; + + constructor( + endpoint: string, + count: number, + rejectOperations = false, + runtimeId = "sample.counter-daemon", + responseRuntimeId = runtimeId, + ) { + this.route = { + verseId: "sample.counter", + authorityRuntimeId: "sample.counter-daemon", + endpoint, + generation: endpoint, + }; + this.#count = count; + this.#rejectOperations = rejectOperations; + this.#runtimeId = runtimeId; + this.#responseRuntimeId = responseRuntimeId; + } + + get activeSubscriptionCount(): number { + let count = 0; + for (const subscriptions of this.#subscriptions.values()) count += subscriptions.size; + return count; + } + + get openCount(): number { + return this.#openCount; + } + + get liveSocketCount(): number { + return this.#subscriptions.size; + } + + dropConnections(): void { + for (const socket of [...this.#subscriptions.keys()]) socket.close(1012, "route dropped"); + } + + open(): FakeSocket { + this.#openCount++; + const socket = new FakeSocket( + wire => this.receive(socket, wire), + () => this.#subscriptions.delete(socket), + ); + this.#subscriptions.set(socket, new Map()); + return socket; + } + + private receive(socket: FakeSocket, wire: Uint8Array): void { + const message = parseCultNetMessage(decode(wire)); + switch (message.schemaVersion) { + case "cultmesh.session_open.v2": + this.acceptSession(socket, message); + return; + case "cultnet.database_subscribe.v0": + this.subscribe(socket, message); + return; + case "cultnet.database_unsubscribe.v0": + this.#subscriptions.get(socket)?.delete(message.subscriptionId); + return; + case "cultnet.operation_request.v0": + this.invoke(socket, message); + return; + default: + return; + } + } + + private acceptSession(socket: FakeSocket, message: CultMeshSessionOpenMessage): void { + socket.deliver({ + schemaVersion: "cultmesh.session_accepted.v2", + messageId: message.messageId, + accepted: true, + verseId: message.verseId, + authorityRuntimeId: this.#runtimeId, + protocolId: message.protocolId, + routeGeneration: message.routeGeneration, + clientNonce: message.clientNonce, + providerKeyId: "", + providerSignature: "", + }); + } + + private subscribe(socket: FakeSocket, message: CultNetDatabaseSubscribeMessage): void { + this.#subscriptions.get(socket)!.set(message.subscriptionId, message); + socket.deliver({ + schemaVersion: "cultnet.snapshot_response_raw.v0", + messageId: message.messageId, + documents: [this.record()], + }); + } + + private invoke(socket: FakeSocket, message: CultNetOperationRequestMessage): void { + assert.equal(message.targetRuntimeId, this.#runtimeId); + if (this.#rejectOperations) { + socket.deliver({ + schemaVersion: "cultnet.operation_response.v0", + messageId: message.messageId, + serviceId: message.serviceId, + operation: message.operation, + status: "invalid", + payloadSchema: "gamecult.cultnet.operation_failure.v1", + payloadEncoding: "messagepack-base64", + payload: bytesToBase64(encode({ + code: "request-schema-mismatch", + message: "Expected payload schema 'sample.increment.v1'.", + })), + diagnostics: ["Request schema did not match."], + sourceRuntimeId: this.#responseRuntimeId, + }); + return; + } + const payload = decode(base64ToBytes(message.payload)) as { amount: number }; + this.#count += payload.amount; + for (const [peer, subscriptions] of this.#subscriptions) { + for (const subscriptionId of subscriptions.keys()) { + peer.deliver({ + schemaVersion: "cultnet.database_change_raw.v0", + messageId: `change-${message.messageId}`, + subscriptionId, + changeKind: "updated", + document: this.record(), + }); + } + } + socket.deliver({ + schemaVersion: "cultnet.operation_response.v0", + messageId: message.messageId, + serviceId: message.serviceId, + operation: message.operation, + status: "accepted", + payloadSchema: "sample.increment_receipt.v1", + payloadEncoding: "messagepack-base64", + payload: bytesToBase64(encode({ count: this.#count })), + sourceRuntimeId: this.#responseRuntimeId, + }); + } + + private record(): CultNetRawDocumentRecord { + return { + schemaId: "sha256:sample-counter-state", + schemaName: "sample.counter_state", + schemaVersion: "sample.counter_state.v1", + schemaContentHash: "sha256:sample-counter-state-content", + recordKey: "counter:main", + storedAt: "2026-08-17T00:00:00Z", + payloadEncoding: "messagepack", + payload: encode({ count: this.#count }), + sourceRuntimeId: this.#runtimeId, + }; + } +} + +class FakeOdin { + providerEndpoint: string; + decoyEndpoint: string | undefined; + omitAuthorityRoutes = false; + authorityRoutesOverride: NonNullable | undefined; + + constructor(providerEndpoint: string) { + this.providerEndpoint = providerEndpoint; + } + + open(): FakeSocket { + const socket = new FakeSocket(wire => { + const message = parseCultNetMessage(decode(wire)); + if (message.schemaVersion !== "cultmesh.verse_catalog_request.v0") return; + socket.deliver({ + schemaVersion: "cultmesh.verse_catalog_response.v0", + messageId: message.messageId, + verses: [{ + verseId: "sample.counter", + displayName: "Counter", + authorityModel: "ServerAuthoritative", + compatibility: { + transportVersion: "cultmesh.v1", + rulesHash: "counter-v1", + compatibleVerseIds: [], + requiredPluginIds: [], + optionalPluginIds: [], + }, + discoveryEndpoints: [this.providerEndpoint], + authorityRuntimeIds: this.decoyEndpoint + ? ["sample.counter-daemon", "sample.decoy-daemon"] + : ["sample.counter-daemon"], + ...(!this.omitAuthorityRoutes ? { authorityRoutes: this.authorityRoutesOverride ?? [ + ...(this.decoyEndpoint ? [{ + authorityRuntimeId: "sample.decoy-daemon", + endpoint: this.decoyEndpoint, + protocolIds: ["cultmesh.documents.v1"], + priority: 0, + generation: "decoy-generation", + }] : []), + { + authorityRuntimeId: "sample.counter-daemon", + endpoint: this.providerEndpoint, + protocolIds: ["cultmesh.documents.v1"], + priority: 10, + generation: this.providerEndpoint, + }, + ] } : {}), + }], + }); + }, () => undefined); + return socket; + } +} + +function routeForCatalog(route: CultMeshBrowserRoute): NonNullable< + CultMeshVerseCatalogResponseMessage["verses"][number]["authorityRoutes"] +>[number] { + return { + authorityRuntimeId: route.authorityRuntimeId, + endpoint: route.endpoint, + protocolIds: [...(route.protocolIds ?? [route.protocolId ?? "cultmesh.documents.v1"])], + priority: route.priority ?? 0, + generation: route.generation, + ...(route.certificate ? { + certificate: { + providerKeyId: route.certificate.providerKey.keyId, + providerPublicKeyX: route.certificate.providerKey.x, + providerPublicKeyY: route.certificate.providerKey.y, + odinKeyId: route.certificate.odinKeyId, + issuedAtUnixMilliseconds: route.certificate.issuedAtUnixMilliseconds, + expiresAtUnixMilliseconds: route.certificate.expiresAtUnixMilliseconds, + signature: route.certificate.signature, + }, + } : {}), + }; +} + +function handshakeOnlySocket(route: CultMeshBrowserRoute): FakeSocket { + let socket: FakeSocket; + socket = new FakeSocket(wire => { + const message = parseCultNetMessage(decode(wire)); + if (message.schemaVersion !== "cultmesh.session_open.v2") return; + socket.deliver({ + schemaVersion: "cultmesh.session_accepted.v2", + messageId: message.messageId, + accepted: true, + verseId: route.verseId, + authorityRuntimeId: route.authorityRuntimeId, + protocolId: message.protocolId, + routeGeneration: message.routeGeneration, + clientNonce: message.clientNonce, + providerKeyId: "", + providerSignature: "", + }); + }, () => undefined); + return socket; +} + +function localRoute(endpoint: string, generation: string): CultMeshBrowserRoute { + return { + verseId: "sample.counter", + authorityRuntimeId: "sample.counter-daemon", + endpoint, + protocolId: "cultmesh.documents.v1", + protocolIds: ["cultmesh.documents.v1"], + priority: 0, + generation, + }; +} + +class FakeSocket implements CultMeshBrowserSocket { + binaryType: BinaryType = "arraybuffer"; + readyState = 0; + onopen: ((event: Event) => void) | null = null; + onmessage: ((event: MessageEvent) => void) | null = null; + onerror: ((event: Event) => void) | null = null; + onclose: ((event: CloseEvent) => void) | null = null; + #receive: (wire: Uint8Array) => void; + #closed: () => void; + + constructor(receive: (wire: Uint8Array) => void, closed: () => void) { + this.#receive = receive; + this.#closed = closed; + queueMicrotask(() => { + this.readyState = 1; + this.onopen?.({} as Event); + }); + } + + send(data: ArrayBuffer | ArrayBufferView): void { + if (this.readyState !== 1) throw new Error("Fake socket is not open."); + const wire = data instanceof ArrayBuffer + ? new Uint8Array(data) + : new Uint8Array(data.buffer, data.byteOffset, data.byteLength); + this.#receive(wire); + } + + close(code = 1000, reason = ""): void { + if (this.readyState === 3) return; + this.readyState = 3; + this.#closed(); + this.onclose?.({ code, reason } as CloseEvent); + } + + deliver(message: CultNetMessage): void { + const wire = encode(encodeCultNetMessageForWire(message)); + queueMicrotask(() => this.onmessage?.({ data: wire.slice().buffer } as MessageEvent)); + } +} + +async function createSignedRoute(endpoint: string): Promise<{ + route: CultMeshBrowserRoute; + odinPublic: CultMeshBrowserP256PublicKey; + providerPrivate: CryptoKey; + expiresAt: number; +}> { + const odin = await crypto.subtle.generateKey({ name: "ECDSA", namedCurve: "P-256" }, true, ["sign", "verify"]); + const provider = await crypto.subtle.generateKey({ name: "ECDSA", namedCurve: "P-256" }, true, ["sign", "verify"]); + const odinPublic = await exportPublic("odin-root-1", odin.publicKey); + const providerPublic = await exportPublic("provider-1", provider.publicKey); + const issuedAt = Date.now() - 1_000; + const expiresAt = Date.now() + 60_000; + const unsigned: CultMeshBrowserRoute = { + verseId: "sample.counter", + authorityRuntimeId: "sample.counter-daemon", + endpoint, + protocolId: "cultmesh.documents.v1", + protocolIds: ["cultmesh.documents.v1"], + priority: 0, + generation: "signed-generation-1", + certificate: { + providerKey: providerPublic, + odinKeyId: odinPublic.keyId, + issuedAtUnixMilliseconds: issuedAt, + expiresAtUnixMilliseconds: expiresAt, + signature: "", + }, + }; + const signature = await crypto.subtle.sign( + { name: "ECDSA", hash: "SHA-256" }, + odin.privateKey, + testCanonicalRoute(unsigned).slice().buffer as ArrayBuffer, + ); + return { + route: { ...unsigned, certificate: { ...unsigned.certificate!, signature: toBase64(new Uint8Array(signature)) } }, + odinPublic, + providerPrivate: provider.privateKey, + expiresAt, + }; +} + +function signedHandshakeSocket(route: CultMeshBrowserRoute, providerPrivate: CryptoKey): FakeSocket { + let socket: FakeSocket; + socket = new FakeSocket(wire => { + const message = parseCultNetMessage(decode(wire)); + if (message.schemaVersion !== "cultmesh.session_open.v2") return; + void crypto.subtle.sign( + { name: "ECDSA", hash: "SHA-256" }, + providerPrivate, + testCanonicalSession(message, route.endpoint).slice().buffer as ArrayBuffer, + ).then(signature => socket.deliver({ + schemaVersion: "cultmesh.session_accepted.v2", + messageId: message.messageId, + accepted: true, + verseId: message.verseId, + authorityRuntimeId: message.authorityRuntimeId, + protocolId: message.protocolId, + routeGeneration: message.routeGeneration, + clientNonce: message.clientNonce, + providerKeyId: route.certificate!.providerKey.keyId, + providerSignature: toBase64(new Uint8Array(signature)), + })); + }, () => undefined); + return socket; +} + +async function exportPublic(keyId: string, key: CryptoKey): Promise { + const jwk = await crypto.subtle.exportKey("jwk", key); + return { keyId, x: urlBase64ToBase64(jwk.x!), y: urlBase64ToBase64(jwk.y!) }; +} + +function testCanonicalRoute(route: CultMeshBrowserRoute): Uint8Array { + const certificate = route.certificate!; + return testCanonical( + "gamecult.cultmesh.route-certificate.v1", route.verseId, route.authorityRuntimeId, route.endpoint, + [...route.protocolIds!].sort().join("\u001f"), String(route.priority), route.generation, + certificate.providerKey.keyId, certificate.providerKey.x, certificate.providerKey.y, + certificate.odinKeyId, String(certificate.issuedAtUnixMilliseconds), String(certificate.expiresAtUnixMilliseconds), + ); +} + +function testCanonicalSession(message: CultMeshSessionOpenMessage, endpoint: string): Uint8Array { + return testCanonical( + "gamecult.cultmesh.session-proof.v1", message.clientNonce, message.messageId, message.sourceRuntimeId, + message.verseId, message.authorityRuntimeId, message.protocolId, endpoint, message.routeGeneration, + ); +} + +function testCanonical(...values: string[]): Uint8Array { + const chunks = values.map(value => new TextEncoder().encode(value)); + const bytes = new Uint8Array(chunks.reduce((sum, chunk) => sum + 4 + chunk.length, 0)); + const view = new DataView(bytes.buffer); + let offset = 0; + for (const chunk of chunks) { + view.setUint32(offset, chunk.length, false); + offset += 4; + bytes.set(chunk, offset); + offset += chunk.length; + } + return bytes; +} + +function toBase64(bytes: Uint8Array): string { + return Buffer.from(bytes).toString("base64"); +} + +function urlBase64ToBase64(value: string): string { + return Buffer.from(value, "base64url").toString("base64"); +} + +function bytesToBase64(bytes: Uint8Array): string { + let binary = ""; + for (const byte of bytes) binary += String.fromCharCode(byte); + return btoa(binary); +} + +function base64ToBytes(value: string): Uint8Array { + const binary = atob(value); + return Uint8Array.from(binary, character => character.charCodeAt(0)); +} diff --git a/packages/cultmesh-browser/tsconfig.json b/packages/cultmesh-browser/tsconfig.json new file mode 100644 index 00000000..b9b7caba --- /dev/null +++ b/packages/cultmesh-browser/tsconfig.json @@ -0,0 +1,16 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "ES2022", + "moduleResolution": "Bundler", + "lib": ["ES2022", "DOM"], + "strict": true, + "declaration": true, + "declarationMap": true, + "sourceMap": true, + "outDir": "dist", + "rootDir": "src", + "skipLibCheck": true + }, + "include": ["src/**/*.ts"] +} diff --git a/packages/cultmesh-browser/tsconfig.test.json b/packages/cultmesh-browser/tsconfig.test.json new file mode 100644 index 00000000..4d148683 --- /dev/null +++ b/packages/cultmesh-browser/tsconfig.test.json @@ -0,0 +1,12 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "types": ["node"], + "declaration": false, + "declarationMap": false, + "sourceMap": false, + "outDir": "dist-test", + "rootDir": "." + }, + "include": ["src/**/*.ts", "test/**/*.ts"] +} diff --git a/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/CultMesh.kt b/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/CultMesh.kt index 968e7669..104bd1b9 100644 --- a/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/CultMesh.kt +++ b/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/CultMesh.kt @@ -2587,6 +2587,7 @@ data class CultNetRudpPacket( data class CultNetRudpDeliveredFrame(val channelId: String, val payload: ByteArray, val sequence: Long) data class CultNetRudpReceiveResult( val delivered: List = emptyList(), + val readyToSend: List = emptyList(), val reply: CultNetRudpPacket? = null, val pong: Boolean = false, val pongPayload: ByteArray = ByteArray(0), @@ -2619,10 +2620,16 @@ private data class FragmentBuffer( ) class CultNetRudpSession(options: CultNetRudpSessionOptions) { + companion object { + const val ReliableSendWindowPackets = 32 + private const val ReceivedSequenceWindow = 4_096 + } + val connectionId: Long = uint32(options.connectionId, "connectionId") val resendDelayMs: Long = options.resendDelayMs private val maxPendingReliablePackets: Int? = options.maxPendingReliablePackets private var nextSequence = uint32(options.initialSequence, "initialSequence") + private val nextSequencedByChannel = linkedMapOf() private var nextFragmentId = 1 var connected: Boolean = false private set @@ -2630,13 +2637,19 @@ class CultNetRudpSession(options: CultNetRudpSessionOptions) { private set private var highestReceivedSequence: Long? = null private val receivedSequences = linkedSetOf() + private val latestSequencedByChannel = linkedMapOf() private val pendingReliable = linkedMapOf() + private val queuedReliable = ArrayDeque() private val orderedNextSequenceByChannel = linkedMapOf() private val orderedBuffers = linkedMapOf>() private val fragmentBuffers = linkedMapOf, FragmentBuffer>() val pendingReliableSequences: List get() = pendingReliable.keys.sorted() + val queuedReliablePacketCount: Int + get() = queuedReliable.size + val outstandingReliablePacketCount: Int + get() = pendingReliable.size + queuedReliable.size init { if (maxPendingReliablePackets != null && maxPendingReliablePackets <= 0) { @@ -2663,6 +2676,9 @@ class CultNetRudpSession(options: CultNetRudpSessionOptions) { } fun send(channelId: String, payload: ByteArray, options: CultNetRudpSendOptions = CultNetRudpSendOptions()): CultNetRudpPacket { + if (options.reliable && pendingReliable.size >= ReliableSendWindowPackets) { + throw IOException("RUDP reliable send window is full; receive acknowledgements before sending") + } return sendMany(channelId, payload, options).first() } @@ -2673,13 +2689,14 @@ class CultNetRudpSession(options: CultNetRudpSessionOptions) { maxFragmentBytes: Int? = null, ): List { if (!connected) throw IOException("Cannot send RUDP data before the session is connected") + if (options.ordered && !options.reliable) throw IOException("RUDP ordered delivery requires reliability") if (maxFragmentBytes != null && maxFragmentBytes <= 0) throw IOException("RUDP maxFragmentBytes must be greater than zero") if (maxFragmentBytes != null && payload.size > maxFragmentBytes) { val fragmentCount = (payload.size + maxFragmentBytes - 1) / maxFragmentBytes if (fragmentCount > 0xffff) throw IOException("RUDP payload requires more than 65535 fragments") ensureReliableCapacity(if (options.reliable) fragmentCount else 0) val fragmentId = allocateFragmentId() - return (0 until fragmentCount).map { index -> + val packets = (0 until fragmentCount).map { index -> val start = index * maxFragmentBytes val packet = createPacket( channelId = channelId, @@ -2692,14 +2709,13 @@ class CultNetRudpSession(options: CultNetRudpSessionOptions) { fragmentIndex = index, fragmentCount = fragmentCount, ) - if (packet.reliable) trackReliable(packet, options.nowMs) packet } + return if (options.reliable) admitReliablePackets(packets, options.nowMs) else packets } ensureReliableCapacity(if (options.reliable) 1 else 0) val packet = createPacket(channelId = channelId, packetType = CultNetRudpPacketType.Data, payload = payload, reliable = options.reliable, ordered = options.ordered, sequenced = options.sequenced) - if (packet.reliable) trackReliable(packet, options.nowMs) - return listOf(packet) + return if (packet.reliable) admitReliablePackets(listOf(packet), options.nowMs) else listOf(packet) } fun receive(packet: CultNetRudpPacket, nowMs: Long = 0): CultNetRudpReceiveResult { @@ -2707,42 +2723,85 @@ class CultNetRudpSession(options: CultNetRudpSessionOptions) { val ignoredNow = nowMs requireConnection(packet) applyAcknowledgements(packet) + val readyToSend = promoteQueuedReliable(nowMs) lastReceivedAtMs = nowMs val expectedSequenceIfUninitialized = (highestReceivedSequence ?: (packet.sequence - 1)) + 1 when (packet.packetType) { CultNetRudpPacketType.Accept -> { rememberReceived(packet.sequence) connected = true - return CultNetRudpReceiveResult() + return CultNetRudpReceiveResult(readyToSend = readyToSend) } CultNetRudpPacketType.Ping -> { - rememberReceived(packet.sequence) - return CultNetRudpReceiveResult(reply = createPacket(CultNetRudpPacketType.Pong, "control", packet.payload)) + return CultNetRudpReceiveResult(readyToSend = readyToSend, reply = createPacket(CultNetRudpPacketType.Pong, "control", packet.payload)) } CultNetRudpPacketType.Ack, CultNetRudpPacketType.Pong -> { - rememberReceived(packet.sequence) return CultNetRudpReceiveResult( + readyToSend = readyToSend, pong = packet.packetType == CultNetRudpPacketType.Pong, pongPayload = if (packet.packetType == CultNetRudpPacketType.Pong) packet.payload.copyOf() else ByteArray(0), ) } CultNetRudpPacketType.Disconnect -> { - rememberReceived(packet.sequence) connected = false - return CultNetRudpReceiveResult(disconnected = true, disconnectReason = packet.payload.copyOf()) + return CultNetRudpReceiveResult(readyToSend = readyToSend, disconnected = true, disconnectReason = packet.payload.copyOf()) } CultNetRudpPacketType.Data -> Unit - else -> return CultNetRudpReceiveResult() + else -> return CultNetRudpReceiveResult(readyToSend = readyToSend) } - val duplicate = receivedSequences.contains(packet.sequence) - rememberReceived(packet.sequence) - if (duplicate) return CultNetRudpReceiveResult() - val reassembled = reassemble(packet) ?: return CultNetRudpReceiveResult() - return CultNetRudpReceiveResult(delivered = if (reassembled.ordered) deliverOrdered(reassembled.frame, reassembled.nextSequence, expectedSequenceIfUninitialized) else listOf(reassembled.frame)) + val duplicate = packet.reliable && (receivedSequences.contains(packet.sequence) || ( + highestReceivedSequence != null && + packet.sequence < highestReceivedSequence!! && + highestReceivedSequence!! - packet.sequence >= ReceivedSequenceWindow.toLong() + )) + if (packet.reliable) rememberReceived(packet.sequence) + if (duplicate) return CultNetRudpReceiveResult(readyToSend = readyToSend) + val reassembled = reassemble(packet) ?: return CultNetRudpReceiveResult(readyToSend = readyToSend) + if (!reassembled.ordered && packet.sequenced) { + val newestSequence = reassembled.nextSequence - 1 + val latestSequence = latestSequencedByChannel[reassembled.frame.channelId] + if (latestSequence != null && newestSequence <= latestSequence) { + return CultNetRudpReceiveResult(readyToSend = readyToSend) + } + latestSequencedByChannel[reassembled.frame.channelId] = newestSequence + } + return CultNetRudpReceiveResult( + delivered = if (reassembled.ordered) deliverOrdered(reassembled.frame, reassembled.nextSequence, expectedSequenceIfUninitialized) else listOf(reassembled.frame), + readyToSend = readyToSend, + ) } - fun createAck(): CultNetRudpPacket = createPacket(CultNetRudpPacketType.Ack, "control", ByteArray(0)) + fun createAck(): CultNetRudpPacket { + val (ack, ackMask) = ackState() + return CultNetRudpPacket( + packetType = CultNetRudpPacketType.Ack, + connectionId = connectionId, + sequence = 0, + ack = ack, + ackMask = ackMask, + channelId = "control", + ) + } + + fun createAckFor(sequence: Long): CultNetRudpPacket = CultNetRudpPacket( + packetType = CultNetRudpPacketType.Ack, + connectionId = connectionId, + sequence = 0, + ack = uint32(sequence, "ack sequence"), + ackMask = 0, + channelId = "control", + ) + + fun createAckForReceived(sequence: Long): CultNetRudpPacket { + val receivedSequence = uint32(sequence, "received sequence") + val (ack, _) = ackState() + return if (ack >= receivedSequence && ack - receivedSequence <= 32L) { + createAck() + } else { + createAckFor(receivedSequence) + } + } fun createPing(payload: ByteArray = ByteArray(0)): CultNetRudpPacket = createPacket(CultNetRudpPacketType.Ping, "control", payload) @@ -2777,8 +2836,15 @@ class CultNetRudpSession(options: CultNetRudpSessionOptions) { fragmentIndex: Int = 0, fragmentCount: Int = 0, ): CultNetRudpPacket { - val sequence = nextSequence - nextSequence = uint32(nextSequence + 1, "sequence") + val sequence = when { + reliable -> nextSequence.also { + nextSequence = uint32(nextSequence + 1, "reliable sequence") + } + sequenced -> (nextSequencedByChannel[channelId] ?: 1).also { + nextSequencedByChannel[channelId] = uint32(it + 1, "sequenced channel sequence") + } + else -> 0 + } val (ack, ackMask) = ackState() return CultNetRudpPacket(packetType, connectionId, sequence, ack, ackMask, channelId, reliable, ordered, sequenced, fragmentId, fragmentIndex, fragmentCount, payload.copyOf()) } @@ -2787,9 +2853,28 @@ class CultNetRudpSession(options: CultNetRudpSessionOptions) { pendingReliable[packet.sequence] = PendingReliablePacket(packet.copy(payload = packet.payload.copyOf()), nowMs) } + private fun admitReliablePackets(packets: List, nowMs: Long): List { + val available = maxOf(0, ReliableSendWindowPackets - pendingReliable.size) + val ready = packets.take(available) + ready.forEach { trackReliable(it, nowMs) } + packets.drop(available).forEach { queuedReliable.addLast(it.copy(payload = it.payload.copyOf())) } + return ready + } + + private fun promoteQueuedReliable(nowMs: Long): List { + val available = maxOf(0, ReliableSendWindowPackets - pendingReliable.size) + val ready = mutableListOf() + while (ready.size < available && !queuedReliable.isEmpty()) { + val packet = queuedReliable.removeFirst() + trackReliable(packet, nowMs) + ready.add(packet) + } + return ready + } + private fun ensureReliableCapacity(packetCount: Int) { if (packetCount == 0 || maxPendingReliablePackets == null) return - if (pendingReliable.size + packetCount > maxPendingReliablePackets) { + if (outstandingReliablePacketCount + packetCount > maxPendingReliablePackets) { throw IOException("RUDP reliable send queue is full") } } @@ -2806,6 +2891,10 @@ class CultNetRudpSession(options: CultNetRudpSessionOptions) { private fun rememberReceived(sequence: Long) { receivedSequences.add(sequence) if (highestReceivedSequence == null || sequence > highestReceivedSequence!!) highestReceivedSequence = sequence + if (receivedSequences.size > ReceivedSequenceWindow) { + val keepFrom = maxOf(0L, (highestReceivedSequence ?: sequence) - ReceivedSequenceWindow.toLong() + 1) + receivedSequences.removeIf { it < keepFrom } + } } private fun ackState(): Pair { @@ -3010,6 +3099,7 @@ class CultNetRudpSocketTransportConnection( maxPendingReliablePackets = maxPendingReliablePackets, ) val connected: Boolean get() = session.connected + val outstandingReliablePacketCount: Int get() = session.outstandingReliablePacketCount val localPort: Int get() = socket.localPort override val stats: CultNetTransportStats get() = CultNetTransportStats(bytesReceived, bytesSent, framesReceived, framesSent) @@ -3093,6 +3183,27 @@ class CultNetRudpSocketTransportConnection( override fun receiveSchemaMessage(timeoutMs: Long, pollIntervalMs: Long): CultNetMessage? = receiveSchema(timeoutMs, pollIntervalMs)?.let { parseCultNetMessage(it) } + fun flushReliable(timeoutMs: Long = 30_000, pollIntervalMs: Long = 5) { + val deadline = System.nanoTime() + timeoutMs.coerceAtLeast(0) * 1_000_000L + val preserved = ArrayDeque() + while (!delivered.isEmpty()) preserved.addLast(delivered.removeFirst()) + try { + while (session.outstandingReliablePacketCount > 0) { + if (System.nanoTime() >= deadline) { + throw SocketTimeoutException( + "RUDP reliable flush timed out with ${session.outstandingReliablePacketCount} packets outstanding" + ) + } + receiveOnce()?.let { preserved.addLast(it) } + pollResends() + Thread.sleep(pollIntervalMs) + } + } finally { + while (!delivered.isEmpty()) preserved.addLast(delivered.removeFirst()) + while (!preserved.isEmpty()) delivered.addLast(preserved.removeFirst()) + } + } + fun receiveOnce(): CultNetTransportFrame? { if (!delivered.isEmpty()) return delivered.removeFirst() val buffer = ByteArray(65535) @@ -3116,6 +3227,7 @@ class CultNetRudpSocketTransportConnection( } val result = session.receive(packet, nowMs()) result.reply?.let { sendPacket(it) } + result.readyToSend.forEach { sendPacket(it) } if (result.pong) { pongPayloads.add(result.pongPayload.copyOf()) } @@ -3128,7 +3240,9 @@ class CultNetRudpSocketTransportConnection( framesReceived += 1 } val frame = if (delivered.isEmpty()) null else delivered.removeFirst() - if (packet.packetType == CultNetRudpPacketType.Accept || frame != null) sendPacket(session.createAck()) + if (packet.reliable || packet.packetType == CultNetRudpPacketType.Accept || frame != null) { + sendPacket(session.createAckForReceived(packet.sequence)) + } return frame } @@ -3252,6 +3366,7 @@ fun main(args: Array) { rudpSessionPingsAndDetectsReceiveTimeout() rudpSessionBoundsPendingReliablePacketsBeforeEnqueue() rudpSessionFragmentsAndReassemblesReliableOrderedPayloads() + rudpSessionAdvancesLargeFragmentSetsThroughBoundedReliableWindow() rudpSocketTransportErgonomicFactoriesCarrySchemaFrames() rudpSocketTransportHandshakesAndCarriesReliableOrderedSchemaFrames() rudpSocketTransportCarriesFragmentedReliableOrderedSchemaFrames() @@ -3893,7 +4008,10 @@ private fun startInteropRudpServer( } val result = session.receive(packet, nowMs()) result.reply?.let { sendInteropRudpPacket(socket, remote, it) } - if (packet.packetType == CultNetRudpPacketType.Data) sendInteropRudpPacket(socket, remote, session.createAck()) + result.readyToSend.forEach { sendInteropRudpPacket(socket, remote, it) } + if (packet.reliable || packet.packetType == CultNetRudpPacketType.Data) { + sendInteropRudpPacket(socket, remote, session.createAckForReceived(packet.sequence)) + } for (frame in result.delivered) { if (frame.channelId != "schema") continue val sender = InteropRudpSessionSender(socket, remote, session) @@ -5223,6 +5341,43 @@ private fun rudpSessionFragmentsAndReassemblesReliableOrderedPayloads() { check(delivered.first().sequence == packets.first().sequence) } +private fun rudpSessionAdvancesLargeFragmentSetsThroughBoundedReliableWindow() { + val connectionId = 457L + val sender = CultNetRudpSession(CultNetRudpSessionOptions(connectionId = connectionId, initialSequence = 1)) + val receiver = CultNetRudpSession(CultNetRudpSessionOptions(connectionId = connectionId, initialSequence = 100)) + sender.receive(CultNetRudpPacket(CultNetRudpPacketType.Accept, connectionId, 0, 0, 0, "control")) + receiver.receive(CultNetRudpPacket(CultNetRudpPacketType.Accept, connectionId, 0, 0, 0, "control")) + + val fragmentCount = CultNetRudpSession.ReliableSendWindowPackets + 17 + val payload = ByteArray(fragmentCount * 8) { index -> (index % 251).toByte() } + val wire = ArrayDeque(sender.sendMany( + "schema", + payload, + CultNetRudpSendOptions(reliable = true, ordered = true, nowMs = 1), + maxFragmentBytes = 8, + )) + check(wire.size == CultNetRudpSession.ReliableSendWindowPackets) + val oldestSequence = wire.first().sequence + check(sender.pendingReliableSequences.size == wire.size) + check(sender.queuedReliablePacketCount == 17) + + val delivered = mutableListOf() + while (!wire.isEmpty()) { + val admitted = wire.size + repeat(admitted) { + delivered.addAll(receiver.receive(wire.removeFirst(), 2).delivered) + } + val acknowledged = sender.receive(receiver.createAck(), 3) + acknowledged.readyToSend.forEach { wire.addLast(it) } + } + check(sender.outstandingReliablePacketCount == 0) + check(delivered.size == 1) + check(delivered.first().payload.contentEquals(payload)) + val oldAck = receiver.createAckForReceived(oldestSequence) + check(oldAck.ack == oldestSequence) + check(oldAck.ackMask == 0L) +} + private fun rudpSocketTransportErgonomicFactoriesCarrySchemaFrames() { val connectionId = 0x10203042L CultMesh.createRudpServer( diff --git a/packages/cultmesh-py/README.md b/packages/cultmesh-py/README.md new file mode 100644 index 00000000..63799e52 --- /dev/null +++ b/packages/cultmesh-py/README.md @@ -0,0 +1,252 @@ +# cultmesh-py + +`cultmesh-py` is the Python CultMesh runtime: a cache-backed local node, +reactive and observed documents, the branded `CultMesh` facade, Verse and peer +discovery clients, snapshot and observation fan-out, peer health monitoring, +game sessions with prediction reconciliation, committed simulation facts, the +local TCP/RUDP server, and a package daemon. + +It depends on `cultcache-py` and `cultnet-py`, mirroring `cultmesh-rs` and +`cultmesh-ts`. The Verse catalog, peer exchange, authority lease, stream +negotiation, and simulation fact contracts are defined in +`cultnet_py.cultmesh_contracts` and re-exported here. + +This is the package a Python process embeds to serve a Verse — the Brokkr +Blender add-on serves its editor mirror through `CultMesh.serve_node(...)` +from inside Blender's own interpreter. + +## Example + +```python +from datetime import UTC, datetime, timedelta + +from cultcache_py import define_database_entry_type +from cultnet_py import ( + CultNetClientAuthorityScope, + CultNetFileShardReplicaCursorStore, + CultNetRawClient, + CultNetSchemaShardLogFetcher, + CultNetSchemaShardSnapshotFetcher, + CultNetShardDescriptor, + CultNetShardReplicator, + CultNetShardReplicatorOptions, +) +from cultmesh_py import ( + CultMesh, + CultMeshAuthorityLease, + CultMeshGameSessionOptions, + CultMeshHmacAuthorityLeaseVerifier, + CultMeshNodeOptions, + CultMeshSnapshotFanout, + peer_exchange_request, +) + +note_doc = define_database_entry_type( + "mesh.note", + [("body", 0), ("name", 1), ("kind", 2)], + name="name", + indexes={"kind": "kind"}, +) +node = CultMesh.start_node( + "mesh.cc", + runtime_id="python-runtime", + options=CultMeshNodeOptions(enable_durable_shard_logs=True), +) +node.database.register_document(note_doc) +unsubscribe = node.database.watch_record(note_doc, "note:1", lambda change: print(change.change_kind)) +unsubscribe_named = node.database.watch_by_name(note_doc, "intro", lambda change: print(change.record_key)) +unsubscribe_kind = node.database.watch_by_index(note_doc, "kind", "guide", lambda change: print(change.record_key)) +node.database.put(note_doc, "note:1", {"body": "hello", "name": "intro", "kind": "guide"}) +live_note = node.database.get_required(note_doc, "note:1") +put_message = node.database.put_raw_message( + note_doc, + "note:2", + {"body": "wire me", "name": "wire", "kind": "interop"}, + shard_id="interop", + shard_epoch=1, +) +snapshot_response = node.database.build_snapshot_response(schema_ids=[note_doc.catalog_entry().schema_id]) +snapshot_wire = node.database.create_snapshot_response(schema_ids=[note_doc.catalog_entry().schema_id]) +delete_message = node.database.delete_raw_message(note_doc, "note:2", shard_id="interop", shard_epoch=1) +shard_log_response = node.database.build_shard_log_response(shard_id="interop", shard_epoch=1) +shard_log_wire = node.database.create_shard_log_response(shard_id="interop", shard_epoch=1) + +station_stock_doc = define_database_entry_type( + "aetheria.station_stock", + [("missiles", 0), ("coolant", 1)], + schema_id="gamecult.aetheria.station_stock.v1", +) +station_stock_ui_doc = define_database_entry_type( + "aetheria.station_stock.ui", + [("missiles", 0), ("coolant", 1)], + schema_id="gamecult.aetheria.station_stock.v1", +) +node.database.register_document(station_stock_doc) +node.database.put(station_stock_ui_doc, "station:starbridge:stock", {"missiles": 24, "coolant": 80}) +stock = node.database.get_required(station_stock_ui_doc, "station:starbridge:stock") + +peers = CultMesh.create_peer_catalog() +response = peers.create_response(peer_exchange_request("pex-1", verse_id="local")) +lease_verifier = CultMeshHmacAuthorityLeaseVerifier({"odin": b"shared-lease-key"}) +lease_catalog = CultMesh.create_authority_lease_catalog( + signature_verifier=lease_verifier.verify, + require_verified_signatures=True, +) +lease_valid_from = datetime.now(UTC) +lease = lease_verifier.issue(CultMeshAuthorityLease( + lease_id="lease:python-runtime", + verse_id="local", + peer_id="python-runtime", + roles=("shard-primary",), + valid_from=lease_valid_from, + expires_at=lease_valid_from + timedelta(minutes=5), + shard_ids=("interop",), + issuer_runtime_id="odin", +)) +lease_catalog.upsert(lease) +session = CultMesh.create_game_session( + node, + CultMeshGameSessionOptions( + client_authority_scopes=( + CultNetClientAuthorityScope("python-runtime", schema_ids=(note_doc.catalog_entry().schema_id,), key_prefix="input:python"), + ), + ), +) +server = CultMesh.serve_node( + node, + peer_catalog=peers, + observation_hub=session.observation_hub, + port=3075, + max_snapshot_documents=1000, + max_snapshot_bytes=4 * 1024 * 1024, +) +``` + +The same local server can be launched as a package daemon when another runtime +or operator process needs a CultNet endpoint without embedding Python glue: + +```powershell +python -m cultmesh_py.daemon --runtime-id python-runtime --display-name "Python Runtime" --host 127.0.0.1 --port 3075 --cache-file mesh.cc --enable-durable-shard-logs --seed-interop-note --seed-shard-id interop --verse-id local --role shard-primary --ready-file mesh.ready.json +``` + +The ready file and stdout line are process readiness hints. The served state and +capability truth still live on the framed CultNet/CultMesh endpoint. Use +`--register-interop-note` to expose the package interop-note schema without +seeding a note, or `--seed-interop-note` to register it and publish one local +record for wire probes. Seeded records are committed through the raw mutation +path, so peers can read them through both snapshot and shard-log catch-up. Add +`--verse-id` and one or more `--role` values to publish the launched endpoint in +its own Verse and peer catalogs for discovery clients. When the daemon is +launched with `--cache-file`, `--enable-durable-shard-logs`, and +`--shard-log-file`, a later daemon process can rehydrate and serve the persisted +snapshot and shard log without reseeding. + +`CultMeshPeerHealthMonitor` probes peer-card endpoints with `cultnet.hello.v0` +and preserves the runtime id, display name, document types, message versions, +and mutation contracts reported by the peer. + +```python +client = CultMesh.create_verse_discovery_client("127.0.0.1", 3075) +verses = client.fetch_verses(transport_version="cultmesh.v0") +mesh_peers = client.fetch_peers(verse_id="python-interop", roles=["read-replica"]) +client.sync_peer_catalog(peers, verse_id="python-interop", roles=["read-replica"]) +client.fanout_peer_catalog(peers, verse_id="python-interop", roles=["read-replica"]) +replica_notes = client.sync_documents( + node.database, + peers, + verse_id="python-interop", + roles=["read-replica"], + documents=[(note_doc, "note:remote")], +) +snapshot_fanout = CultMeshSnapshotFanout( + client, + node.database, + peers, + verse_id="python-interop", + roles=["read-replica"], + documents=[(note_doc, "note:remote")], + on_document=lambda note: print(note), +) +snapshot_fanout.sync_once() + +raw_client = CultNetRawClient("127.0.0.1", 3075) +synced_note = node.database.sync_document(raw_client, note_doc, "note:remote") +synced_stock = CultMesh.sync_document( + node, + raw_client, + station_stock_ui_doc, + "station:starbridge:stock", +) +with CultMesh.subscribe_document( + node, + raw_client, + station_stock_ui_doc, + "station:starbridge:stock", +) as stock_subscription: + stock = stock_subscription.sync_initial() + next_change = stock_subscription.read_next_change() +node.database.sync_shard_log(raw_client, shard_id="interop", shard_epoch=1) +replicator = CultNetShardReplicator( + node.database, + CultNetShardReplicatorOptions( + fetcher=CultNetSchemaShardLogFetcher(), + snapshot_fetcher=CultNetSchemaShardSnapshotFetcher(), + cursor_store=CultNetFileShardReplicaCursorStore("mesh.replica-cursors.msgpack"), + poll_interval_seconds=1.0, + on_error=lambda error: print(error), + ), +) +replica_shard = CultNetShardDescriptor( + shard_id="interop", + owner_runtime_id="primary-runtime", + epoch=1, + schema_ids=(note_doc.catalog_entry().schema_id,), + primary_endpoints=("cultnet://127.0.0.1:3075",), +) +replicator.pull_once(replica_shard) +replicator.start([replica_shard]) +replicator.stop() +server.stop() + +streams = CultMesh.create_stream_catalog() + +facts = CultMesh.create_simulation_fact_committer(node) +committed = facts.commit({ + "shardId": "arena", + "shardEpoch": 4, + "frame": 100, + "subjectId": "bob", + "claimKind": "hit", + "claimHash": "accepted-claim-hash", + "witnessCount": 2, + "supportWeight": 2.0, + "totalWeight": 2.0, + "confidence": 1.0, + "hasQuorum": True, +}) +prediction = session.predict(note_doc, "input:python:move", {"body": "predicted input"}) +unsubscribe() +``` + +## Verify + +`cultmesh_py.verify` checks the public exports and `py.typed` markers of all +three Python packages, smokes a local node over the wire, checks the served +capability truth, and runs the benchmark for sanity: + +```powershell +python -m cultmesh_py.verify --json +``` + +## Tests + +```powershell +$env:PYTHONPATH="$PWD\packages\cultcache-py\src;$PWD\packages\cultnet-py\src;$PWD\packages\cultmesh-py\src" +python -m unittest discover -s packages\cultmesh-py\tests +``` + +`packages\cultnet-ts\test\interop\cultnet-interop.test.ts` verifies the public +`CultMeshDiscoveryClient` can fetch typed Verse and peer descriptors from the +live Python peer. See +[docs/python-runtime-parity.md](../../docs/python-runtime-parity.md) for the +evidence map across the three Python packages. diff --git a/packages/cultmesh-py/pyproject.toml b/packages/cultmesh-py/pyproject.toml new file mode 100644 index 00000000..a1df5ee3 --- /dev/null +++ b/packages/cultmesh-py/pyproject.toml @@ -0,0 +1,28 @@ +[build-system] +requires = ["setuptools>=68"] +build-backend = "setuptools.build_meta" + +[project] +name = "cultmesh-py" +version = "0.1.0" +description = "Python CultMesh node, discovery, sessions, and local server for GameCult typed state." +readme = "README.md" +requires-python = ">=3.10" +license = "MIT" +authors = [{ name = "GameCult" }] +classifiers = [ + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", +] +dependencies = ["cultcache-py>=0.2.0", "cultnet-py>=0.1.0"] +[project.scripts] +cultmesh-py-daemon = "cultmesh_py.daemon:main" +cultmesh-py-verify = "cultmesh_py.verify:main" + +[tool.setuptools.packages.find] +where = ["src"] + +[tool.setuptools.package-data] +cultmesh_py = ["py.typed"] diff --git a/packages/cultcache-py/src/cultmesh_py/__init__.py b/packages/cultmesh-py/src/cultmesh_py/__init__.py similarity index 95% rename from packages/cultcache-py/src/cultmesh_py/__init__.py rename to packages/cultmesh-py/src/cultmesh_py/__init__.py index 95aee189..af4da272 100644 --- a/packages/cultcache-py/src/cultmesh_py/__init__.py +++ b/packages/cultmesh-py/src/cultmesh_py/__init__.py @@ -14,8 +14,10 @@ from .node import ( CultMeshDatabase, CultMeshDatabaseChange, CultMeshDocumentPublicationSource, + CultMeshDocumentWriter, CultMeshNode, CultMeshNodeOptions, + CultMeshObservedDocument, CultMeshPublicationDocumentBinding, CultMeshReactiveDocument, CultMeshReactiveDocumentOptions, @@ -37,7 +39,7 @@ from .simulation import ( CultMeshSimulationFactCommitter, simulation_fact_document, ) -from .wire import ( +from cultnet_py.cultmesh_contracts import ( PEER_EXCHANGE_REQUEST, PEER_EXCHANGE_RESPONSE, VERSE_CATALOG_REQUEST, @@ -68,8 +70,10 @@ __all__ = [ "CultMeshDatabase", "CultMeshDatabaseChange", "CultMeshDocumentPublicationSource", + "CultMeshDocumentWriter", "CultMesh", "CultMeshPublicationDocumentBinding", + "CultMeshObservedDocument", "CultMeshReactiveDocument", "CultMeshReactiveDocumentOptions", "CultMeshReactiveDocumentReconciliation", diff --git a/packages/cultcache-py/src/cultmesh_py/client.py b/packages/cultmesh-py/src/cultmesh_py/client.py similarity index 99% rename from packages/cultcache-py/src/cultmesh_py/client.py rename to packages/cultmesh-py/src/cultmesh_py/client.py index e69e0403..2e1c3ab1 100644 --- a/packages/cultcache-py/src/cultmesh_py/client.py +++ b/packages/cultmesh-py/src/cultmesh_py/client.py @@ -9,7 +9,7 @@ from urllib.parse import urlparse from cultcache_py.documents import DocumentDefinition from cultnet_py import CultNetDatabaseSubscription, CultNetRawClient, CultNetRawSnapshotResponse, hello -from .wire import ( +from cultnet_py.cultmesh_contracts import ( PEER_EXCHANGE_RESPONSE, VERSE_CATALOG_RESPONSE, CultMeshPeerCard, diff --git a/packages/cultcache-py/src/cultmesh_py/daemon.py b/packages/cultmesh-py/src/cultmesh_py/daemon.py similarity index 99% rename from packages/cultcache-py/src/cultmesh_py/daemon.py rename to packages/cultmesh-py/src/cultmesh_py/daemon.py index 087dc218..d21d42d8 100644 --- a/packages/cultcache-py/src/cultmesh_py/daemon.py +++ b/packages/cultmesh-py/src/cultmesh_py/daemon.py @@ -13,7 +13,7 @@ from cultnet_py import CultNetSimulationConsensusOptions, CultNetSimulationObser from .facade import CultMesh from .server import CultMeshLocalServer -from .wire import ( +from cultnet_py.cultmesh_contracts import ( CultMeshPeerCard, CultMeshPeerCatalog, CultMeshVerseCatalog, diff --git a/packages/cultcache-py/src/cultmesh_py/facade.py b/packages/cultmesh-py/src/cultmesh_py/facade.py similarity index 97% rename from packages/cultcache-py/src/cultmesh_py/facade.py rename to packages/cultmesh-py/src/cultmesh_py/facade.py index 115259f3..53328aab 100644 --- a/packages/cultcache-py/src/cultmesh_py/facade.py +++ b/packages/cultmesh-py/src/cultmesh_py/facade.py @@ -24,17 +24,17 @@ from cultnet_py import ( from .client import CultMeshDocumentSubscription, CultMeshPeerExchangeClient, CultMeshVerseDiscoveryClient from .node import ( CultMeshDocumentPublicationSource, + CultMeshDocumentWriter, CultMeshNode, CultMeshNodeOptions, + CultMeshObservedDocument, CultMeshPublicationDocumentBinding, - CultMeshReactiveDocument, - CultMeshReactiveDocumentOptions, create_node, ) from .server import CultMeshLocalServer from .session import CultMeshGameSession, CultMeshGameSessionOptions from .simulation import CultMeshSimulationFactCommitter -from .wire import ( +from cultnet_py.cultmesh_contracts import ( AuthorityLeaseVerifier, CultMeshAuthorityLeaseCatalog, CultMeshPeerCard, @@ -301,13 +301,20 @@ class CultMesh: ) @staticmethod - def reactive_document( + def authoritative_writer( node: CultMeshNode, document: DocumentDefinition[Any], key: str, - options: CultMeshReactiveDocumentOptions | None = None, - ) -> CultMeshReactiveDocument: - return node.reactive_document(document, key, options) + ) -> CultMeshDocumentWriter: + return node.authoritative_writer(document, key) + + @staticmethod + def observe_document( + node: CultMeshNode, + document: DocumentDefinition[Any], + key: str, + ) -> CultMeshObservedDocument: + return node.observe_document(document, key) @staticmethod def publication_source_from_peer_snapshot( diff --git a/packages/cultcache-py/src/cultmesh_py/node.py b/packages/cultmesh-py/src/cultmesh_py/node.py similarity index 87% rename from packages/cultcache-py/src/cultmesh_py/node.py rename to packages/cultmesh-py/src/cultmesh_py/node.py index 142fe226..09ac2012 100644 --- a/packages/cultcache-py/src/cultmesh_py/node.py +++ b/packages/cultmesh-py/src/cultmesh_py/node.py @@ -1,6 +1,9 @@ from __future__ import annotations import threading +import heapq +import time +import traceback from dataclasses import dataclass, field from datetime import UTC, datetime from pathlib import Path @@ -84,10 +87,90 @@ class CultMeshReactiveDocumentReconciliation: @dataclass(frozen=True) class CultMeshReactiveDocumentOptions: flush_delay_seconds: float = 0.016 - detect_local_changes: bool = True replace_dirty_current_on_canonical_snapshot: bool = False +class _CultMeshScheduledCall: + def __init__(self, callback: Callable[[], None], wake: Callable[[], None]) -> None: + self._callback: Callable[[], None] | None = callback + self._wake = wake + self._lock = threading.Lock() + + def cancel(self) -> None: + with self._lock: + self._callback = None + self._wake() + + def take(self) -> Callable[[], None] | None: + with self._lock: + callback = self._callback + self._callback = None + return callback + + @property + def cancelled(self) -> bool: + with self._lock: + return self._callback is None + + +class _CultMeshReactiveScheduler: + """One lazy daemon worker owns delayed reactive flushes for the process.""" + + def __init__(self) -> None: + self._condition = threading.Condition() + self._queue: list[tuple[float, int, _CultMeshScheduledCall]] = [] + self._sequence = 0 + self._worker: threading.Thread | None = None + + def schedule(self, delay_seconds: float, callback: Callable[[], None]) -> _CultMeshScheduledCall: + call = _CultMeshScheduledCall(callback, self._wake) + with self._condition: + self._sequence += 1 + heapq.heappush( + self._queue, + (time.monotonic() + max(0.0, delay_seconds), self._sequence, call), + ) + if self._worker is None: + self._worker = threading.Thread( + target=self._run, + name="cultmesh-reactive-scheduler", + daemon=True, + ) + self._worker.start() + self._condition.notify() + return call + + def _wake(self) -> None: + with self._condition: + self._condition.notify() + + def _run(self) -> None: + while True: + callback: Callable[[], None] | None = None + with self._condition: + while callback is None: + while self._queue and self._queue[0][2].cancelled: + heapq.heappop(self._queue) + if not self._queue: + self._condition.wait() + continue + deadline, _, call = self._queue[0] + remaining = deadline - time.monotonic() + if remaining > 0: + self._condition.wait(remaining) + continue + heapq.heappop(self._queue) + callback = call.take() + if callback is not None: + try: + callback() + except Exception: + traceback.print_exc() + + +_REACTIVE_SCHEDULER = _CultMeshReactiveScheduler() + + @dataclass class CultMeshNode: cache: CultCache = field(default_factory=CultCache) @@ -248,13 +331,19 @@ class CultMeshNode: shard_epoch=shard_epoch, ) - def reactive_document( + def authoritative_writer( self, document: DocumentDefinition[Any], key: str, - options: CultMeshReactiveDocumentOptions | None = None, - ) -> "CultMeshReactiveDocument": - return self.database.reactive_document(document, key, options) + ) -> "CultMeshDocumentWriter": + return self.database.authoritative_writer(document, key) + + def observe_document( + self, + document: DocumentDefinition[Any], + key: str, + ) -> "CultMeshObservedDocument": + return self.database.observe_document(document, key) def sync_document_from_publication( self, @@ -744,17 +833,27 @@ class CultMeshDatabase: ) return self.get_required(requested, key) - def reactive_document( + def authoritative_writer( self, document: DocumentDefinition[Any], key: str, - options: CultMeshReactiveDocumentOptions | None = None, - ) -> "CultMeshReactiveDocument": + ) -> "CultMeshDocumentWriter": if not key: raise ValueError("key must be non-empty") if not any(self._documents_alias(document, registered) for registered in self.documents): self.register_document(document) - return CultMeshReactiveDocument(self, document, key, options) + return CultMeshDocumentWriter(self, document, key) + + def observe_document( + self, + document: DocumentDefinition[Any], + key: str, + ) -> "CultMeshObservedDocument": + if not key: + raise ValueError("key must be non-empty") + if not any(self._documents_alias(document, registered) for registered in self.documents): + self.register_document(document) + return CultMeshObservedDocument(self, document, key) def sync_document_from_publication( self, @@ -1254,32 +1353,119 @@ def _document_matches_schema_id(document: DocumentDefinition[Any], schema_id: st return _infer_schema_name(schema_id) == entry.schema_name -class CultMeshReactiveDocument: +class CultMeshDocumentWriter: def __init__( self, database: CultMeshDatabase, document: DocumentDefinition[Any], key: str, + ) -> None: + self.database = database + self.document = document + self.key = key + + def write(self, value: Any) -> None: + self.database.put(self.document, self.key, self._clone(value)) + + def update(self, callback: Callable[[Any], Any | None]) -> Any: + draft = self._clone(self.database.get_required(self.document, self.key)) + replacement = callback(draft) + next_value = draft if replacement is None else replacement + self.write(next_value) + return self._clone(next_value) + + def reactive( + self, options: CultMeshReactiveDocumentOptions | None = None, + ) -> "CultMeshReactiveDocument": + return CultMeshReactiveDocument(self, options) + + def _clone(self, value: Any) -> Any: + return self.document.decode_payload(self.document.encode_payload(value)) + + +class CultMeshObservedDocument: + def __init__( + self, + database: CultMeshDatabase, + document: DocumentDefinition[Any], + key: str, ) -> None: self._database = database self._document = document self._key = key + self._lock = threading.RLock() + self._disposed = False + self._current = self._clone(database.get_required(document, key)) + self._unsubscribe = database.watch_record(document, key, self._apply_canonical_change) + + @property + def current(self) -> Any: + with self._lock: + self._raise_if_disposed() + return self._current + + def refresh(self) -> Any: + with self._lock: + self._raise_if_disposed() + self._current = self._clone( + self._database.get_required(self._document, self._key) + ) + return self._current + + def dispose(self) -> None: + with self._lock: + if self._disposed: + return + self._disposed = True + self._unsubscribe() + + def __enter__(self) -> "CultMeshObservedDocument": + return self + + def __exit__(self, _exc_type: Any, _exc: Any, _tb: Any) -> None: + self.dispose() + + def _apply_canonical_change(self, change: CultMeshDatabaseChange) -> None: + if change.value is None: + return + with self._lock: + if not self._disposed: + self._current = self._clone(change.value) + + def _clone(self, value: Any) -> Any: + return self._document.decode_payload(self._document.encode_payload(value)) + + def _raise_if_disposed(self) -> None: + if self._disposed: + raise RuntimeError("CultMeshObservedDocument has been disposed") + + +class CultMeshReactiveDocument: + def __init__( + self, + writer: CultMeshDocumentWriter, + options: CultMeshReactiveDocumentOptions | None = None, + ) -> None: + self.writer = writer + self._database = writer.database + self._document = writer.document + self._key = writer.key self._options = options or CultMeshReactiveDocumentOptions() self._lock = threading.RLock() self._disposed = False self._dirty = False self._flushing = False self._flush_queued = False - self._flush_timer: threading.Timer | None = None - self._detect_timer: threading.Timer | None = None + self._flush_timer: _CultMeshScheduledCall | None = None self._reconciliation_version = 0 self.reconciliation: CultMeshReactiveDocumentReconciliation | None = None - self.current = self._clone(self._database.get_required(document, key)) - self._last_clean_payload = self._serialize(self.current) - self._unsubscribe = self._database.watch_record(document, key, self._apply_canonical_change) - if self._options.detect_local_changes: - self._schedule_detection() + self._current = self._clone(self._database.get_required(self._document, self._key)) + self._unsubscribe = self._database.watch_record( + self._document, + self._key, + self._apply_canonical_change, + ) @property def document(self) -> DocumentDefinition[Any]: @@ -1294,42 +1480,43 @@ class CultMeshReactiveDocument: with self._lock: return self._dirty - def update(self, callback: Callable[[Any], None]) -> Any: + @property + def snapshot(self) -> Any: with self._lock: self._raise_if_disposed() - callback(self.current) - self._dirty = True - self._schedule_flush_locked() - return self.current + return self._clone(self._current) - def set_current(self, value: Any) -> Any: + def update(self, callback: Callable[[Any], Any | None]) -> Any: with self._lock: self._raise_if_disposed() - self.current = self._clone(value) + draft = self._clone(self._current) + replacement = callback(draft) + self._current = self._clone(draft if replacement is None else replacement) self._dirty = True self._schedule_flush_locked() - return self.current + return self._clone(self._current) - def mark_dirty(self) -> None: + def replace_local(self, value: Any) -> Any: with self._lock: self._raise_if_disposed() + self._current = self._clone(value) self._dirty = True self._schedule_flush_locked() + return self._clone(self._current) def refresh(self) -> Any: with self._lock: self._raise_if_disposed() - self.current = self._clone(self._database.get_required(self._document, self._key)) - self._last_clean_payload = self._serialize(self.current) + self._current = self._clone( + self._database.get_required(self._document, self._key) + ) self._dirty = False self.reconciliation = None - return self.current + return self._clone(self._current) def flush(self) -> None: with self._lock: self._raise_if_disposed() - if not self._dirty: - self._detect_local_changes_locked() if not self._dirty: return if self._flushing: @@ -1340,16 +1527,14 @@ class CultMeshReactiveDocument: self._flush_timer = None self._flushing = True self._dirty = False - predicted = self._clone(self.current) - self._last_clean_payload = self._serialize(predicted) + predicted = self._clone(self._current) try: - self._database.put(self._document, self._key, predicted) + self.writer.write(predicted) finally: should_flush_again = False with self._lock: self._flushing = False - self._detect_local_changes_locked() should_flush_again = self._flush_queued or self._dirty self._flush_queued = False if should_flush_again: @@ -1367,8 +1552,6 @@ class CultMeshReactiveDocument: self._disposed = True if self._flush_timer is not None: self._flush_timer.cancel() - if self._detect_timer is not None: - self._detect_timer.cancel() self._unsubscribe() def __enter__(self) -> "CultMeshReactiveDocument": @@ -1383,11 +1566,13 @@ class CultMeshReactiveDocument: with self._lock: if self._disposed: return - canonical = self._clone(change.value) + replacement = None if self._dirty or self._flushing: - predicted = self._clone(self.current) - delta = self._create_delta(predicted, canonical) + delta = self._create_delta(self._current, change.value) if delta: + canonical = self._clone(change.value) + predicted = self._clone(self._current) + replacement = canonical self._reconciliation_version += 1 self.reconciliation = CultMeshReactiveDocumentReconciliation( canonical=canonical, @@ -1400,45 +1585,20 @@ class CultMeshReactiveDocument: self.reconciliation = None if not self._options.replace_dirty_current_on_canonical_snapshot: return - self.current = canonical - self._last_clean_payload = self._serialize(self.current) + self._current = ( + replacement + if replacement is not None + else self._clone(change.value) + ) self.reconciliation = None - def _schedule_detection(self) -> None: - delay = max(self._options.flush_delay_seconds, 0.001) - timer = threading.Timer(delay, self._detect_timer_elapsed) - timer.daemon = True - self._detect_timer = timer - timer.start() - - def _detect_timer_elapsed(self) -> None: - with self._lock: - if self._disposed: - return - if not self._dirty and not self._flushing: - self._detect_local_changes_locked() - if self._dirty: - self._schedule_flush_locked() - if not self._disposed and self._options.detect_local_changes: - self._schedule_detection() - - def _detect_local_changes_locked(self) -> None: - if self._serialize(self.current) != self._last_clean_payload: - self._dirty = True - def _schedule_flush_locked(self) -> None: - if self._options.flush_delay_seconds <= 0: - threading.Thread(target=self.flush, daemon=True).start() - return if self._flush_timer is not None: self._flush_timer.cancel() - timer = threading.Timer(self._options.flush_delay_seconds, self.flush) - timer.daemon = True - self._flush_timer = timer - timer.start() - - def _serialize(self, value: Any) -> bytes: - return self._document.encode_payload(value) + self._flush_timer = _REACTIVE_SCHEDULER.schedule( + self._options.flush_delay_seconds, + self.flush, + ) def _clone(self, value: Any) -> Any: return self._document.decode_payload(self._document.encode_payload(value)) diff --git a/packages/cultcache-py/src/cultmesh_py/py.typed b/packages/cultmesh-py/src/cultmesh_py/py.typed similarity index 100% rename from packages/cultcache-py/src/cultmesh_py/py.typed rename to packages/cultmesh-py/src/cultmesh_py/py.typed diff --git a/packages/cultcache-py/src/cultmesh_py/server.py b/packages/cultmesh-py/src/cultmesh_py/server.py similarity index 98% rename from packages/cultcache-py/src/cultmesh_py/server.py rename to packages/cultmesh-py/src/cultmesh_py/server.py index d560cc83..e0c7fdd0 100644 --- a/packages/cultcache-py/src/cultmesh_py/server.py +++ b/packages/cultmesh-py/src/cultmesh_py/server.py @@ -24,7 +24,7 @@ from cultnet_py import ( ) from .node import CultMeshNode -from .wire import ( +from cultnet_py.cultmesh_contracts import ( PEER_EXCHANGE_REQUEST, VERSE_CATALOG_REQUEST, CultMeshPeerCatalog, @@ -245,6 +245,8 @@ class CultMeshLocalServer: result = peer.session.receive(packet, now_ms) if result.reply is not None: self._send_rudp_packet(rudp_socket, remote_addr, result.reply) + for ready in result.ready_to_send: + self._send_rudp_packet(rudp_socket, remote_addr, ready) if result.disconnected: peers.pop(remote_addr, None) continue @@ -265,8 +267,12 @@ class CultMeshLocalServer: peer.session, msgpack.packb(response, use_bin_type=True), ) - if packet.packet_type == CultNetRudpPacketType.DATA or result.delivered: - self._send_rudp_packet(rudp_socket, remote_addr, peer.session.create_ack()) + if packet.reliable or packet.packet_type == CultNetRudpPacketType.DATA or result.delivered: + self._send_rudp_packet( + rudp_socket, + remote_addr, + peer.session.create_ack_for_received(packet.sequence), + ) def _poll_rudp_resends( self, diff --git a/packages/cultcache-py/src/cultmesh_py/session.py b/packages/cultmesh-py/src/cultmesh_py/session.py similarity index 98% rename from packages/cultcache-py/src/cultmesh_py/session.py rename to packages/cultmesh-py/src/cultmesh_py/session.py index 7e4b1bc1..c02696f0 100644 --- a/packages/cultcache-py/src/cultmesh_py/session.py +++ b/packages/cultmesh-py/src/cultmesh_py/session.py @@ -23,7 +23,7 @@ from .simulation import ( CultMeshSimulationFactCommitter, simulation_fact_document, ) -from .wire import CultMeshAuthorityLeaseCatalog, CultMeshPeerCatalog, CultMeshVerseCatalog +from cultnet_py.cultmesh_contracts import CultMeshAuthorityLeaseCatalog, CultMeshPeerCatalog, CultMeshVerseCatalog if TYPE_CHECKING: from .server import CultMeshLocalServer diff --git a/packages/cultmesh-py/src/cultmesh_py/simulation.py b/packages/cultmesh-py/src/cultmesh_py/simulation.py new file mode 100644 index 00000000..a89ba4a2 --- /dev/null +++ b/packages/cultmesh-py/src/cultmesh_py/simulation.py @@ -0,0 +1,47 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + +from cultcache_py.documents import DocumentDefinition +from cultnet_py.cultmesh_contracts import ( + SIMULATION_FACT_DOCUMENT_TYPE, + SIMULATION_FACT_SCHEMA_VERSION, + CultMeshSimulationFact, + simulation_fact_document, +) + +from .node import CultMeshNode + +__all__ = [ + "SIMULATION_FACT_DOCUMENT_TYPE", + "SIMULATION_FACT_SCHEMA_VERSION", + "CultMeshSimulationFact", + "CultMeshSimulationFactCommit", + "CultMeshSimulationFactCommitter", + "simulation_fact_document", +] + + +@dataclass(frozen=True) +class CultMeshSimulationFactCommit: + key: str + fact: CultMeshSimulationFact + + +@dataclass +class CultMeshSimulationFactCommitter: + node: CultMeshNode + document: DocumentDefinition[CultMeshSimulationFact] = simulation_fact_document + + def __post_init__(self) -> None: + if all(document.type != self.document.type for document in self.node.documents): + self.node.register_document(self.document) + + def commit(self, candidate: dict[str, Any], *, committed_at: str | None = None) -> CultMeshSimulationFactCommit: + if not bool(candidate.get("hasQuorum") if "hasQuorum" in candidate else candidate.get("has_quorum")): + raise ValueError("Simulation candidate cannot be committed before quorum") + key = CultMeshSimulationFact.create_record_key(candidate) + fact = CultMeshSimulationFact.from_candidate(candidate, committed_at=committed_at) + self.node.put(self.document, key, fact) + return CultMeshSimulationFactCommit(key=key, fact=fact) diff --git a/packages/cultcache-py/src/cultcache_py/verify.py b/packages/cultmesh-py/src/cultmesh_py/verify.py similarity index 98% rename from packages/cultcache-py/src/cultcache_py/verify.py rename to packages/cultmesh-py/src/cultmesh_py/verify.py index b6a85430..068f5dc2 100644 --- a/packages/cultcache-py/src/cultcache_py/verify.py +++ b/packages/cultmesh-py/src/cultmesh_py/verify.py @@ -6,7 +6,7 @@ import json from importlib import resources from typing import Any -from .benchmark import run_benchmark +from cultnet_py.benchmark import run_benchmark EXPECTED_EXPORTS: dict[str, tuple[str, ...]] = { @@ -150,7 +150,7 @@ EXPECTED_EXPORTS: dict[str, tuple[str, ...]] = { def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser(prog="cultcache-py-verify") + parser = argparse.ArgumentParser(prog="cultmesh-py-verify") parser.add_argument("--records", type=int, default=64) parser.add_argument("--json", action="store_true", dest="emit_json") args = parser.parse_args(argv) @@ -159,7 +159,7 @@ def main(argv: list[str] | None = None) -> int: if args.emit_json: print(json.dumps(result, indent=2, sort_keys=True)) else: - print(f"cultcache-py verify: {result['status']}") + print(f"cultmesh-py verify: {result['status']}") for check in result["checks"]: print(f"- {check['name']}: {check['status']}") return 0 if result["status"] == "ok" else 1 diff --git a/packages/cultmesh-py/tests/test_cultmesh.py b/packages/cultmesh-py/tests/test_cultmesh.py new file mode 100644 index 00000000..b6a21bb8 --- /dev/null +++ b/packages/cultmesh-py/tests/test_cultmesh.py @@ -0,0 +1,4491 @@ +from __future__ import annotations + +import json +import os +import socket +import subprocess +import sys +import tempfile +import threading +import time +import unittest +from unittest.mock import patch +from dataclasses import dataclass +from datetime import UTC, datetime, timedelta +from pathlib import Path +from typing import Any, Callable +from cultcache_py import ( + CultCache, + SingleFileMessagePackBackingStore, + define_database_entry_type, + define_document_type, +) +from cultcache_py.interop import INTEROP_SCHEMA_VERSION, interop_note_document +from cultmesh_py.verify import verify +from cultnet_py import ( + compute_simulation_claim_hash, + CultNetClientAuthorityScope, + CultNetRawClient, + CultNetRawDocumentRecord, + CultNetRawSnapshotResponse, + CultNetSchemaCatalog, + CultNetShardCatalog, + CultNetShardDescriptor, + CultNetShardLogResponse, + CultNetFileShardReplicaCursorStore, + CultNetFileShardMutationLogStore, + CultNetInMemoryShardReplicaCursorStore, + CultNetSchemaWriteForwarder, + CultNetSchemaShardLogFetcher, + CultNetSchemaShardSnapshotFetcher, + CultNetPeerError, + CultNetShardReplicator, + CultNetShardReplicatorOptions, + CultNetShardMutationLogStore, + CultNetShardWriteForwarder, + CultNetSimulationConsensusOptions, + CultNetSimulationConsensusCandidate, + CultNetSimulationObservation, + CultNetSimulationObservationHub, + CultNetRudpSocketMode, + CultNetRudpSocketTransportConnection, + CultNetRudpSocketTransportOptions, + apply_shard_log_response, + document_delete, + document_put_raw, + hello, + simulation_observation, + snapshot_request, +) +from cultmesh_py import create_node +from cultmesh_py import ( + CultMesh, + CultMeshDatabase, + CultMeshDatabaseChange, + CultMeshGameSessionOptions, + CultMeshNodeOptions, + CultMeshReactiveDocumentOptions, + CultMeshPeerCard, + CultMeshPeerCatalog, + CultMeshSimulationFact, + CultMeshAuthorityLease, + CultMeshAuthorityLeaseCatalog, + CultMeshHmacAuthorityLeaseVerifier, + CultMeshDiscoveryClient, + CultMeshPeerExchangeClient, + CultMeshPeerHealthMonitor, + CultMeshSimulationObservationFanout, + CultMeshSnapshotFanout, + CultMeshStreamCatalog, + CultMeshStreamConsumerProfile, + CultMeshStreamDescriptor, + CultMeshStreamFrameHandle, + CultMeshVerseCatalog, + CultMeshVerseCompatibility, + CultMeshVerseDescriptor, + CultMeshVerseDiscoveryClient, + READY_SCHEMA_VERSION, + peer_exchange_request, + simulation_fact_document, + verse_catalog_request, +) + + +def bind_udp_socket() -> socket.socket: + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + sock.bind(("127.0.0.1", 0)) + sock.settimeout(0.02) + return sock + + +def pump_rudp_handshake( + client: CultNetRudpSocketTransportConnection, + server: CultNetRudpSocketTransportConnection, +) -> None: + for _ in range(20): + server.receive_once() + client.receive_once() + server.receive_once() + if client.connected and server.connected: + return + time.sleep(0.005) + raise AssertionError("RUDP socket handshake did not complete") + + +def receive_rudp_frame(transport: CultNetRudpSocketTransportConnection): + for _ in range(20): + frame = transport.receive_once() + if frame is not None: + return frame + time.sleep(0.005) + raise AssertionError("RUDP socket frame was not delivered") + + +@dataclass +class Item: + name: str + category: str + value: int + + +class CultMeshTests(unittest.TestCase): + def _wait_until(self, predicate: Callable[[], bool], *, timeout_seconds: float = 2.0) -> None: + deadline = time.monotonic() + timeout_seconds + while time.monotonic() < deadline: + if predicate(): + return + time.sleep(0.01) + self.fail("Timed out waiting for condition") + + def test_verify_reports_python_runtime_surface_health(self) -> None: + result = verify(records=4) + + self.assertEqual(result["status"], "ok") + self.assertEqual( + {check["name"] for check in result["checks"]}, + { + "public_exports", + "typed_markers", + "local_cultmesh_wire_smoke", + "cultmesh_capability_truth", + "benchmark_sanity", + }, + ) + by_name = {check["name"]: check for check in result["checks"]} + self.assertEqual(by_name["public_exports"]["missing"], {}) + self.assertEqual(by_name["cultmesh_capability_truth"]["failures"], []) + + def test_cultmesh_client_uses_rudp_schema_transport_endpoint(self) -> None: + import msgpack # type: ignore + + server_socket = bind_udp_socket() + server_host, server_port = server_socket.getsockname()[:2] + connection_id = 0x43554C54 + server = CultNetRudpSocketTransportConnection( + CultNetRudpSocketTransportOptions( + runtime_id="python-rudp-schema-server", + socket=server_socket, + mode=CultNetRudpSocketMode.SERVER, + connection_id=connection_id, + initial_sequence=100, + resend_delay_ms=25, + ) + ) + received_messages: list[dict[str, object]] = [] + + def serve_once() -> None: + deadline = time.monotonic() + 2.0 + try: + while time.monotonic() < deadline: + frame = server.receive_once() + server.poll_resends() + if frame is None: + time.sleep(0.005) + continue + received = msgpack.unpackb(frame.payload, raw=False) + received_messages.append(received) + response = { + "schemaVersion": "cultnet.schema_catalog_response.v0", + "messageId": received["messageId"], + "schemas": [], + } + server.send("schema", msgpack.packb(response, use_bin_type=True)) + return + raise AssertionError("RUDP schema server did not receive a request") + finally: + server.close() + + server_thread = threading.Thread(target=serve_once, daemon=True) + server_thread.start() + client = CultMesh.create_client( + endpoint=f"rudp://{server_host}:{server_port}", + timeout_seconds=2.0, + connection_id=connection_id, + runtime_id="python-rudp-schema-client", + ) + + response = client.fetch_schema_catalog(message_id="rudp-transport-factory") + server_thread.join(timeout=2.0) + + self.assertFalse(server_thread.is_alive()) + self.assertEqual(response["schemaVersion"], "cultnet.schema_catalog_response.v0") + self.assertEqual(response["messageId"], "rudp-transport-factory") + self.assertEqual(received_messages[0]["schemaVersion"], "cultnet.schema_catalog_request.v0") + self.assertEqual(received_messages[0]["messageId"], "rudp-transport-factory") + + def test_cultmesh_facade_creates_rudp_client_from_peer_endpoint(self) -> None: + connection_id = 0x10203042 + server = CultMesh.create_rudp_server( + "python-cultmesh-rudp-server", + connection_id, + initial_sequence=100, + resend_delay_ms=25, + max_fragment_bytes=1024, + max_pending_reliable_packets=16, + ) + server_host, server_port = server.socket.getsockname()[:2] + endpoint = CultMesh.parse_rudp_endpoint(f"rudp://{server_host}:{server_port}") + self.assertEqual(endpoint.host, server_host) + self.assertEqual(endpoint.port, server_port) + peer = CultMeshPeerCard( + peer_id="python-cultmesh-rudp-server", + verse_id="local", + endpoints=(endpoint.uri,), + roles=("schema",), + authority_lease_id="lease:python-cultmesh-rudp-server", + ) + peers = CultMesh.create_peer_catalog() + leases = CultMesh.create_authority_lease_catalog() + peers.upsert(peer) + with self.assertRaisesRegex(ValueError, "No authorized RUDP peer"): + CultMesh.create_rudp_client_for_authorized_peer( + "python-cultmesh-rudp-client", + connection_id, + peers, + leases, + "local", + "schema", + resend_delay_ms=25, + max_fragment_bytes=1024, + max_pending_reliable_packets=16, + ) + leases.upsert( + CultMeshAuthorityLease( + lease_id="lease:python-cultmesh-rudp-server", + verse_id="local", + peer_id="python-cultmesh-rudp-server", + roles=("schema",), + valid_from=datetime.now(UTC) - timedelta(seconds=1), + expires_at=datetime.now(UTC) + timedelta(seconds=30), + issuer_runtime_id="python-authority", + ) + ) + client = CultMesh.create_rudp_client_for_authorized_peer( + "python-cultmesh-rudp-client", + connection_id, + peers, + leases, + "local", + "schema", + resend_delay_ms=25, + max_fragment_bytes=1024, + max_pending_reliable_packets=16, + ) + + try: + client.connect(b"join") + pump_rudp_handshake(client, server) + self.assertTrue(client.connected) + self.assertTrue(server.connected) + + client.send("schema", b"client-state") + server_frame = receive_rudp_frame(server) + self.assertEqual(server_frame.channel_id, "schema") + self.assertEqual(server_frame.payload, b"client-state") + self.assertEqual(server.profile["transports"][0]["protocol"], "rudp") + self.assertEqual(client.profile["transports"][0]["protocol"], "rudp") + finally: + client.close() + server.close() + + def test_cultmesh_node_applies_foreign_schema_snapshot_as_local_update(self) -> None: + document = define_database_entry_type( + "mesh.runtime-policy", + [ + ("schema_version", 0), + ("name", 1), + ("value", 2), + ], + schema_id="mesh.runtime-policy.current", + schema_name="mesh.runtime_policy", + schema_version="mesh.runtime_policy.v1", + ) + node = create_node(runtime_id="python-node") + node.register_document(document) + node.put(document, "policy:1", { + "schema_version": "mesh.runtime_policy.v1", + "name": "policy", + "value": "old", + }) + observed: list[CultMeshDatabaseChange] = [] + unsubscribe = node.database.watch_record(document, "policy:1", observed.append) + try: + applied = node.database.apply_snapshot_response({ + "schemaVersion": "cultnet.snapshot_response_raw.v0", + "messageId": "foreign-schema-snapshot", + "documents": [ + { + "schemaId": "sha256:foreign-mesh-runtime-policy", + "recordKey": "policy:1", + "storedAt": "2026-06-25T00:00:00Z", + "payloadEncoding": "messagepack", + "payload": document.encode_payload({ + "schema_version": "mesh.runtime_policy.v1", + "name": "policy", + "value": "new", + }), + }, + ], + }) + finally: + unsubscribe() + + local_schema_id = document.catalog_entry().schema_id + self.assertEqual(applied[0].schema_id, local_schema_id) + self.assertEqual(observed[0].schema_id, local_schema_id) + self.assertEqual(observed[0].change_kind, "updated") + self.assertEqual(observed[0].previous_value["value"], "old") + self.assertEqual(observed[0].value["value"], "new") + self.assertEqual(node.cache.get_required_envelope(document, "policy:1").schema_id, local_schema_id) + + def test_cultmesh_node_syncs_snapshot_and_shard_log_through_cultnet_client(self) -> None: + import msgpack # type: ignore + from cultnet_py import read_frame, write_frame + + document = define_database_entry_type( + "mesh.sync_item", + [ + ("name", 0), + ("category", 1), + ("value", 2, 0), + ], + cls=Item, + ) + node = create_node(runtime_id="python-node") + node.register_document(document) + schema_id = document.catalog_entry().schema_id + ready = threading.Event() + server_error: list[BaseException] = [] + port_holder: list[int] = [] + + def serve_requests() -> None: + try: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as server: + server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + server.bind(("127.0.0.1", 0)) + port_holder.append(server.getsockname()[1]) + server.listen(2) + ready.set() + for _ in range(2): + connection, _ = server.accept() + with connection: + stream = connection.makefile("rwb") + request = msgpack.unpackb(read_frame(stream), raw=False) + if request["schemaVersion"] == "cultnet.snapshot_request.v0": + response = { + "schemaVersion": "cultnet.snapshot_response_raw.v0", + "messageId": request["messageId"], + "documents": [ + { + "schemaId": schema_id, + "recordKey": "item:node", + "storedAt": "2026-06-13T00:00:00Z", + "payloadEncoding": "messagepack", + "payload": document.encode_payload(Item("node", "mesh", 1)), + } + ], + } + elif request["schemaVersion"] == "cultnet.shard_log_request.v0": + response = { + "schemaVersion": "cultnet.shard_log_response.v0", + "messageId": request["messageId"], + "shardId": request["shardId"], + "shardEpoch": request["shardEpoch"], + "resyncRequired": False, + "entries": [ + { + "sequence": 1, + "changeKind": "updated", + "put": { + "schemaVersion": "cultnet.document_put_raw.v0", + "messageId": "put-node", + "document": { + "schemaId": schema_id, + "recordKey": "item:node", + "storedAt": "2026-06-13T00:00:01Z", + "payloadEncoding": "messagepack", + "payload": document.encode_payload(Item("node", "mesh", 9)), + }, + "shardId": request["shardId"], + "shardEpoch": request["shardEpoch"], + }, + } + ], + } + else: + raise AssertionError(f"unexpected request {request['schemaVersion']}") + write_frame(stream, msgpack.packb(response, use_bin_type=True)) + stream.flush() + except BaseException as error: + server_error.append(error) + ready.set() + + thread = threading.Thread(target=serve_requests, daemon=True) + thread.start() + self.assertTrue(ready.wait(2.0)) + self.assertFalse(server_error) + + client = CultNetRawClient("127.0.0.1", port_holder[0], timeout_seconds=2.0) + snapshot_changes = node.sync_snapshot(client, schema_ids=[schema_id]) + log_changes = node.sync_shard_log(client, shard_id="mesh", shard_epoch=1) + + thread.join(2.0) + self.assertFalse(server_error) + self.assertEqual(snapshot_changes[0].record_key, "item:node") + self.assertEqual(log_changes[0].change_kind, "updated") + self.assertEqual(node.get_required(document, "item:node").value, 9) + + def test_cultmesh_node_sync_document_returns_requested_alias_type(self) -> None: + @dataclass + class CanonicalNote: + body: str + + @dataclass + class UiNote: + body: str + + document = define_database_entry_type( + "mesh.sync_alias_note", + [("body", 0)], + cls=CanonicalNote, + schema_id="mesh.sync_alias_note.v1", + schema_name="mesh.sync_alias_note", + schema_version="mesh.sync_alias_note.v1", + ) + alias = define_database_entry_type( + "mesh.sync_alias_note.ui", + [("body", 0)], + cls=UiNote, + schema_id="mesh.sync_alias_note.v1", + schema_name="mesh.sync_alias_note", + schema_version="mesh.sync_alias_note.v1", + ) + node = create_node(runtime_id="python-node") + node.register_document(document) + requests: list[dict[str, Any]] = [] + + class SnapshotClient: + def fetch_snapshot_response( + self, + *, + schema_ids: list[str] | None = None, + record_keys: list[str] | None = None, + shard_id: str | None = None, + shard_epoch: int | None = None, + ) -> dict[str, Any]: + requests.append({ + "schema_ids": schema_ids, + "record_keys": record_keys, + "shard_id": shard_id, + "shard_epoch": shard_epoch, + }) + return { + "schemaVersion": "cultnet.snapshot_response_raw.v0", + "messageId": "sync-document", + "documents": [ + { + "schemaId": document.catalog_entry().schema_id, + "recordKey": "note:remote", + "storedAt": "2026-06-27T00:00:00Z", + "payloadEncoding": "messagepack", + "payload": document.encode_payload(CanonicalNote("synced-alias")), + } + ], + } + + synced = node.sync_document(SnapshotClient(), alias, "note:remote") + facade_synced = CultMesh.sync_document(node, SnapshotClient(), alias, "note:remote") + + self.assertIsInstance(synced, UiNote) + self.assertEqual(synced.body, "synced-alias") + self.assertIsInstance(facade_synced, UiNote) + self.assertEqual(facade_synced.body, "synced-alias") + self.assertEqual(node.get_required(document, "note:remote").body, "synced-alias") + self.assertIsInstance(node.get_required(alias, "note:remote"), UiNote) + self.assertEqual(requests, [ + { + "schema_ids": ["mesh.sync_alias_note.v1"], + "record_keys": ["note:remote"], + "shard_id": None, + "shard_epoch": None, + }, + { + "schema_ids": ["mesh.sync_alias_note.v1"], + "record_keys": ["note:remote"], + "shard_id": None, + "shard_epoch": None, + }, + ]) + + def test_cultmesh_node_sync_document_from_publication_returns_requested_alias_type(self) -> None: + @dataclass + class CanonicalNote: + body: str + + @dataclass + class UiNote: + body: str + + document = define_database_entry_type( + "mesh.publication_alias_note", + [("body", 0)], + cls=CanonicalNote, + schema_id="mesh.publication_alias_note.v1", + schema_name="mesh.publication_alias_note", + schema_version="mesh.publication_alias_note.v1", + ) + alias = define_database_entry_type( + "mesh.publication_alias_note.ui", + [("body", 0)], + cls=UiNote, + schema_id="mesh.publication_alias_note.v1", + schema_name="mesh.publication_alias_note", + schema_version="mesh.publication_alias_note.v1", + ) + node = create_node(runtime_id="python-publication-node") + node.register_document(document) + requests: list[dict[str, Any]] = [] + + class SnapshotClient: + def fetch_snapshot_response( + self, + *, + schema_ids: list[str] | None = None, + record_keys: list[str] | None = None, + shard_id: str | None = None, + shard_epoch: int | None = None, + ) -> dict[str, Any]: + requests.append({ + "schema_ids": schema_ids, + "record_keys": record_keys, + "shard_id": shard_id, + "shard_epoch": shard_epoch, + }) + return { + "schemaVersion": "cultnet.snapshot_response_raw.v0", + "messageId": "publication-sync-document", + "documents": [ + { + "schemaId": document.catalog_entry().schema_id, + "recordKey": "note:published-peer", + "storedAt": "2026-06-27T00:00:00Z", + "payloadEncoding": "messagepack", + "payload": document.encode_payload(CanonicalNote("peer-publication-alias")), + } + ], + } + + source = CultMesh.publication_source_from_peer_snapshot( + SnapshotClient(), + shard_id="notes", + shard_epoch=3, + ) + + synced = node.sync_document_from_publication(source, alias, "note:published-peer") + facade_synced = CultMesh.sync_document_from_publication(node, source, alias, "note:published-peer") + + self.assertIsInstance(synced, UiNote) + self.assertEqual(synced.body, "peer-publication-alias") + self.assertIsInstance(facade_synced, UiNote) + self.assertEqual(facade_synced.body, "peer-publication-alias") + self.assertEqual(node.get_required(document, "note:published-peer").body, "peer-publication-alias") + self.assertEqual(requests, [ + { + "schema_ids": ["mesh.publication_alias_note.v1"], + "record_keys": ["note:published-peer"], + "shard_id": "notes", + "shard_epoch": 3, + }, + { + "schema_ids": ["mesh.publication_alias_note.v1"], + "record_keys": ["note:published-peer"], + "shard_id": "notes", + "shard_epoch": 3, + }, + ]) + + def test_cultmesh_node_sync_document_from_single_file_publication_uses_node_aliases(self) -> None: + @dataclass + class CanonicalNote: + body: str + + @dataclass + class UiNote: + body: str + + document = define_database_entry_type( + "mesh.file_publication_alias_note", + [("body", 0)], + cls=CanonicalNote, + schema_id="mesh.file_publication_alias_note.v1", + schema_name="mesh.file_publication_alias_note", + schema_version="mesh.file_publication_alias_note.v1", + ) + alias = define_database_entry_type( + "mesh.file_publication_alias_note.ui", + [("body", 0)], + cls=UiNote, + schema_id="mesh.file_publication_alias_note.v1", + schema_name="mesh.file_publication_alias_note", + schema_version="mesh.file_publication_alias_note.v1", + ) + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "publication.ccmp" + writer = CultCache() + writer.register_document_type(document) + writer.add_generic_store(SingleFileMessagePackBackingStore(path)) + writer.put(document, "note:published-file", CanonicalNote("file-publication-alias")) + + node = create_node(runtime_id="python-file-publication-node") + node.register_document(document) + source = CultMesh.publication_source_from_single_file(path) + + synced = CultMesh.sync_document_from_publication(node, source, alias, "note:published-file") + + self.assertIsInstance(synced, UiNote) + self.assertEqual(synced.body, "file-publication-alias") + self.assertEqual(node.get_required(document, "note:published-file").body, "file-publication-alias") + self.assertIsInstance(node.get_required(alias, "note:published-file"), UiNote) + + def test_cultmesh_node_sync_documents_from_publication_uses_binding_sources_and_aliases(self) -> None: + @dataclass + class CanonicalNote: + body: str + + @dataclass + class UiNote: + body: str + + document = define_database_entry_type( + "mesh.catalog_publication_alias_note", + [("body", 0)], + cls=CanonicalNote, + schema_id="mesh.catalog_publication_alias_note.v1", + schema_name="mesh.catalog_publication_alias_note", + schema_version="mesh.catalog_publication_alias_note.v1", + ) + alias = define_database_entry_type( + "mesh.catalog_publication_alias_note.ui", + [("body", 0)], + cls=UiNote, + schema_id="mesh.catalog_publication_alias_note.v1", + schema_name="mesh.catalog_publication_alias_note", + schema_version="mesh.catalog_publication_alias_note.v1", + ) + with tempfile.TemporaryDirectory() as tmp: + first_path = Path(tmp) / "first.ccmp" + second_path = Path(tmp) / "second.ccmp" + first = CultCache() + first.register_document_type(document) + first.add_generic_store(SingleFileMessagePackBackingStore(first_path)) + first.put(document, "note:first", CanonicalNote("first catalog publication")) + second = CultCache() + second.register_document_type(document) + second.add_generic_store(SingleFileMessagePackBackingStore(second_path)) + second.put(document, "note:second", CanonicalNote("second catalog publication")) + + node = create_node(runtime_id="python-catalog-publication-node") + node.register_document(document) + source = CultMesh.publication_source_from_single_file(first_path) + bindings = [ + CultMesh.publication_document(alias, "note:first"), + CultMesh.publication_document( + alias, + "note:second", + source=CultMesh.publication_source_from_single_file(second_path), + ), + ] + + synced = node.sync_documents_from_publication(source, bindings) + facade_synced = CultMesh.sync_documents_from_publication(node, source, bindings) + + self.assertEqual([value.body for value in synced], [ + "first catalog publication", + "second catalog publication", + ]) + self.assertTrue(all(isinstance(value, UiNote) for value in synced)) + self.assertEqual([value.body for value in facade_synced], [ + "first catalog publication", + "second catalog publication", + ]) + self.assertEqual(node.get_required(document, "note:first").body, "first catalog publication") + self.assertEqual(node.get_required(document, "note:second").body, "second catalog publication") + self.assertIsInstance(node.get_required(alias, "note:second"), UiNote) + + def test_cultmesh_node_emits_raw_put_and_delete_messages_for_local_writes(self) -> None: + document = define_database_entry_type( + "mesh.emit_item", + [ + ("name", 0), + ("category", 1), + ("value", 2, 0), + ], + cls=Item, + ) + node = create_node(runtime_id="python-node") + node.register_document(document) + + put = node.put_raw_message( + document, + "item:emit", + Item("wand", "gear", 4), + message_id="put-emit", + shard_id="mesh", + shard_epoch=1, + ).to_wire() + self.assertEqual(node.get_required(document, "item:emit").name, "wand") + self.assertEqual(put["schemaVersion"], "cultnet.document_put_raw.v0") + self.assertEqual(put["messageId"], "put-emit") + self.assertEqual(put["document"]["schemaId"], document.catalog_entry().schema_id) + self.assertEqual(put["document"]["recordKey"], "item:emit") + self.assertEqual(put["document"]["sourceRuntimeId"], "python-node") + self.assertEqual(put["shardId"], "mesh") + self.assertEqual(put["shardEpoch"], 1) + + delete = node.delete_raw_message( + document, + "item:emit", + message_id="delete-emit", + shard_id="mesh", + shard_epoch=1, + ).to_wire() + self.assertIsNone(node.get(document, "item:emit")) + self.assertEqual(delete["schemaVersion"], "cultnet.document_delete.v0") + self.assertEqual(delete["messageId"], "delete-emit") + self.assertEqual(delete["schemaId"], document.catalog_entry().schema_id) + self.assertEqual(delete["recordKey"], "item:emit") + self.assertEqual(delete["shardId"], "mesh") + self.assertEqual(delete["shardEpoch"], 1) + + def test_cultmesh_node_resolves_same_schema_aliases_for_local_state(self) -> None: + @dataclass + class CanonicalNote: + body: str + + @dataclass + class UiNote: + body: str + + document = define_database_entry_type( + "mesh.alias_note", + [("body", 0)], + cls=CanonicalNote, + schema_id="mesh.alias_note.v1", + schema_name="mesh.alias_note", + schema_version="mesh.alias_note.v1", + ) + alias = define_database_entry_type( + "mesh.alias_note.ui", + [("body", 0)], + cls=UiNote, + schema_id="mesh.alias_note.v1", + schema_name="mesh.alias_note", + schema_version="mesh.alias_note.v1", + ) + node = create_node(runtime_id="python-node") + node.register_document(document) + seen: list[CultMeshDatabaseChange] = [] + all_seen: list[CultMeshDatabaseChange] = [] + node.database.watch(all_seen.append, document=alias) + node.database.watch_record(alias, "note:alias", seen.append) + + node.put(alias, "note:alias", UiNote("from-alias-put")) + self.assertEqual(node.get_required(document, "note:alias").body, "from-alias-put") + alias_value = node.get_required(alias, "note:alias") + self.assertIsInstance(alias_value, UiNote) + self.assertEqual(alias_value.body, "from-alias-put") + alias_collection = node.get_all(alias) + self.assertEqual(len(alias_collection), 1) + self.assertIsInstance(alias_collection[0], UiNote) + self.assertEqual(alias_collection[0].body, "from-alias-put") + self.assertEqual(seen[0].document.type, alias.type) + self.assertEqual(seen[0].schema_id, "mesh.alias_note.v1") + self.assertIsInstance(seen[0].value, UiNote) + self.assertEqual(seen[0].value.body, "from-alias-put") + self.assertEqual(all_seen[0].document.type, alias.type) + self.assertIsInstance(all_seen[0].value, UiNote) + self.assertEqual(all_seen[0].value.body, "from-alias-put") + + node.put(document, "note:alias", CanonicalNote("from-canonical-put")) + self.assertEqual(seen[1].document.type, alias.type) + self.assertIsInstance(seen[1].value, UiNote) + self.assertIsInstance(seen[1].previous_value, UiNote) + self.assertEqual(seen[1].value.body, "from-canonical-put") + self.assertEqual(seen[1].previous_value.body, "from-alias-put") + + put = node.put_raw_message( + alias, + "note:alias", + UiNote("from-alias-raw-put"), + message_id="alias-put", + ).to_wire() + self.assertEqual(put["document"]["schemaId"], "mesh.alias_note.v1") + self.assertEqual(put["document"]["recordKey"], "note:alias") + self.assertEqual(node.get_required(alias, "note:alias").body, "from-alias-raw-put") + + delete = node.delete_raw_message(alias, "note:alias", message_id="alias-delete").to_wire() + self.assertEqual(delete["schemaId"], "mesh.alias_note.v1") + self.assertIsNone(node.get(alias, "note:alias")) + self.assertEqual(node.get_all(alias), []) + + def test_cultmesh_reactive_document_coalesces_explicit_alias_updates(self) -> None: + @dataclass + class CanonicalNote: + body: str + revision: int + + @dataclass + class UiNote: + body: str + revision: int + + document = define_database_entry_type( + "mesh.reactive_alias_note", + [("body", 0), ("revision", 1)], + cls=CanonicalNote, + schema_id="mesh.reactive_alias_note.v1", + schema_name="mesh.reactive_alias_note", + schema_version="mesh.reactive_alias_note.v1", + ) + alias = define_database_entry_type( + "mesh.reactive_alias_note.ui", + [("body", 0), ("revision", 1)], + cls=UiNote, + schema_id="mesh.reactive_alias_note.v1", + schema_name="mesh.reactive_alias_note", + schema_version="mesh.reactive_alias_note.v1", + ) + node = create_node(runtime_id="python-reactive") + node.register_document(document) + node.put(document, "note:reactive", CanonicalNote("initial", 1)) + seen: list[CultMeshDatabaseChange] = [] + node.database.watch_record(alias, "note:reactive", seen.append) + + reactive = node.authoritative_writer(alias, "note:reactive").reactive( + CultMeshReactiveDocumentOptions(flush_delay_seconds=0.02) + ) + try: + self.assertIsInstance(reactive.snapshot, UiNote) + reactive.update(lambda note: setattr(note, "body", "first-local-edit")) + reactive.update(lambda note: setattr(note, "body", "second-local-edit")) + reactive.update(lambda note: setattr(note, "revision", 2)) + + self._wait_until(lambda: node.get_required(alias, "note:reactive").body == "second-local-edit") + + self.assertFalse(reactive.is_dirty) + self.assertEqual(node.get_required(document, "note:reactive").revision, 2) + self.assertEqual(seen[-1].value.body, "second-local-edit") + finally: + reactive.dispose() + + def test_cultmesh_reactive_document_tracks_canonical_reconciliation_delta(self) -> None: + @dataclass + class Note: + body: str + revision: int + + document = define_database_entry_type( + "mesh.reactive_reconcile_note", + [("body", 0), ("revision", 1)], + cls=Note, + schema_id="mesh.reactive_reconcile_note.v1", + schema_name="mesh.reactive_reconcile_note", + schema_version="mesh.reactive_reconcile_note.v1", + ) + node = create_node(runtime_id="python-reactive-reconcile") + node.register_document(document) + node.put(document, "note:reconcile", Note("initial", 1)) + + reactive = CultMesh.authoritative_writer( + node, + document, + "note:reconcile", + ).reactive(CultMeshReactiveDocumentOptions(flush_delay_seconds=60)) + try: + reactive.update(lambda note: setattr(note, "body", "local-prediction")) + reactive.update(lambda note: setattr(note, "revision", 2)) + + node.put(document, "note:reconcile", Note("authoritative", 7)) + + self.assertEqual(reactive.snapshot.body, "local-prediction") + self.assertIsNotNone(reactive.reconciliation) + self.assertEqual(reactive.reconciliation.canonical.body, "authoritative") + self.assertEqual(reactive.reconciliation.predicted.body, "local-prediction") + self.assertEqual(reactive.reconciliation.delta["body"], "local-prediction") + self.assertEqual(reactive.reconciliation.delta["revision"], -5) + + reactive.flush() + self.assertEqual(node.get_required(document, "note:reconcile").body, "local-prediction") + self.assertIsNone(reactive.reconciliation) + finally: + reactive.dispose() + + def test_cultmesh_reactive_document_is_idle_until_explicit_nested_update(self) -> None: + document = define_document_type( + "mesh.nested_loadout", + encode=lambda value: value, + decode=lambda value: value, + ) + node = create_node(runtime_id="python-reactive-nested") + node.register_document(document) + node.put( + document, + "ship:nested", + {"loadout": {"weapon": {"tuning": {"damage": 10}}}}, + ) + + with patch("cultmesh_py.node.threading.Timer") as timer: + reactive = node.authoritative_writer(document, "ship:nested").reactive() + timer.assert_not_called() + + try: + borrowed = reactive.snapshot + borrowed["loadout"]["weapon"]["tuning"]["damage"] = 99 + self.assertEqual( + node.get_required(document, "ship:nested")["loadout"]["weapon"]["tuning"]["damage"], + 10, + ) + + reactive.update( + lambda draft: draft["loadout"]["weapon"]["tuning"].__setitem__("damage", 12) + ) + reactive.flush() + + self.assertEqual(reactive.snapshot["loadout"]["weapon"]["tuning"]["damage"], 12) + self.assertEqual( + node.get_required(document, "ship:nested")["loadout"]["weapon"]["tuning"]["damage"], + 12, + ) + finally: + reactive.dispose() + + def test_cultmesh_reactive_scheduling_scales_with_changed_documents_only(self) -> None: + document = define_document_type( + "mesh.reactive_scale_note", + encode=lambda value: value, + decode=lambda value: value, + ) + + for document_count in (1, 100, 1000): + node = create_node(runtime_id=f"python-reactive-scale-{document_count}") + node.register_document(document) + for index in range(document_count): + node.put(document, f"note:{index}", {"body": "initial", "revision": 1}) + + scheduled_timers: list[Any] = [] + + class FakeScheduledCall: + def __init__(self, delay_seconds: float, callback: Callable[[], None]) -> None: + self.delay_seconds = delay_seconds + self.callback = callback + self.cancelled = False + + def cancel(self) -> None: + self.cancelled = True + + class FakeScheduler: + def schedule( + self, + delay_seconds: float, + callback: Callable[[], None], + ) -> FakeScheduledCall: + call = FakeScheduledCall(delay_seconds, callback) + scheduled_timers.append(call) + return call + + reactive_documents: list[Any] = [] + with patch("cultmesh_py.node._REACTIVE_SCHEDULER", FakeScheduler()): + try: + for index in range(document_count): + reactive_documents.append( + node.authoritative_writer(document, f"note:{index}").reactive() + ) + + self.assertEqual( + len(scheduled_timers), + 0, + f"{document_count} idle documents must schedule no timers", + ) + + changed_document_count = max(1, document_count // 100) + for index in range(changed_document_count): + reactive_documents[index].update( + lambda draft, changed=index: draft.update( + body=f"changed-{changed}", + revision=2, + ) + ) + + self.assertEqual(len(scheduled_timers), changed_document_count) + self.assertEqual( + sum(reactive.is_dirty for reactive in reactive_documents), + changed_document_count, + ) + finally: + for reactive in reactive_documents: + reactive.dispose() + + self.assertTrue(all(timer.cancelled for timer in scheduled_timers)) + + def test_cultmesh_reactive_documents_share_one_lazy_scheduler_thread(self) -> None: + document = define_document_type( + "mesh.reactive_shared_scheduler_note", + encode=lambda value: value, + decode=lambda value: value, + ) + node = create_node(runtime_id="python-reactive-shared-scheduler") + node.register_document(document) + reactive_documents: list[Any] = [] + try: + for index in range(100): + node.put(document, f"note:{index}", {"body": "initial", "revision": 1}) + reactive_documents.append( + node.authoritative_writer(document, f"note:{index}").reactive( + CultMeshReactiveDocumentOptions(flush_delay_seconds=60.0) + ) + ) + + before = sum( + thread.name == "cultmesh-reactive-scheduler" + for thread in threading.enumerate() + ) + for index, reactive in enumerate(reactive_documents): + reactive.update( + lambda draft, changed=index: draft.update( + body=f"changed-{changed}", + revision=2, + ) + ) + after = sum( + thread.name == "cultmesh-reactive-scheduler" + for thread in threading.enumerate() + ) + + self.assertLessEqual(before, 1) + self.assertEqual(after, 1) + finally: + for reactive in reactive_documents: + reactive.dispose() + + def test_cultmesh_observed_document_cannot_publish_borrowed_mutation(self) -> None: + document = define_document_type( + "mesh.observed_note", + encode=lambda value: value, + decode=lambda value: value, + ) + node = create_node(runtime_id="python-observed") + node.register_document(document) + node.put(document, "note:observed", {"body": "initial"}) + + observed = node.observe_document(document, "note:observed") + try: + observed.current["body"] = "borrowed-local-edit" + self.assertEqual(node.get_required(document, "note:observed")["body"], "initial") + + node.put(document, "note:observed", {"body": "canonical"}) + self.assertEqual(observed.current["body"], "canonical") + finally: + observed.dispose() + + def test_cultmesh_node_uses_cultcache_store(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + document = define_database_entry_type("mesh.note", [("body", 0)]) + store_path = Path(tmp) / "mesh.cc" + node = create_node(store_path, runtime_id="mesh-test") + self.assertIsInstance(node.database, CultMeshDatabase) + node.database.register_document(document) + node.database.put(document, "note:1", {"body": "hello"}) + self.assertEqual(node.get_required(document, "note:1")["body"], "hello") + + reopened = create_node(store_path, runtime_id="mesh-test") + reopened.database.register_document(document) + reopened.database.pull() + self.assertEqual(reopened.database.get(document, "note:1")["body"], "hello") + + def test_cultmesh_node_options_attach_default_durable_shard_log_store(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + document = define_database_entry_type( + "mesh.durable_option_note", + [("body", 0)], + schema_id="mesh.durable_option_note.v1", + ) + cache_path = Path(tmp) / "world.cc" + node = CultMesh.create_node( + cache_path, + runtime_id="durable-option-primary", + options=CultMeshNodeOptions(enable_durable_shard_logs=True), + ) + node.database.register_document(document) + node.database.put_raw_message(document, "note:1", {"body": "stored"}, shard_id="notes", shard_epoch=5) + + log_path = Path(tmp) / "world.cultmesh" / "shard-logs" + self.assertTrue(log_path.exists()) + + reopened = CultMesh.start_node( + cache_path, + runtime_id="durable-option-primary", + enable_durable_shard_logs=True, + ) + reopened.database.register_document(document) + response = reopened.database.build_shard_log_response(shard_id="notes", shard_epoch=5) + + self.assertEqual(reopened.database.shard_ids(), ["notes"]) + self.assertEqual([entry.sequence for entry in response.entries], [1]) + self.assertEqual(response.entries[0].raw_document.record_key, "note:1") + + def test_cultmesh_database_watchers_observe_local_changes_and_unsubscribe(self) -> None: + document = define_database_entry_type("mesh.watch", [("body", 0)]) + node = CultMesh.create_node(runtime_id="mesh-watch") + node.database.register_document(document) + changes: list[CultMeshDatabaseChange] = [] + all_changes: list[CultMeshDatabaseChange] = [] + + unsubscribe_record = node.database.watch_record(document, "note:1", changes.append) + unsubscribe_all = node.database.watch(all_changes.append, document=document) + + node.database.put(document, "note:1", {"body": "hello"}) + node.database.put(document, "note:2", {"body": "skip"}) + node.database.put(document, "note:1", {"body": "updated"}) + node.database.delete(document, "note:1") + unsubscribe_record() + node.database.put(document, "note:1", {"body": "after-unsubscribe"}) + unsubscribe_all() + + self.assertEqual([change.change_kind for change in changes], ["added", "updated", "removed"]) + self.assertEqual([change.record_key for change in changes], ["note:1", "note:1", "note:1"]) + self.assertEqual(changes[1].previous_value, {"body": "hello"}) + self.assertIsNone(changes[2].value) + self.assertEqual([change.record_key for change in all_changes], ["note:1", "note:2", "note:1", "note:1", "note:1"]) + + def test_cultmesh_database_register_document_is_idempotent(self) -> None: + document = define_database_entry_type("mesh.idempotent", [("body", 0)]) + conflicting = define_database_entry_type("mesh.idempotent", [("title", 0)]) + node = CultMesh.create_node(runtime_id="mesh-idempotent") + + node.database.register_document(document) + node.database.register_document(document) + node.database.put(document, "note:1", {"body": "still registered once"}) + + self.assertEqual(node.database.get_required(document, "note:1")["body"], "still registered once") + self.assertEqual([registered.type for registered in node.documents], ["mesh.idempotent"]) + with self.assertRaises(ValueError): + node.database.register_document(conflicting) + + def test_cultmesh_database_global_document_facade_uses_singleton_key_and_watchers(self) -> None: + document = define_database_entry_type( + "mesh.global_settings", + [("theme", 0)], + global_document=True, + ) + node = CultMesh.create_node(runtime_id="mesh-global") + node.database.register_document(document) + changes: list[CultMeshDatabaseChange] = [] + + node.database.watch_global(document, changes.append) + node.database.put_global(document, {"theme": "ash"}) + node.put_global(document, {"theme": "ember"}) + node.delete_global(document) + + self.assertIsNone(node.database.get_global(document)) + self.assertEqual([change.record_key for change in changes], [node.cache.GLOBAL_KEY] * 3) + self.assertEqual([change.change_kind for change in changes], ["added", "updated", "removed"]) + self.assertEqual(changes[1].previous_value, {"theme": "ash"}) + self.assertEqual(changes[2].previous_value, {"theme": "ember"}) + with self.assertRaises(Exception): + node.database.put(document, "settings:wrong", {"theme": "bad"}) + + def test_cultmesh_database_creates_raw_snapshot_response_from_envelopes(self) -> None: + document = define_database_entry_type( + "mesh.snapshot_note", + [("body", 0)], + schema_id="mesh.snapshot_note.v1", + ) + source = CultMesh.create_node(runtime_id="mesh-snapshot-source") + source.database.register_document(document) + source.database.put(document, "note:1", {"body": "include"}) + source.database.put(document, "note:2", {"body": "skip"}) + + typed_response = source.database.build_snapshot_response( + message_id="snapshot-1", + schema_ids=["mesh.snapshot_note.v1"], + record_keys=["note:1"], + shard_id="notes", + shard_epoch=4, + shard_log_sequence=7, + ) + response = source.database.create_snapshot_response( + message_id="snapshot-1", + schema_ids=["mesh.snapshot_note.v1"], + record_keys=["note:1"], + shard_id="notes", + shard_epoch=4, + shard_log_sequence=7, + ) + + self.assertIsInstance(typed_response, CultNetRawSnapshotResponse) + self.assertEqual(typed_response.message_id, "snapshot-1") + self.assertEqual(typed_response.documents[0].record_key, "note:1") + self.assertEqual(typed_response.to_wire(), response) + self.assertEqual(source.build_snapshot_response(schema_ids=["mesh.snapshot_note.v1"]).documents[0].record_key, "note:1") + self.assertEqual(response["schemaVersion"], "cultnet.snapshot_response_raw.v0") + self.assertEqual(response["messageId"], "snapshot-1") + self.assertEqual(response["shardId"], "notes") + self.assertEqual(response["shardEpoch"], 4) + self.assertEqual(response["shardLogSequence"], 7) + self.assertEqual([record["recordKey"] for record in response["documents"]], ["note:1"]) + self.assertEqual(response["documents"][0]["schemaId"], "mesh.snapshot_note.v1") + self.assertEqual(response["documents"][0]["payloadEncoding"], "messagepack") + + target = CultMesh.create_node(runtime_id="mesh-snapshot-target") + target.database.register_document(document) + applied = target.database.apply_snapshot_response(response) + self.assertEqual([(record.schema_id, record.record_key) for record in applied], [("mesh.snapshot_note.v1", "note:1")]) + self.assertEqual(target.database.get_required(document, "note:1"), {"body": "include"}) + self.assertIsNone(target.database.get(document, "note:2")) + + typed_target = CultMesh.create_node(runtime_id="mesh-snapshot-typed-target") + typed_target.database.register_document(document) + typed_applied = typed_target.database.apply_snapshot_response(typed_response) + self.assertEqual([(record.schema_id, record.record_key) for record in typed_applied], [("mesh.snapshot_note.v1", "note:1")]) + self.assertEqual(typed_target.database.get_required(document, "note:1"), {"body": "include"}) + + def test_cultmesh_database_filters_snapshot_by_logged_shard_membership(self) -> None: + document = define_database_entry_type( + "mesh.sharded_snapshot_note", + [("body", 0)], + schema_id="mesh.sharded_snapshot_note.v1", + ) + source = CultMesh.create_node(runtime_id="mesh-sharded-snapshot-source") + source.database.register_document(document) + source.database.put_raw_message(document, "note:notes", {"body": "include"}, shard_id="notes", shard_epoch=2) + source.database.put_raw_message(document, "note:other", {"body": "skip"}, shard_id="other", shard_epoch=1) + + response = source.database.build_snapshot_response( + schema_ids=["mesh.sharded_snapshot_note.v1"], + shard_id="notes", + shard_epoch=2, + ) + + self.assertEqual(response.shard_id, "notes") + self.assertEqual(response.shard_epoch, 2) + self.assertEqual([record.record_key for record in response.documents], ["note:notes"]) + + def test_cultmesh_database_creates_shard_log_response_from_raw_mutations(self) -> None: + document = define_database_entry_type( + "mesh.log_note", + [("body", 0)], + schema_id="mesh.log_note.v1", + ) + source = CultMesh.create_node(runtime_id="mesh-log-source") + source.database.register_document(document) + source.database.put_raw_message(document, "note:1", {"body": "first"}, shard_id="notes", shard_epoch=3) + source.database.put_raw_message(document, "note:1", {"body": "second"}, shard_id="notes", shard_epoch=3) + source.database.put_raw_message(document, "note:2", {"body": "other"}, shard_id="other", shard_epoch=1) + source.database.delete_raw_message(document, "note:1", shard_id="notes", shard_epoch=3) + + self.assertEqual(source.database.shard_ids(), ["notes", "other"]) + self.assertEqual(source.database.shard_schema_ids("notes"), ["mesh.log_note.v1"]) + + typed_response = source.database.build_shard_log_response( + message_id="log-1", + shard_id="notes", + shard_epoch=3, + after_sequence=1, + limit=2, + ) + response = source.database.create_shard_log_response( + message_id="log-1", + shard_id="notes", + shard_epoch=3, + after_sequence=1, + limit=2, + ) + + self.assertIsInstance(typed_response, CultNetShardLogResponse) + self.assertEqual(typed_response.message_id, "log-1") + self.assertEqual([entry.sequence for entry in typed_response.entries], [2, 3]) + self.assertEqual(typed_response.to_wire(), response) + self.assertEqual([entry.sequence for entry in source.build_shard_log_response(shard_id="notes").entries], [1, 2, 3]) + self.assertEqual(response["schemaVersion"], "cultnet.shard_log_response.v0") + self.assertEqual(response["messageId"], "log-1") + self.assertEqual(response["shardId"], "notes") + self.assertEqual(response["shardEpoch"], 3) + self.assertFalse(response["resyncRequired"]) + self.assertEqual([entry["sequence"] for entry in response["entries"]], [2, 3]) + self.assertEqual([entry["changeKind"] for entry in response["entries"]], ["updated", "removed"]) + self.assertEqual(response["entries"][0]["put"]["document"]["recordKey"], "note:1") + self.assertEqual(response["entries"][1]["delete"]["recordKey"], "note:1") + empty = source.database.create_shard_log_response(shard_id="missing") + self.assertEqual(empty["shardEpoch"], 0) + self.assertEqual(empty["entries"], []) + self.assertTrue(empty["resyncRequired"]) + self.assertEqual(empty["reason"], "unknown_shard") + stale = source.database.create_shard_log_response(shard_id="notes", shard_epoch=2) + self.assertEqual(stale["shardEpoch"], 3) + self.assertEqual(stale["entries"], []) + self.assertTrue(stale["resyncRequired"]) + self.assertEqual(stale["reason"], "stale_epoch") + with self.assertRaises(ValueError): + source.database.create_shard_log_response(shard_id="notes", after_sequence=-1) + with self.assertRaises(ValueError): + source.database.create_shard_log_response(shard_id="notes", limit=-1) + + target = CultMesh.create_node(runtime_id="mesh-log-target") + target.database.register_document(document) + target.database.put(document, "note:1", {"body": "first"}) + applied = target.database.apply_shard_log_response(response) + self.assertEqual([record.change_kind for record in applied], ["updated", "removed"]) + self.assertIsNone(target.database.get(document, "note:1")) + self.assertIsNone(target.database.get(document, "note:2")) + + typed_target = CultMesh.create_node(runtime_id="mesh-log-typed-target") + typed_target.database.register_document(document) + typed_target.database.put(document, "note:1", {"body": "first"}) + typed_applied = typed_target.database.apply_shard_log_response(typed_response) + self.assertEqual([record.change_kind for record in typed_applied], ["updated", "removed"]) + self.assertIsNone(typed_target.database.get(document, "note:1")) + + def test_cultnet_file_shard_mutation_log_store_persists_authoritative_entries(self) -> None: + document = define_database_entry_type( + "mesh.durable_log_note", + [("body", 0)], + schema_id="mesh.durable_log_note.v1", + ) + with tempfile.TemporaryDirectory() as tmp: + store: CultNetShardMutationLogStore = CultNetFileShardMutationLogStore(Path(tmp) / "logs") + source = CultMesh.create_node(runtime_id="durable-primary") + source.database.register_document(document) + source.database.use_shard_mutation_log_store(store) + source.database.put_raw_message(document, "note:1", {"body": "first"}, shard_id="durable", shard_epoch=4) + source.database.put_raw_message(document, "note:2", {"body": "second"}, shard_id="durable", shard_epoch=4) + + reopened = CultMesh.create_node(runtime_id="durable-primary") + reopened.database.register_document(document) + reopened.database.use_shard_mutation_log_store(CultNetFileShardMutationLogStore(Path(tmp) / "logs")) + response = reopened.database.build_shard_log_response(shard_id="durable", shard_epoch=4) + self.assertEqual(reopened.database.shard_ids(), ["durable"]) + self.assertEqual(reopened.database.shard_schema_ids("durable"), ["mesh.durable_log_note.v1"]) + self.assertEqual([entry.sequence for entry in response.entries], [1, 2]) + self.assertEqual(response.entries[0].raw_document.record_key, "note:1") + + store.compact_through("durable", 1) + compacted = reopened.database.build_shard_log_response(shard_id="durable", shard_epoch=4, after_sequence=0) + retained = reopened.database.build_shard_log_response(shard_id="durable", shard_epoch=4, after_sequence=1) + self.assertTrue(compacted.resync_required) + self.assertEqual(compacted.reason, "compacted_history") + self.assertEqual(compacted.compacted_through, 1) + self.assertEqual([entry.sequence for entry in retained.entries], [2]) + + def test_cultnet_shard_replicator_pulls_logs_and_resyncs_from_snapshot(self) -> None: + document = define_database_entry_type( + "mesh.replica_note", + [("body", 0)], + schema_id="mesh.replica_note.v1", + ) + source = CultMesh.create_node(runtime_id="primary") + source.database.register_document(document) + source.database.put_raw_message(document, "note:1", {"body": "one"}, shard_id="notes", shard_epoch=3) + source.database.put_raw_message(document, "note:2", {"body": "two"}, shard_id="notes", shard_epoch=3) + with tempfile.TemporaryDirectory() as tmp: + cursor_path = Path(tmp) / "replica-cursors.msgpack" + server = CultMesh.serve_node(source) + try: + shard = CultNetShardDescriptor( + shard_id="notes", + owner_runtime_id="primary", + epoch=3, + is_primary=False, + schema_ids=("mesh.replica_note.v1",), + primary_endpoints=(f"cultnet://127.0.0.1:{server.port}",), + ) + target = CultMesh.create_node(runtime_id="replica") + target.database.register_document(document) + cursor_store = CultNetFileShardReplicaCursorStore(cursor_path) + replicator = CultNetShardReplicator( + target.database, + CultNetShardReplicatorOptions( + fetcher=CultNetSchemaShardLogFetcher(timeout_seconds=2.0), + snapshot_fetcher=CultNetSchemaShardSnapshotFetcher(timeout_seconds=2.0), + cursor_store=cursor_store, + batch_size=16, + ), + ) + + first_sequence = replicator.pull_once(shard) + restarted_replicator = CultNetShardReplicator( + target.database, + CultNetShardReplicatorOptions( + fetcher=CultNetSchemaShardLogFetcher(timeout_seconds=2.0), + snapshot_fetcher=CultNetSchemaShardSnapshotFetcher(timeout_seconds=2.0), + cursor_store=CultNetFileShardReplicaCursorStore(cursor_path), + batch_size=16, + ), + ) + second_sequence = restarted_replicator.pull_once(shard) + + self.assertEqual(first_sequence, 2) + self.assertEqual(second_sequence, 2) + self.assertEqual(target.get_required(document, "note:1")["body"], "one") + self.assertEqual(target.get_required(document, "note:2")["body"], "two") + cursor = CultNetFileShardReplicaCursorStore(cursor_path).read("notes") + self.assertIsNotNone(cursor) + assert cursor is not None + self.assertEqual(cursor.last_applied_sequence, 2) + self.assertEqual(cursor.shard_epoch, 3) + finally: + server.stop() + + resync_target = CultMesh.create_node(runtime_id="resync-replica") + resync_target.database.register_document(document) + snapshot = source.database.build_snapshot_response(shard_id="notes", shard_epoch=3, shard_log_sequence=2) + + class CompactingFetcher: + def fetch(self, shard: CultNetShardDescriptor, *, after_sequence: int, limit: int | None = None) -> CultNetShardLogResponse: + return CultNetShardLogResponse( + message_id="compact", + shard_id=shard.shard_id, + shard_epoch=shard.epoch, + entries=(), + resync_required=True, + reason="compacted_history", + compacted_through=after_sequence, + ) + + class SnapshotFetcher: + def fetch(self, shard: CultNetShardDescriptor) -> CultNetRawSnapshotResponse: + return snapshot + + resync_replicator = CultNetShardReplicator( + resync_target.database, + CultNetShardReplicatorOptions( + fetcher=CompactingFetcher(), + snapshot_fetcher=SnapshotFetcher(), + cursor_store=CultNetInMemoryShardReplicaCursorStore(), + ), + ) + resync_sequence = resync_replicator.pull_once(CultNetShardDescriptor( + shard_id="notes", + owner_runtime_id="primary", + epoch=3, + is_primary=False, + schema_ids=("mesh.replica_note.v1",), + primary_endpoints=("cultnet://primary:3075",), + )) + + self.assertEqual(resync_sequence, 2) + self.assertEqual(resync_target.get_required(document, "note:1")["body"], "one") + + def test_cultnet_shard_replicator_background_loop_polls_non_primary_shards(self) -> None: + document = define_database_entry_type( + "mesh.background_replica_note", + [("body", 0)], + schema_id="mesh.background_replica_note.v1", + ) + source = CultMesh.create_node(runtime_id="background-primary") + source.database.register_document(document) + target = CultMesh.create_node(runtime_id="background-replica") + target.database.register_document(document) + cursor_store = CultNetInMemoryShardReplicaCursorStore() + errors: list[Exception] = [] + + class FlakySourceFetcher: + def __init__(self) -> None: + self.calls = 0 + + def fetch(self, shard: CultNetShardDescriptor, *, after_sequence: int, limit: int | None = None) -> CultNetShardLogResponse: + self.calls += 1 + if self.calls == 1: + raise RuntimeError("transient fetch failure") + return source.database.build_shard_log_response( + shard_id=shard.shard_id, + shard_epoch=shard.epoch, + after_sequence=after_sequence, + limit=limit, + ) + + fetcher = FlakySourceFetcher() + shard = CultNetShardDescriptor( + shard_id="background", + owner_runtime_id="background-primary", + epoch=7, + is_primary=False, + schema_ids=("mesh.background_replica_note.v1",), + primary_endpoints=("cultnet://127.0.0.1:3075",), + ) + replicator = CultNetShardReplicator( + target.database, + CultNetShardReplicatorOptions( + fetcher=fetcher, + cursor_store=cursor_store, + batch_size=1, + poll_interval_seconds=0.01, + on_error=errors.append, + ), + ) + try: + source.database.put_raw_message(document, "note:1", {"body": "one"}, shard_id="background", shard_epoch=7) + replicator.start([shard]) + self._wait_until(lambda: target.database.get(document, "note:1") is not None) + source.database.put_raw_message(document, "note:2", {"body": "two"}, shard_id="background", shard_epoch=7) + self._wait_until(lambda: target.database.get(document, "note:2") is not None) + finally: + replicator.stop() + + cursor = cursor_store.read("background") + self.assertEqual(target.get_required(document, "note:1")["body"], "one") + self.assertEqual(target.get_required(document, "note:2")["body"], "two") + self.assertIsNotNone(cursor) + assert cursor is not None + self.assertEqual(cursor.last_applied_sequence, 2) + self.assertEqual(cursor.shard_epoch, 7) + self.assertEqual(len(errors), 1) + self.assertEqual(str(errors[0]), "transient fetch failure") + + def test_cultnet_schema_write_forwarder_sends_raw_writes_to_primary(self) -> None: + document = define_database_entry_type( + "mesh.forward_note", + [("body", 0)], + schema_id="mesh.forward_note.v1", + ) + primary = CultMesh.create_node(runtime_id="forward-primary") + primary.database.register_document(document) + server = CultMesh.serve_node(primary) + try: + shard = CultNetShardDescriptor( + shard_id="forward", + owner_runtime_id="forward-primary", + epoch=9, + is_primary=False, + schema_ids=("mesh.forward_note.v1",), + primary_endpoints=(f"cultnet://127.0.0.1:{server.port}",), + ) + forwarder: CultNetShardWriteForwarder = CultNetSchemaWriteForwarder(timeout_seconds=2.0) + put = document_put_raw( + message_id="forward-put", + key="note:forward", + schema_id=document.catalog_entry().schema_id, + stored_at="2026-06-14T00:00:00Z", + payload=document.encode_payload({"body": "forwarded"}), + ).to_wire() + delete = document_delete( + message_id="forward-delete", + schema_id=document.catalog_entry().schema_id, + record_key="note:forward", + ).to_wire() + + forwarder.forward_put(shard, put) + self._eventually(lambda: primary.database.get(document, "note:forward") is not None) + self.assertEqual(primary.database.get_required(document, "note:forward")["body"], "forwarded") + forwarder.forward_delete(shard, delete) + self._eventually(lambda: primary.database.get(document, "note:forward") is None) + + log = primary.database.build_shard_log_response(shard_id="forward", shard_epoch=9) + self.assertIsNone(primary.database.get(document, "note:forward")) + self.assertEqual([entry.change_kind for entry in log.entries], ["added", "removed"]) + self.assertEqual(log.entries[0].put["shardId"], "forward") + self.assertEqual(log.entries[1].delete["shardEpoch"], 9) + finally: + server.stop() + + def _eventually(self, predicate: Callable[[], bool], *, timeout_seconds: float = 2.0) -> None: + deadline = time.monotonic() + timeout_seconds + while time.monotonic() < deadline: + if predicate(): + return + time.sleep(0.01) + self.assertTrue(predicate()) + + def test_cultmesh_database_watchers_observe_name_and_index_changes(self) -> None: + document = define_database_entry_type( + "mesh.named_watch", + [("name", 0), ("kind", 1)], + name="name", + indexes={"kind": "kind"}, + ) + node = CultMesh.create_node(runtime_id="mesh-named-watch") + node.database.register_document(document) + name_changes: list[CultMeshDatabaseChange] = [] + index_changes: list[CultMeshDatabaseChange] = [] + + node.database.watch_by_name(document, "Potion", name_changes.append) + node.database.watch_by_index(document, "kind", "consumable", index_changes.append) + + node.database.put(document, "item:1", {"name": "Potion", "kind": "consumable"}) + node.database.put(document, "item:2", {"name": "Sword", "kind": "weapon"}) + node.database.put(document, "item:1", {"name": "Elixir", "kind": "rare"}) + node.database.delete(document, "item:1") + + self.assertEqual([change.change_kind for change in name_changes], ["added", "updated"]) + self.assertEqual([change.record_key for change in name_changes], ["item:1", "item:1"]) + self.assertEqual(name_changes[1].previous_value, {"name": "Potion", "kind": "consumable"}) + self.assertEqual([change.change_kind for change in index_changes], ["added", "updated"]) + self.assertEqual([change.record_key for change in index_changes], ["item:1", "item:1"]) + self.assertEqual(index_changes[1].value, {"name": "Elixir", "kind": "rare"}) + + def test_cultmesh_database_collection_reads_resolve_same_schema_aliases(self) -> None: + @dataclass + class CanonicalItem: + name: str + kind: str + + @dataclass + class UiItem: + name: str + kind: str + + document = define_database_entry_type( + "mesh.alias_collection_item", + [("name", 0), ("kind", 1)], + cls=CanonicalItem, + name="name", + indexes={"kind": "kind"}, + schema_id="mesh.alias_collection_item.v1", + schema_name="mesh.alias_collection_item", + schema_version="mesh.alias_collection_item.v1", + ) + alias = define_database_entry_type( + "mesh.alias_collection_item.ui", + [("name", 0), ("kind", 1)], + cls=UiItem, + name="name", + indexes={"kind": "kind"}, + schema_id="mesh.alias_collection_item.v1", + schema_name="mesh.alias_collection_item", + schema_version="mesh.alias_collection_item.v1", + ) + node = CultMesh.create_node(runtime_id="mesh-alias-collection") + node.database.register_document(document) + + node.database.put(alias, "item:potion", UiItem("Potion", "consumable")) + node.database.put(document, "item:sword", CanonicalItem("Sword", "weapon")) + + all_items = node.database.get_all(alias) + potion = node.database.get_by_name(alias, "Potion") + sword = node.get_by_index(alias, "kind", "weapon") + + self.assertEqual([item.name for item in all_items], ["Potion", "Sword"]) + self.assertTrue(all(isinstance(item, UiItem) for item in all_items)) + self.assertIsInstance(potion, UiItem) + self.assertEqual(potion.kind, "consumable") + self.assertIsInstance(sword, UiItem) + self.assertEqual(sword.name, "Sword") + self.assertIsNone(node.database.get_by_name(alias, "Missing")) + + def test_cultmesh_database_alias_collection_watches_emit_alias_values(self) -> None: + @dataclass + class CanonicalItem: + name: str + kind: str + + @dataclass + class UiItem: + name: str + kind: str + + document = define_database_entry_type( + "mesh.alias_collection_watch_item", + [("name", 0), ("kind", 1)], + cls=CanonicalItem, + name="name", + indexes={"kind": "kind"}, + schema_id="mesh.alias_collection_watch_item.v1", + schema_name="mesh.alias_collection_watch_item", + schema_version="mesh.alias_collection_watch_item.v1", + ) + alias = define_database_entry_type( + "mesh.alias_collection_watch_item.ui", + [("name", 0), ("kind", 1)], + cls=UiItem, + name="name", + indexes={"kind": "kind"}, + schema_id="mesh.alias_collection_watch_item.v1", + schema_name="mesh.alias_collection_watch_item", + schema_version="mesh.alias_collection_watch_item.v1", + ) + node = CultMesh.create_node(runtime_id="mesh-alias-collection-watch") + node.database.register_document(document) + name_changes: list[CultMeshDatabaseChange] = [] + index_changes: list[CultMeshDatabaseChange] = [] + + node.database.watch_by_name(alias, "Potion", name_changes.append) + node.database.watch_by_index(alias, "kind", "consumable", index_changes.append) + + node.database.put(document, "item:potion", CanonicalItem("Potion", "consumable")) + node.database.put(document, "item:potion", CanonicalItem("Elixir", "rare")) + + self.assertEqual([change.change_kind for change in name_changes], ["added", "updated"]) + self.assertTrue(all(change.document.type == alias.type for change in name_changes)) + self.assertIsInstance(name_changes[0].value, UiItem) + self.assertIsInstance(name_changes[1].value, UiItem) + self.assertIsInstance(name_changes[1].previous_value, UiItem) + self.assertEqual(name_changes[1].previous_value.name, "Potion") + self.assertEqual(name_changes[1].value.name, "Elixir") + self.assertEqual([change.change_kind for change in index_changes], ["added", "updated"]) + self.assertTrue(all(change.document.type == alias.type for change in index_changes)) + self.assertIsInstance(index_changes[0].value, UiItem) + self.assertIsInstance(index_changes[1].previous_value, UiItem) + + def test_cultmesh_database_watch_by_name_and_index_validate_lookup_shape(self) -> None: + document = define_database_entry_type("mesh.unnamed_watch", [("body", 0)]) + node = CultMesh.create_node(runtime_id="mesh-watch-validation") + node.database.register_document(document) + + with self.assertRaises(ValueError): + node.database.watch_by_name(document, "missing", lambda _: None) + with self.assertRaises(ValueError): + node.database.watch_by_index(document, "missing", "value", lambda _: None) + with self.assertRaises(ValueError): + node.database.watch_by_index(document, "", "value", lambda _: None) + with self.assertRaises(ValueError): + node.database.watch_by_index(document, "missing", "", lambda _: None) + + def test_cultmesh_facade_matches_peer_runtime_entrypoints(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + store_path = Path(tmp) / "mesh.cc" + node = CultMesh.start_node(store_path, runtime_id="mesh-facade") + + self.assertEqual(node.runtime_id, "mesh-facade") + self.assertIsInstance(CultMesh.create_node(), type(node)) + self.assertIsInstance(CultMesh.create_verse_catalog(), CultMeshVerseCatalog) + self.assertIsInstance(CultMesh.create_peer_catalog(), CultMeshPeerCatalog) + self.assertIsInstance(CultMesh.create_authority_lease_catalog(), CultMeshAuthorityLeaseCatalog) + self.assertIsInstance(CultMesh.create_stream_catalog(), CultMeshStreamCatalog) + self.assertIsInstance(CultMesh.create_schema_catalog(), CultNetSchemaCatalog) + self.assertIsInstance(CultMesh.create_builtin_schema_catalog(), CultNetSchemaCatalog) + self.assertIsInstance(CultMesh.create_shard_catalog(), CultNetShardCatalog) + builtins = CultMesh.create_builtin_schema_catalog( + include_schema_json=False, + kinds=["shared_contract"], + ) + self.assertEqual( + builtins.list()[0].schema_id, + "https://github.com/GameCult/cultnet-ts/contracts/cultnet.transport-profile.schema.json", + ) + self.assertIsNone(builtins.list()[0].schema_json) + peer_exchange_schema_id = ( + "https://github.com/GameCult/cultnet-ts/contracts/cultmesh.peer-exchange-request.schema.json" + ) + filtered_builtin = CultMesh.create_builtin_schema_catalog( + include_schema_json=True, + schema_ids=[peer_exchange_schema_id], + ) + self.assertEqual( + filtered_builtin.list()[0].schema_version, + "cultmesh.peer_exchange_request.v0", + ) + self.assertIn( + "cultmesh.peer_exchange_request.v0", + filtered_builtin.list()[0].schema_json or "", + ) + shard_catalog = CultMesh.create_shard_catalog() + shard_catalog.upsert( + CultNetShardDescriptor( + shard_id="python-notes-a", + owner_runtime_id="mesh-facade", + epoch=7, + is_primary=True, + schema_ids=("python.note.v1",), + key_prefix="note:", + ) + ) + self.assertEqual( + [ + shard.shard_id + for shard in shard_catalog.list(schema_ids=["python.note.v1"], record_keys=["note:1"]) + ], + ["python-notes-a"], + ) + verse_client = CultMesh.create_verse_discovery_client("127.0.0.1", 4010, timeout_seconds=1.5) + peer_client = CultMesh.create_peer_exchange_client("127.0.0.1", 4011, timeout_seconds=1.5) + raw_client = CultMesh.create_client("127.0.0.1", 4012, timeout_seconds=1.5) + connected_client = CultMesh.connect_client("127.0.0.1", 4013, timeout_seconds=1.5) + self.assertIsInstance(verse_client, CultMeshVerseDiscoveryClient) + self.assertIsInstance(peer_client, CultMeshPeerExchangeClient) + self.assertIsInstance(verse_client, CultMeshDiscoveryClient) + self.assertIsInstance(peer_client, CultMeshDiscoveryClient) + self.assertIsInstance(raw_client, CultNetRawClient) + self.assertIsInstance(connected_client, CultNetRawClient) + self.assertEqual((verse_client.host, verse_client.port, verse_client.timeout_seconds), ("127.0.0.1", 4010, 1.5)) + self.assertEqual((peer_client.host, peer_client.port, peer_client.timeout_seconds), ("127.0.0.1", 4011, 1.5)) + self.assertEqual((raw_client.host, raw_client.port, raw_client.timeout_seconds), ("127.0.0.1", 4012, 1.5)) + self.assertEqual((connected_client.host, connected_client.port, connected_client.timeout_seconds), ("127.0.0.1", 4013, 1.5)) + + def test_cultmesh_simulation_fact_uses_csharp_slot_contract(self) -> None: + import msgpack # type: ignore + + candidate = { + "shardId": "arena", + "shardEpoch": 4, + "frame": 100, + "subjectId": "bob", + "claimKind": "hit", + "claimHash": compute_simulation_claim_hash("hit", "alice", "bob", "frame:100"), + "claimSummary": "alice shot bob first", + "witnessCount": 2, + "supportWeight": 2.0, + "totalWeight": 2.0, + "confidence": 1.0, + "hasQuorum": True, + } + + fact = CultMeshSimulationFact.from_candidate(candidate, committed_at="2026-06-13T00:00:00Z") + payload = simulation_fact_document.encode_payload(fact) + slots = msgpack.unpackb(payload, raw=False) + + self.assertEqual(CultMeshSimulationFact.create_record_key(candidate), f"simulation:{fact.fact_id}") + self.assertEqual(slots[0], fact.fact_id) + self.assertEqual(slots[1], "arena") + self.assertEqual(slots[2], 4) + self.assertEqual(slots[3], 100) + self.assertEqual(slots[4], "bob") + self.assertEqual(slots[5], "hit") + self.assertEqual(slots[6], candidate["claimHash"]) + self.assertEqual(slots[7], "alice shot bob first") + self.assertEqual(slots[8], 2) + self.assertEqual(slots[9], 2.0) + self.assertEqual(slots[10], 2.0) + self.assertEqual(slots[11], 1.0) + self.assertEqual(slots[12], "2026-06-13T00:00:00Z") + + def test_cultmesh_simulation_fact_committer_rejects_without_quorum_and_stores_fact(self) -> None: + candidate = { + "shardId": "arena", + "shardEpoch": 4, + "frame": 100, + "subjectId": "bob", + "claimKind": "hit", + "claimHash": compute_simulation_claim_hash("hit", "alice", "bob", "frame:100"), + "claimSummary": "alice shot bob first", + "witnessCount": 2, + "supportWeight": 2.0, + "totalWeight": 2.0, + "confidence": 1.0, + "hasQuorum": True, + } + + with tempfile.TemporaryDirectory() as tmp: + node = CultMesh.create_node(Path(tmp) / "facts.cc", runtime_id="mesh-facts") + committer = CultMesh.create_simulation_fact_committer(node) + + rejected = dict(candidate) + rejected["hasQuorum"] = False + with self.assertRaisesRegex(ValueError, "before quorum"): + committer.commit(rejected) + + committed = committer.commit(candidate, committed_at="2026-06-13T00:00:00Z") + stored = node.get_required(simulation_fact_document, committed.key) + + self.assertEqual(stored.claim_hash, candidate["claimHash"]) + self.assertEqual(stored.committed_at, "2026-06-13T00:00:00Z") + self.assertEqual(committed.fact.fact_id, stored.fact_id) + + def test_cultmesh_game_session_submits_observations_and_commits_quorum_once(self) -> None: + claim_hash = compute_simulation_claim_hash("hit", "alice", "bob", "frame:100") + first = simulation_observation( + message_id="obs-1", + witness_runtime_id="watcher-1", + shard_id="arena", + shard_epoch=4, + frame=100, + subject_id="bob", + claim_kind="hit", + claim_hash=claim_hash, + claim_summary="alice shot bob first", + ).to_wire() + second = simulation_observation( + message_id="obs-2", + witness_runtime_id="watcher-2", + shard_id="arena", + shard_epoch=4, + frame=100, + subject_id="bob", + claim_kind="hit", + claim_hash=claim_hash, + claim_summary="alice shot bob first", + ).to_wire() + + with tempfile.TemporaryDirectory() as tmp: + node = CultMesh.create_node(Path(tmp) / "session.cc", runtime_id="mesh-session") + session = CultMesh.create_game_session( + node, + CultMeshGameSessionOptions( + consensus_options=CultNetSimulationConsensusOptions(minimum_witnesses=2, quorum_ratio=1.0) + ), + ) + seen_candidates: list[CultNetSimulationConsensusCandidate] = [] + seen_facts: list[CultMeshDatabaseChange] = [] + unsubscribe_candidates = session.watch_candidates(seen_candidates.append) + unsubscribe_facts = session.watch_simulation_facts(seen_facts.append) + + self.assertEqual(session.submit_and_commit(CultNetSimulationObservation.from_wire(first)), []) + typed_candidates = session.submit_observation_candidates(CultNetSimulationObservation.from_wire(second)) + commits = session.commit_quorum_candidate_objects(typed_candidates) + replay = session.commit_quorum_candidates(session.submit_observation(second)) + unsubscribe_candidates() + unsubscribe_facts() + + self.assertEqual(len(typed_candidates), 1) + self.assertTrue(typed_candidates[0].has_quorum) + self.assertEqual(len(commits), 1) + self.assertEqual(replay, []) + self.assertEqual([candidate.claim_hash for candidate in seen_candidates], [claim_hash, claim_hash, claim_hash]) + self.assertEqual(len(seen_facts), 1) + self.assertEqual(seen_facts[0].change_kind, "added") + self.assertEqual(seen_facts[0].value.claim_hash, claim_hash) + stored = node.get_required(simulation_fact_document, commits[0].key) + self.assertEqual(stored.claim_hash, claim_hash) + self.assertEqual(stored.witness_count, 2) + + def test_cultmesh_game_session_serves_catalogs_and_observations_over_wire(self) -> None: + verse_catalog = CultMeshVerseCatalog() + peer_catalog = CultMeshPeerCatalog() + verse_catalog.upsert(CultMeshVerseDescriptor( + verse_id="session-verse", + display_name="Session Verse", + authority_model="local", + compatibility=CultMeshVerseCompatibility("cultmesh.v0", "session-rules"), + )) + peer_catalog.upsert(CultMeshPeerCard( + peer_id="session-peer", + verse_id="session-verse", + endpoints=("cultnet://127.0.0.1:0",), + roles=("simulation-observer", "read-replica"), + shard_ids=("arena",), + )) + session = CultMesh.create_game_session( + CultMesh.create_node(runtime_id="session-peer"), + CultMeshGameSessionOptions( + verse_catalog=verse_catalog, + peer_catalog=peer_catalog, + consensus_options=CultNetSimulationConsensusOptions(minimum_witnesses=2, quorum_ratio=1.0), + ), + ) + server = session.serve(display_name="Session Peer") + try: + raw_client = CultMesh.create_client("127.0.0.1", server.port, timeout_seconds=2.0) + hello_response = raw_client.request( + hello(runtime_id="session-prober"), + expected_schema_version="cultnet.hello.v0", + ) + discovery_client = CultMesh.create_verse_discovery_client("127.0.0.1", server.port, timeout_seconds=2.0) + verses = discovery_client.fetch_verses(transport_version="cultmesh.v0") + peers = discovery_client.fetch_peers(verse_id="session-verse", roles=["simulation-observer"]) + claim_hash = compute_simulation_claim_hash("hit", "alice", "bob", "frame:100") + first = raw_client.request( + simulation_observation( + message_id="session-wire-1", + witness_runtime_id="watcher-1", + shard_id="arena", + shard_epoch=4, + frame=100, + subject_id="bob", + claim_kind="hit", + claim_hash=claim_hash, + claim_summary="alice shot bob first", + ), + expected_schema_version="cultnet.simulation_consensus_candidate.v0", + ) + second = raw_client.request( + simulation_observation( + message_id="session-wire-2", + witness_runtime_id="watcher-2", + shard_id="arena", + shard_epoch=4, + frame=100, + subject_id="bob", + claim_kind="hit", + claim_hash=claim_hash, + claim_summary="alice shot bob first", + ), + expected_schema_version="cultnet.simulation_consensus_candidate.v0", + ) + finally: + server.stop() + + self.assertIn("cultnet.simulation_observation.v0", hello_response["supportedMessageVersions"]) + self.assertEqual(verses[0].verse_id, "session-verse") + self.assertEqual(peers[0].peer_id, "session-peer") + self.assertIn("simulation-observer", peers[0].roles) + self.assertFalse(first["hasQuorum"]) + self.assertEqual(first["witnessCount"], 1) + self.assertTrue(second["hasQuorum"]) + self.assertEqual(second["witnessCount"], 2) + + def test_cultmesh_game_session_can_disable_served_simulation_observations(self) -> None: + verse_catalog = CultMeshVerseCatalog() + verse_catalog.upsert(CultMeshVerseDescriptor( + verse_id="session-quiet", + display_name="Quiet Session", + authority_model="local", + compatibility=CultMeshVerseCompatibility("cultmesh.v0", "quiet-rules"), + )) + session = CultMesh.create_game_session( + CultMesh.create_node(runtime_id="quiet-session-peer"), + CultMeshGameSessionOptions( + verse_catalog=verse_catalog, + serve_simulation_observations=False, + ), + ) + server = session.serve(display_name="Quiet Session Peer") + try: + raw_client = CultMesh.create_client("127.0.0.1", server.port, timeout_seconds=2.0) + hello_response = raw_client.request( + hello(runtime_id="quiet-session-prober"), + expected_schema_version="cultnet.hello.v0", + ) + discovery_client = CultMesh.create_verse_discovery_client("127.0.0.1", server.port, timeout_seconds=2.0) + verses = discovery_client.fetch_verses(transport_version="cultmesh.v0") + with self.assertRaisesRegex(CultNetPeerError, "Simulation observations are not enabled") as raised: + raw_client.request( + simulation_observation( + message_id="quiet-session-sim", + witness_runtime_id="watcher-1", + shard_id="arena", + shard_epoch=4, + frame=100, + subject_id="bob", + claim_kind="hit", + claim_hash=compute_simulation_claim_hash("hit", "alice", "bob", "frame:100"), + ), + expected_schema_version="cultnet.simulation_consensus_candidate.v0", + ) + finally: + server.stop() + + self.assertNotIn("cultnet.simulation_observation.v0", hello_response["supportedMessageVersions"]) + self.assertEqual(verses[0].verse_id, "session-quiet") + self.assertEqual(raised.exception.response["code"], "simulation_observations_disabled") + + def test_cultmesh_game_session_prediction_requires_scope_and_reconciles_shard_log(self) -> None: + note_doc = define_database_entry_type( + "mesh.input", + [("body", 0)], + schema_id="mesh.input.v1", + ) + with tempfile.TemporaryDirectory() as tmp: + node = CultMesh.create_node(Path(tmp) / "prediction.cc", runtime_id="client-a") + node.register_document(note_doc) + session = CultMesh.create_game_session( + node, + CultMeshGameSessionOptions( + client_authority_scopes=( + CultNetClientAuthorityScope( + "client-a", + schema_ids=("mesh.input.v1",), + key_prefix="input:client-a", + ), + ) + ), + ) + + with self.assertRaisesRegex(ValueError, "does not have client prediction authority"): + session.predict(note_doc, "input:other", {"body": "nope"}) + + prediction = session.predict(note_doc, "input:client-a:move", {"body": "predicted"}) + put = document_put_raw( + message_id="authoritative", + key=prediction.key, + schema_id=prediction.schema_id, + stored_at="2026-06-13T00:00:00Z", + payload=note_doc.encode_payload({"body": "authoritative"}), + shard_id="inputs", + shard_epoch=1, + ) + authoritative_response = { + "schemaVersion": "cultnet.shard_log_response.v0", + "messageId": "inputs-log", + "shardId": "inputs", + "shardEpoch": 1, + "entries": [ + { + "sequence": 1, + "committedAt": "2026-06-13T00:00:00Z", + "changeKind": "updated", + "put": put.to_wire(), + } + ], + "resyncRequired": False, + } + changes = session.apply_shard_log_response(CultNetShardLogResponse.from_wire(authoritative_response)) + + self.assertEqual(changes[0].change_kind, "reconciled") + self.assertEqual(node.get_required(note_doc, prediction.key)["body"], "authoritative") + self.assertEqual(session.pending_predictions(), ()) + self.assertEqual(session.resimulation_inputs(), ()) + + def test_cultmesh_game_session_partial_reconciliation_keeps_unmatched_predictions(self) -> None: + note_doc = define_database_entry_type( + "mesh.partial_input", + [("body", 0)], + schema_id="mesh.partial_input.v1", + ) + node = CultMesh.create_node(runtime_id="client-partial") + node.register_document(note_doc) + session = CultMesh.create_game_session( + node, + CultMeshGameSessionOptions( + client_authority_scopes=( + CultNetClientAuthorityScope( + "client-partial", + schema_ids=("mesh.partial_input.v1",), + key_prefix="input:client-partial", + ), + ) + ), + ) + pending = session.predict(note_doc, "input:client-partial:pending", {"body": "pending-local"}) + accepted = session.predict(note_doc, "input:client-partial:accepted", {"body": "accepted-local"}) + put = document_put_raw( + message_id="authoritative-partial", + key=accepted.key, + schema_id=accepted.schema_id, + stored_at="2026-06-13T00:00:00Z", + payload=note_doc.encode_payload({"body": "accepted-authority"}), + shard_id="inputs", + shard_epoch=1, + ) + + changes = session.apply_shard_log_response({ + "schemaVersion": "cultnet.shard_log_response.v0", + "messageId": "inputs-partial-log", + "shardId": "inputs", + "shardEpoch": 1, + "entries": [ + { + "sequence": 1, + "committedAt": "2026-06-13T00:00:00Z", + "changeKind": "updated", + "put": put.to_wire(), + } + ], + "resyncRequired": False, + }) + + self.assertEqual([(change.change_kind, change.record_key) for change in changes], [ + ("reconciled", accepted.key), + ]) + self.assertEqual(node.get_required(note_doc, accepted.key)["body"], "accepted-authority") + self.assertEqual(node.get_required(note_doc, pending.key)["body"], "pending-local") + self.assertEqual([prediction.key for prediction in session.pending_predictions()], [pending.key]) + self.assertEqual([prediction.key for prediction in session.resimulation_inputs()], [pending.key]) + + def test_cultmesh_game_session_rolls_back_pending_predictions(self) -> None: + note_doc = define_database_entry_type( + "mesh.rollback_input", + [("body", 0)], + schema_id="mesh.rollback_input.v1", + ) + node = CultMesh.create_node(runtime_id="client-rollback") + node.register_document(note_doc) + node.put(note_doc, "input:client-rollback:existing", {"body": "previous"}) + session = CultMesh.create_game_session( + node, + CultMeshGameSessionOptions( + client_authority_scopes=( + CultNetClientAuthorityScope( + "client-rollback", + schema_ids=("mesh.rollback_input.v1",), + key_prefix="input:client-rollback", + ), + ) + ), + ) + + existing = session.predict(note_doc, "input:client-rollback:existing", {"body": "predicted-existing"}) + created = session.predict(note_doc, "input:client-rollback:new", {"body": "predicted-new"}) + self.assertEqual( + [prediction.key for prediction in session.pending_predictions()], + ["input:client-rollback:existing", "input:client-rollback:new"], + ) + self.assertEqual([prediction.key for prediction in session.resimulation_inputs()], [ + "input:client-rollback:existing", + "input:client-rollback:new", + ]) + + restored = session.rollback_prediction(existing) + self.assertIsNotNone(restored) + assert restored is not None + self.assertEqual(restored.change_kind, "rolled_back") + self.assertEqual(restored.value, {"body": "previous"}) + self.assertEqual(node.get_required(note_doc, "input:client-rollback:existing")["body"], "previous") + + rolled_back = session.rollback_predictions() + self.assertEqual([(change.change_kind, change.record_key, change.value) for change in rolled_back], [ + ("rolled_back", created.key, None), + ]) + self.assertIsNone(node.get(note_doc, created.key)) + self.assertEqual(session.pending_predictions(), ()) + + def test_cultmesh_game_session_authoritative_delete_rolls_back_prediction(self) -> None: + note_doc = define_database_entry_type( + "mesh.deleted_input", + [("body", 0)], + schema_id="mesh.deleted_input.v1", + ) + node = CultMesh.create_node(runtime_id="client-delete") + node.register_document(note_doc) + session = CultMesh.create_game_session( + node, + CultMeshGameSessionOptions( + client_authority_scopes=( + CultNetClientAuthorityScope( + "client-delete", + schema_ids=("mesh.deleted_input.v1",), + key_prefix="input:client-delete", + ), + ) + ), + ) + + prediction = session.predict(note_doc, "input:client-delete:move", {"body": "predicted"}) + delete = document_delete( + message_id="authoritative-delete", + schema_id=prediction.schema_id, + record_key=prediction.key, + shard_id="inputs", + shard_epoch=1, + ) + changes = session.apply_shard_log_response({ + "schemaVersion": "cultnet.shard_log_response.v0", + "messageId": "inputs-delete-log", + "shardId": "inputs", + "shardEpoch": 1, + "entries": [ + { + "sequence": 1, + "committedAt": "2026-06-13T00:00:00Z", + "changeKind": "removed", + "delete": delete.to_wire(), + } + ], + "resyncRequired": False, + }) + + self.assertEqual(changes[0].change_kind, "rolled_back") + self.assertIsNone(node.get(note_doc, prediction.key)) + self.assertEqual(session.pending_predictions(), ()) + + def test_cultmesh_database_watchers_observe_authoritative_shard_log_reconciliation(self) -> None: + note_doc = define_database_entry_type( + "mesh.watched_input", + [("body", 0)], + schema_id="mesh.watched_input.v1", + ) + node = CultMesh.create_node(runtime_id="client-watch") + node.database.register_document(note_doc) + seen: list[CultMeshDatabaseChange] = [] + node.database.watch_record(note_doc, "input:client-watch:move", seen.append) + session = CultMesh.create_game_session( + node, + CultMeshGameSessionOptions( + client_authority_scopes=( + CultNetClientAuthorityScope( + "client-watch", + schema_ids=("mesh.watched_input.v1",), + key_prefix="input:client-watch", + ), + ) + ), + ) + + prediction = session.predict(note_doc, "input:client-watch:move", {"body": "predicted"}) + put = document_put_raw( + message_id="authoritative-watch", + key=prediction.key, + schema_id=prediction.schema_id, + stored_at="2026-06-13T00:00:00Z", + payload=note_doc.encode_payload({"body": "authoritative"}), + shard_id="inputs", + shard_epoch=1, + ) + session_changes = session.apply_shard_log_response({ + "schemaVersion": "cultnet.shard_log_response.v0", + "messageId": "inputs-watch-log", + "shardId": "inputs", + "shardEpoch": 1, + "entries": [ + { + "sequence": 1, + "committedAt": "2026-06-13T00:00:00Z", + "changeKind": "updated", + "put": put.to_wire(), + } + ], + "resyncRequired": False, + }) + + self.assertEqual([change.change_kind for change in seen], ["added", "updated"]) + self.assertEqual(seen[1].value, {"body": "authoritative"}) + self.assertEqual(seen[1].previous_value, {"body": "predicted"}) + self.assertEqual(session_changes[0].change_kind, "reconciled") + + def test_cultmesh_verse_catalog_response_matches_schema_v0_wire_shape(self) -> None: + import msgpack # type: ignore + + catalog = CultMeshVerseCatalog() + catalog.upsert( + CultMeshVerseDescriptor( + verse_id="aetheria-main", + display_name="Aetheria", + authority_model="OperatorCluster", + compatibility=CultMeshVerseCompatibility( + transport_version="cultmesh.v0", + rules_hash="rules", + compatible_verse_ids=("aetheria-modded",), + required_plugin_ids=("core",), + optional_plugin_ids=("skylands",), + ), + discovery_endpoints=("cultmesh://aetheria.example.test:3075",), + authority_runtime_ids=("runtime-a",), + description="main branch", + ) + ) + + response = catalog.create_response(verse_catalog_request("verses-1")) + decoded = msgpack.unpackb(msgpack.packb(response, use_bin_type=True), raw=False) + self.assertEqual(decoded["schemaVersion"], "cultmesh.verse_catalog_response.v0") + self.assertEqual(decoded["messageId"], "verses-1") + self.assertEqual(decoded["verses"][0]["verseId"], "aetheria-main") + self.assertEqual(decoded["verses"][0]["compatibility"]["requiredPluginIds"], ["core"]) + + def test_cultmesh_verse_catalog_watches_updates_and_finds_transfer_targets(self) -> None: + source = CultMeshVerseDescriptor( + verse_id="aetheria-main", + display_name="Aetheria", + authority_model="federated", + compatibility=CultMeshVerseCompatibility( + transport_version="cultmesh.v0", + rules_hash="rules-main", + ), + ) + compatible = CultMeshVerseDescriptor( + verse_id="aetheria-modded", + display_name="Aetheria Modded", + authority_model="federated", + compatibility=CultMeshVerseCompatibility( + transport_version="cultmesh.v0", + rules_hash="rules-modded", + compatible_verse_ids=("aetheria-main",), + ), + ) + incompatible = CultMeshVerseDescriptor( + verse_id="old-world", + display_name="Old World", + authority_model="solo", + compatibility=CultMeshVerseCompatibility( + transport_version="cultmesh.v0", + rules_hash="rules-old", + ), + ) + catalog = CultMeshVerseCatalog() + seen: list[str] = [] + unsubscribe = catalog.watch(lambda verse: seen.append(verse.verse_id)) + + catalog.upsert(source) + catalog.apply_response({ + "schemaVersion": "cultmesh.verse_catalog_response.v0", + "messageId": "verses-watch", + "verses": [compatible.to_wire(), incompatible.to_wire()], + }) + unsubscribe() + catalog.upsert(CultMeshVerseDescriptor( + verse_id="after-unsubscribe", + display_name="After", + authority_model="none", + compatibility=CultMeshVerseCompatibility("cultmesh.v0", "rules-after"), + )) + + self.assertEqual(seen, ["aetheria-main", "aetheria-modded", "old-world"]) + self.assertEqual([verse.verse_id for verse in catalog.verses], ["aetheria-main", "aetheria-modded", "after-unsubscribe", "old-world"]) + self.assertEqual(catalog.get("aetheria-modded"), compatible) + self.assertEqual([verse.verse_id for verse in catalog.find_transfer_targets(source)], ["aetheria-modded"]) + + def test_cultmesh_peer_exchange_response_matches_schema_v0_wire_shape(self) -> None: + import msgpack # type: ignore + + catalog = CultMeshPeerCatalog() + catalog.upsert( + CultMeshPeerCard( + peer_id="peer-a", + verse_id="aetheria-main", + endpoints=("cultnet://peer-a.example.test:3075",), + roles=("discovery", "read-replica"), + shard_ids=("players",), + region="eu-west", + authority_lease_id="lease-1", + expires_at="2026-05-20T12:00:00.0000000Z", + signature="sig", + ) + ) + + response = catalog.create_response( + peer_exchange_request("pex-1", verse_id="aetheria-main", roles=["read-replica"]) + ) + decoded = msgpack.unpackb(msgpack.packb(response, use_bin_type=True), raw=False) + self.assertEqual(decoded["schemaVersion"], "cultmesh.peer_exchange_response.v0") + self.assertEqual(decoded["messageId"], "pex-1") + self.assertEqual(decoded["peers"][0]["peerId"], "peer-a") + self.assertIn("read-replica", decoded["peers"][0]["roles"]) + self.assertEqual(decoded["peers"][0]["authorityLeaseId"], "lease-1") + + def test_cultmesh_peer_catalog_watches_updates_and_gets_peers(self) -> None: + first = CultMeshPeerCard( + peer_id="peer-a", + verse_id="aetheria-main", + endpoints=("cultnet://peer-a.example.test:3075",), + roles=("discovery", "read-replica"), + ) + second = CultMeshPeerCard( + peer_id="peer-b", + verse_id="aetheria-main", + endpoints=("cultnet://peer-b.example.test:3075",), + roles=("shard-primary",), + ) + catalog = CultMeshPeerCatalog() + seen: list[str] = [] + unsubscribe = catalog.watch(lambda peer: seen.append(peer.peer_id)) + + catalog.upsert(first) + catalog.apply_response({ + "schemaVersion": "cultmesh.peer_exchange_response.v0", + "messageId": "peers-watch", + "peers": [second.to_wire()], + }) + unsubscribe() + catalog.upsert(CultMeshPeerCard( + peer_id="peer-c", + verse_id="aetheria-main", + endpoints=("cultnet://peer-c.example.test:3075",), + roles=("read-replica",), + )) + + self.assertEqual(seen, ["peer-a", "peer-b"]) + self.assertTrue(first.has_role("read-replica")) + self.assertEqual([peer.peer_id for peer in catalog.peers], ["peer-a", "peer-b", "peer-c"]) + self.assertEqual(catalog.get("peer-b"), second) + self.assertEqual([peer.peer_id for peer in catalog.find("aetheria-main", role="read-replica")], ["peer-a", "peer-c"]) + + def test_cultmesh_discovery_client_fetches_typed_catalogs_over_cultnet_frames(self) -> None: + import msgpack # type: ignore + from cultnet_py import read_frame, write_frame + + verses = CultMeshVerseCatalog() + verses.upsert( + CultMeshVerseDescriptor( + verse_id="aetheria-main", + display_name="Aetheria", + authority_model="federated", + compatibility=CultMeshVerseCompatibility( + transport_version="cultmesh.v0", + rules_hash="rules-1", + required_plugin_ids=("core",), + ), + discovery_endpoints=("cultmesh://aetheria.example.test:3075",), + authority_runtime_ids=("runtime-a",), + ) + ) + peers = CultMeshPeerCatalog() + peers.upsert( + CultMeshPeerCard( + peer_id="peer-a", + verse_id="aetheria-main", + endpoints=("cultnet://peer-a.example.test:3075",), + roles=("read-replica",), + shard_ids=("players",), + ) + ) + + ready = threading.Event() + server_error: list[BaseException] = [] + + def serve_requests() -> None: + try: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as server: + server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + server.bind(("127.0.0.1", 0)) + port_holder.append(server.getsockname()[1]) + server.listen(8) + ready.set() + for _ in range(8): + connection, _ = server.accept() + with connection: + stream = connection.makefile("rwb") + request = msgpack.unpackb(read_frame(stream), raw=False) + if request["schemaVersion"] == "cultmesh.verse_catalog_request.v0": + response = verses.create_response(request) + elif request["schemaVersion"] == "cultmesh.peer_exchange_request.v0": + response = peers.create_response(request) + else: + raise AssertionError(f"unexpected request {request['schemaVersion']}") + write_frame(stream, msgpack.packb(response, use_bin_type=True)) + stream.flush() + except BaseException as error: + server_error.append(error) + ready.set() + + port_holder: list[int] = [] + thread = threading.Thread(target=serve_requests, daemon=True) + thread.start() + self.assertTrue(ready.wait(2.0)) + self.assertFalse(server_error) + + client = CultMeshDiscoveryClient("127.0.0.1", port_holder[0], timeout_seconds=2.0) + fetched_verses = client.fetch_verses(transport_version="cultmesh.v0") + fetched_peers = client.fetch_peers(verse_id="aetheria-main", roles=["read-replica"]) + local_verses = CultMeshVerseCatalog() + local_peers = CultMeshPeerCatalog() + synced_verses = client.sync_verse_catalog(local_verses, transport_version="cultmesh.v0") + synced_peers = client.sync_peer_catalog(local_peers, verse_id="aetheria-main", roles=["read-replica"]) + discovered_verses = CultMeshVerseCatalog() + discovered_peers = CultMeshPeerCatalog() + endpoint = f"cultnet://127.0.0.1:{port_holder[0]}" + verse_discovery = CultMeshVerseDiscoveryClient.from_endpoint(endpoint, timeout_seconds=2.0) + peer_discovery = CultMeshPeerExchangeClient.from_endpoint(endpoint, timeout_seconds=2.0) + discovered_verse_count = verse_discovery.discover( + discovered_verses, + endpoints=[endpoint, endpoint, ""], + transport_version="cultmesh.v0", + ) + discovered_peer_count = peer_discovery.discover( + discovered_peers, + verse_id="aetheria-main", + endpoints=[endpoint, endpoint, ""], + roles=["read-replica"], + ) + fanout_verses = CultMeshVerseCatalog() + fanout_verses.upsert( + CultMeshVerseDescriptor( + verse_id="aetheria-main", + display_name="Known Aetheria", + authority_model="federated", + compatibility=CultMeshVerseCompatibility("cultmesh.v0", "rules-1"), + discovery_endpoints=(endpoint, endpoint, ""), + ) + ) + fanout_peers = CultMeshPeerCatalog() + fanout_peers.upsert( + CultMeshPeerCard( + peer_id="known-peer", + verse_id="aetheria-main", + endpoints=(endpoint, endpoint, ""), + roles=("read-replica",), + ) + ) + fanout_verse_count = verse_discovery.fanout(fanout_verses, transport_version="cultmesh.v0") + fanout_peer_count = peer_discovery.fanout( + fanout_peers, + verse_id="aetheria-main", + roles=["read-replica"], + ) + + thread.join(2.0) + self.assertFalse(server_error) + self.assertEqual(fetched_verses[0].verse_id, "aetheria-main") + self.assertEqual(fetched_verses[0].compatibility.required_plugin_ids, ("core",)) + self.assertEqual(fetched_peers[0].peer_id, "peer-a") + self.assertEqual(fetched_peers[0].shard_ids, ("players",)) + self.assertEqual(synced_verses[0].verse_id, "aetheria-main") + self.assertEqual(local_peers.find("aetheria-main", role="read-replica")[0].peer_id, "peer-a") + self.assertEqual(synced_peers[0].roles, ("read-replica",)) + self.assertEqual(discovered_verse_count, 1) + self.assertEqual(discovered_peer_count, 1) + self.assertEqual(discovered_verses.get("aetheria-main").display_name, "Aetheria") + self.assertEqual(discovered_peers.find("aetheria-main", role="read-replica")[0].peer_id, "peer-a") + self.assertEqual(fanout_verse_count, 1) + self.assertEqual(fanout_peer_count, 1) + self.assertEqual(fanout_verses.get("aetheria-main").display_name, "Aetheria") + self.assertIn( + "peer-a", + {peer.peer_id for peer in fanout_peers.find("aetheria-main", role="read-replica")}, + ) + + def test_cultmesh_discovery_client_fans_out_simulation_observations_to_peers(self) -> None: + first_hub = CultNetSimulationObservationHub( + CultNetSimulationConsensusOptions(minimum_witnesses=1, quorum_ratio=1.0) + ) + second_hub = CultNetSimulationObservationHub( + CultNetSimulationConsensusOptions(minimum_witnesses=1, quorum_ratio=1.0) + ) + first_server = CultMesh.serve_node( + CultMesh.create_node(runtime_id="sim-peer-a"), + observation_hub=first_hub, + ) + second_server = CultMesh.serve_node( + CultMesh.create_node(runtime_id="sim-peer-b"), + observation_hub=second_hub, + ) + try: + peers = CultMeshPeerCatalog() + peers.upsert(CultMeshPeerCard( + peer_id="sim-peer-a", + verse_id="arena", + endpoints=(f"cultnet://127.0.0.1:{first_server.port}",), + roles=("simulation-witness",), + )) + peers.upsert(CultMeshPeerCard( + peer_id="sim-peer-b", + verse_id="arena", + endpoints=(f"cultnet://127.0.0.1:{second_server.port}",), + roles=("simulation-witness",), + )) + claim_hash = compute_simulation_claim_hash("hit", "alice", "bob", "frame:200") + message = simulation_observation( + message_id="fanout-observation", + witness_runtime_id="python-fanout", + shard_id="arena", + shard_epoch=1, + frame=200, + subject_id="bob", + claim_kind="hit", + claim_hash=claim_hash, + claim_summary="alice hit bob", + ).to_wire() + + client = CultMeshDiscoveryClient("127.0.0.1", first_server.port, timeout_seconds=2.0) + candidates = client.fanout_simulation_observation( + peers, + message, + verse_id="arena", + roles=["simulation-witness"], + ) + finally: + first_server.stop() + second_server.stop() + + self.assertEqual([candidate["schemaVersion"] for candidate in candidates], [ + "cultnet.simulation_consensus_candidate.v0", + "cultnet.simulation_consensus_candidate.v0", + ]) + self.assertEqual({candidate["claimHash"] for candidate in candidates}, {claim_hash}) + self.assertEqual({candidate["messageId"] for candidate in candidates}, {"fanout-observation"}) + self.assertEqual({candidate["hasQuorum"] for candidate in candidates}, {True}) + + def test_cultmesh_simulation_observation_fanout_flushes_queued_messages(self) -> None: + hub = CultNetSimulationObservationHub( + CultNetSimulationConsensusOptions(minimum_witnesses=1, quorum_ratio=1.0) + ) + server = CultMesh.serve_node(CultMesh.create_node(runtime_id="sim-flush-peer"), observation_hub=hub) + try: + peers = CultMeshPeerCatalog() + peers.upsert(CultMeshPeerCard( + peer_id="sim-flush-peer", + verse_id="arena", + endpoints=(f"cultnet://127.0.0.1:{server.port}",), + roles=("simulation-witness",), + )) + claim_hash = compute_simulation_claim_hash("hit", "alice", "bob", "frame:210") + seen_candidates: list[dict[str, object]] = [] + fanout = CultMeshSimulationObservationFanout( + CultMeshDiscoveryClient("127.0.0.1", server.port, timeout_seconds=2.0), + peers, + verse_id="arena", + roles=["simulation-witness"], + on_candidate=seen_candidates.append, + ) + fanout.enqueue(simulation_observation( + message_id="queued-observation", + witness_runtime_id="python-fanout", + shard_id="arena", + shard_epoch=1, + frame=210, + subject_id="bob", + claim_kind="hit", + claim_hash=claim_hash, + claim_summary="alice hit bob", + ).to_wire()) + + candidates = fanout.flush() + finally: + server.stop() + + self.assertEqual(fanout.pending_count(), 0) + self.assertEqual([candidate["claimHash"] for candidate in candidates], [claim_hash]) + self.assertEqual(seen_candidates[0]["messageId"], "queued-observation") + + def test_cultmesh_simulation_observation_fanout_keeps_failed_messages_pending(self) -> None: + closed_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + closed_socket.bind(("127.0.0.1", 0)) + closed_port = closed_socket.getsockname()[1] + closed_socket.close() + peers = CultMeshPeerCatalog() + peers.upsert(CultMeshPeerCard( + peer_id="closed-simulation-peer", + verse_id="arena", + endpoints=(f"cultnet://127.0.0.1:{closed_port}",), + roles=("simulation-witness",), + )) + errors: list[tuple[str, str]] = [] + fanout = CultMeshSimulationObservationFanout( + CultMeshDiscoveryClient("127.0.0.1", closed_port, timeout_seconds=0.2), + peers, + verse_id="arena", + roles=["simulation-witness"], + on_error=lambda endpoint, error: errors.append((endpoint, str(error))), + ) + fanout.enqueue(simulation_observation( + message_id="retry-observation", + witness_runtime_id="python-fanout", + shard_id="arena", + shard_epoch=1, + frame=212, + subject_id="bob", + claim_kind="miss", + claim_hash=compute_simulation_claim_hash("miss", "alice", "bob", "frame:212"), + claim_summary="alice missed bob", + ).to_wire()) + + candidates = fanout.flush() + + self.assertEqual(candidates, []) + self.assertEqual(fanout.pending_count(), 1) + self.assertEqual(len(errors), 1) + self.assertIn(f"127.0.0.1:{closed_port}", errors[0][0]) + + def test_cultmesh_simulation_observation_fanout_runs_background_loop(self) -> None: + hub = CultNetSimulationObservationHub( + CultNetSimulationConsensusOptions(minimum_witnesses=1, quorum_ratio=1.0) + ) + server = CultMesh.serve_node(CultMesh.create_node(runtime_id="sim-loop-peer"), observation_hub=hub) + try: + peers = CultMeshPeerCatalog() + peers.upsert(CultMeshPeerCard( + peer_id="sim-loop-peer", + verse_id="arena", + endpoints=(f"cultnet://127.0.0.1:{server.port}",), + roles=("simulation-witness",), + )) + claim_hash = compute_simulation_claim_hash("block", "dana", "eve", "frame:211") + seen_candidates: list[dict[str, object]] = [] + fanout = CultMeshSimulationObservationFanout( + CultMeshDiscoveryClient("127.0.0.1", server.port, timeout_seconds=2.0), + peers, + verse_id="arena", + roles=["simulation-witness"], + interval_seconds=0.05, + on_candidate=seen_candidates.append, + ) + fanout.start() + fanout.enqueue(simulation_observation( + message_id="loop-observation", + witness_runtime_id="python-fanout", + shard_id="arena", + shard_epoch=1, + frame=211, + subject_id="eve", + claim_kind="block", + claim_hash=claim_hash, + claim_summary="dana blocked eve", + ).to_wire()) + deadline = time.monotonic() + 2.0 + while not seen_candidates and time.monotonic() < deadline: + time.sleep(0.01) + fanout.stop() + finally: + server.stop() + + self.assertEqual(fanout.pending_count(), 0) + self.assertTrue(seen_candidates) + self.assertEqual(seen_candidates[0]["claimHash"], claim_hash) + self.assertEqual(seen_candidates[0]["messageId"], "loop-observation") + + def test_cultmesh_discovery_client_fans_out_snapshots_to_peers(self) -> None: + document = define_database_entry_type( + "mesh.snapshot_fanout_note", + [("body", 0)], + schema_id="mesh.snapshot_fanout_note.v1", + ) + first = CultMesh.create_node(runtime_id="snapshot-peer-a") + first.database.register_document(document) + first.database.put_raw_message(document, "note:a", {"body": "from-a"}, shard_id="notes", shard_epoch=1) + second = CultMesh.create_node(runtime_id="snapshot-peer-b") + second.database.register_document(document) + second.database.put_raw_message(document, "note:b", {"body": "from-b"}, shard_id="notes", shard_epoch=1) + target = CultMesh.create_node(runtime_id="snapshot-target") + target.database.register_document(document) + first_server = CultMesh.serve_node(first) + second_server = CultMesh.serve_node(second) + try: + peers = CultMeshPeerCatalog() + peers.upsert(CultMeshPeerCard( + peer_id="snapshot-peer-a", + verse_id="mesh", + endpoints=(f"cultnet://127.0.0.1:{first_server.port}", f"cultnet://127.0.0.1:{first_server.port}"), + roles=("read-replica",), + )) + peers.upsert(CultMeshPeerCard( + peer_id="snapshot-peer-b", + verse_id="mesh", + endpoints=(f"cultnet://127.0.0.1:{second_server.port}",), + roles=("read-replica",), + )) + + client = CultMeshDiscoveryClient("127.0.0.1", first_server.port, timeout_seconds=2.0) + responses = client.fanout_snapshot_responses( + peers, + verse_id="mesh", + roles=["read-replica"], + schema_ids=["mesh.snapshot_fanout_note.v1"], + ) + applied = client.sync_snapshots( + target.database, + peers, + verse_id="mesh", + roles=["read-replica"], + schema_ids=["mesh.snapshot_fanout_note.v1"], + ) + finally: + first_server.stop() + second_server.stop() + + self.assertEqual(len(responses), 2) + self.assertTrue(all(isinstance(response, CultNetRawSnapshotResponse) for response in responses)) + self.assertEqual( + sorted(record.record_key for response in responses for record in response.documents), + ["note:a", "note:b"], + ) + self.assertEqual(sorted(record.record_key for record in applied), ["note:a", "note:b"]) + self.assertEqual(target.database.get_required(document, "note:a")["body"], "from-a") + self.assertEqual(target.database.get_required(document, "note:b")["body"], "from-b") + + def test_cultmesh_discovery_client_sync_documents_returns_requested_aliases(self) -> None: + @dataclass + class CanonicalNote: + body: str + + @dataclass + class UiNote: + body: str + + document = define_database_entry_type( + "mesh.snapshot_fanout_alias_note", + [("body", 0)], + cls=CanonicalNote, + schema_id="mesh.snapshot_fanout_alias_note.v1", + schema_name="mesh.snapshot_fanout_alias_note", + schema_version="mesh.snapshot_fanout_alias_note.v1", + ) + alias = define_database_entry_type( + "mesh.snapshot_fanout_alias_note.ui", + [("body", 0)], + cls=UiNote, + schema_id="mesh.snapshot_fanout_alias_note.v1", + schema_name="mesh.snapshot_fanout_alias_note", + schema_version="mesh.snapshot_fanout_alias_note.v1", + ) + source = CultMesh.create_node(runtime_id="snapshot-alias-source") + source.database.register_document(document) + source.database.put_raw_message(document, "note:alias", CanonicalNote("from-source")) + target = CultMesh.create_node(runtime_id="snapshot-alias-target") + target.database.register_document(document) + server = CultMesh.serve_node(source) + try: + peers = CultMeshPeerCatalog() + peers.upsert(CultMeshPeerCard( + peer_id="snapshot-alias-source", + verse_id="mesh", + endpoints=(f"cultnet://127.0.0.1:{server.port}",), + roles=("read-replica",), + )) + + values = CultMeshDiscoveryClient("127.0.0.1", server.port, timeout_seconds=2.0).sync_documents( + target.database, + peers, + verse_id="mesh", + roles=["read-replica"], + documents=[(alias, "note:alias")], + ) + finally: + server.stop() + + self.assertEqual(len(values), 1) + self.assertIsInstance(values[0], UiNote) + self.assertEqual(values[0].body, "from-source") + self.assertEqual(target.database.get_required(document, "note:alias").body, "from-source") + self.assertIsInstance(target.database.get_required(alias, "note:alias"), UiNote) + + def test_cultmesh_document_subscription_syncs_alias_snapshot_and_changes(self) -> None: + @dataclass + class CanonicalNote: + body: str + + @dataclass + class UiNote: + body: str + + document = define_database_entry_type( + "mesh.subscription_alias_note", + [("body", 0)], + cls=CanonicalNote, + schema_id="mesh.subscription_alias_note.v1", + schema_name="mesh.subscription_alias_note", + schema_version="mesh.subscription_alias_note.v1", + ) + alias = define_database_entry_type( + "mesh.subscription_alias_note.ui", + [("body", 0)], + cls=UiNote, + schema_id="mesh.subscription_alias_note.v1", + schema_name="mesh.subscription_alias_note", + schema_version="mesh.subscription_alias_note.v1", + ) + source = CultMesh.create_node(runtime_id="subscription-alias-source") + source.database.register_document(document) + source.database.put_raw_message(document, "note:live", CanonicalNote("initial")) + target = CultMesh.create_node(runtime_id="subscription-alias-target") + target.database.register_document(document) + server = CultMesh.serve_node(source) + try: + raw_client = CultMesh.create_client("127.0.0.1", server.port, timeout_seconds=2.0) + with CultMesh.subscribe_document(target, raw_client, alias, "note:live") as subscription: + initial = subscription.sync_initial() + subscription.send(document_put_raw( + message_id="subscription-alias-put", + key="note:live", + schema_id="mesh.subscription_alias_note.v1", + stored_at="2026-06-14T00:00:00Z", + payload=document.encode_payload(CanonicalNote("updated")), + )) + updated = subscription.read_next_change() + subscription.send(document_delete( + message_id="subscription-alias-delete", + schema_id="mesh.subscription_alias_note.v1", + record_key="note:live", + )) + removed = subscription.read_next_change() + finally: + server.stop() + + self.assertIsInstance(initial, UiNote) + self.assertEqual(initial.body, "initial") + self.assertEqual(updated.change_kind, "updated") + self.assertIsInstance(updated.value, UiNote) + self.assertEqual(updated.value.body, "updated") + self.assertIsInstance(updated.previous_value, UiNote) + self.assertEqual(updated.previous_value.body, "initial") + self.assertEqual(removed.change_kind, "removed") + self.assertIsNone(removed.value) + self.assertIsInstance(removed.previous_value, UiNote) + self.assertEqual(removed.previous_value.body, "updated") + self.assertIsNone(target.database.get(alias, "note:live")) + + def test_cultmesh_snapshot_fanout_syncs_once_and_reports_applied_records(self) -> None: + document = define_database_entry_type( + "mesh.snapshot_loop_note", + [("body", 0)], + schema_id="mesh.snapshot_loop_note.v1", + ) + source = CultMesh.create_node(runtime_id="snapshot-loop-source") + source.database.register_document(document) + source.database.put_raw_message(document, "note:sync-once", {"body": "from-source"}) + target = CultMesh.create_node(runtime_id="snapshot-loop-target") + target.database.register_document(document) + server = CultMesh.serve_node(source) + try: + peers = CultMeshPeerCatalog() + peers.upsert(CultMeshPeerCard( + peer_id="snapshot-loop-source", + verse_id="mesh", + endpoints=(f"cultnet://127.0.0.1:{server.port}",), + roles=("read-replica",), + )) + seen: list[str] = [] + fanout = CultMeshSnapshotFanout( + CultMeshDiscoveryClient("127.0.0.1", server.port, timeout_seconds=2.0), + target.database, + peers, + verse_id="mesh", + roles=["read-replica"], + schema_ids=["mesh.snapshot_loop_note.v1"], + on_applied=lambda record: seen.append(record.record_key), + ) + + applied = fanout.sync_once() + finally: + server.stop() + + self.assertEqual([record.record_key for record in applied], ["note:sync-once"]) + self.assertEqual(seen, ["note:sync-once"]) + self.assertEqual(target.database.get_required(document, "note:sync-once")["body"], "from-source") + + def test_cultmesh_snapshot_fanout_syncs_typed_documents_and_reports_alias_values(self) -> None: + @dataclass + class CanonicalNote: + body: str + + @dataclass + class UiNote: + body: str + + document = define_database_entry_type( + "mesh.snapshot_loop_alias_note", + [("body", 0)], + cls=CanonicalNote, + schema_id="mesh.snapshot_loop_alias_note.v1", + schema_name="mesh.snapshot_loop_alias_note", + schema_version="mesh.snapshot_loop_alias_note.v1", + ) + alias = define_database_entry_type( + "mesh.snapshot_loop_alias_note.ui", + [("body", 0)], + cls=UiNote, + schema_id="mesh.snapshot_loop_alias_note.v1", + schema_name="mesh.snapshot_loop_alias_note", + schema_version="mesh.snapshot_loop_alias_note.v1", + ) + source = CultMesh.create_node(runtime_id="snapshot-loop-alias-source") + source.database.register_document(document) + source.database.put_raw_message(document, "note:typed-loop", CanonicalNote("from-source")) + target = CultMesh.create_node(runtime_id="snapshot-loop-alias-target") + target.database.register_document(document) + server = CultMesh.serve_node(source) + try: + peers = CultMeshPeerCatalog() + peers.upsert(CultMeshPeerCard( + peer_id="snapshot-loop-alias-source", + verse_id="mesh", + endpoints=(f"cultnet://127.0.0.1:{server.port}",), + roles=("read-replica",), + )) + seen: list[UiNote] = [] + fanout = CultMeshSnapshotFanout( + CultMeshDiscoveryClient("127.0.0.1", server.port, timeout_seconds=2.0), + target.database, + peers, + verse_id="mesh", + roles=["read-replica"], + documents=[(alias, "note:typed-loop")], + on_document=seen.append, + ) + + values = fanout.sync_once() + finally: + server.stop() + + self.assertEqual(len(values), 1) + self.assertIsInstance(values[0], UiNote) + self.assertEqual(values[0].body, "from-source") + self.assertEqual(seen, [values[0]]) + self.assertEqual(target.database.get_required(document, "note:typed-loop").body, "from-source") + self.assertIsInstance(target.database.get_required(alias, "note:typed-loop"), UiNote) + + def test_cultmesh_snapshot_fanout_runs_background_loop(self) -> None: + document = define_database_entry_type( + "mesh.snapshot_background_note", + [("body", 0)], + schema_id="mesh.snapshot_background_note.v1", + ) + source = CultMesh.create_node(runtime_id="snapshot-background-source") + source.database.register_document(document) + source.database.put_raw_message(document, "note:background", {"body": "from-background"}) + target = CultMesh.create_node(runtime_id="snapshot-background-target") + target.database.register_document(document) + server = CultMesh.serve_node(source) + try: + peers = CultMeshPeerCatalog() + peers.upsert(CultMeshPeerCard( + peer_id="snapshot-background-source", + verse_id="mesh", + endpoints=(f"cultnet://127.0.0.1:{server.port}",), + roles=("read-replica",), + )) + seen: list[str] = [] + fanout = CultMeshSnapshotFanout( + CultMeshDiscoveryClient("127.0.0.1", server.port, timeout_seconds=2.0), + target.database, + peers, + verse_id="mesh", + roles=["read-replica"], + schema_ids=["mesh.snapshot_background_note.v1"], + interval_seconds=0.05, + on_applied=lambda record: seen.append(record.record_key), + ) + fanout.start() + deadline = time.monotonic() + 2.0 + while not seen and time.monotonic() < deadline: + time.sleep(0.01) + fanout.stop() + finally: + server.stop() + + self.assertTrue(seen) + self.assertEqual(target.database.get_required(document, "note:background")["body"], "from-background") + + def test_cultmesh_peer_health_monitor_probes_reachable_and_failed_endpoints(self) -> None: + document = define_database_entry_type( + "mesh.health_note", + [("body", 0)], + schema_id="mesh.health_note.v1", + ) + node = CultMesh.create_node(runtime_id="health-peer") + node.database.register_document(document) + server = CultMesh.serve_node(node) + closed_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + closed_socket.bind(("127.0.0.1", 0)) + closed_port = closed_socket.getsockname()[1] + closed_socket.close() + try: + peers = CultMeshPeerCatalog() + peers.upsert(CultMeshPeerCard( + peer_id="health-peer", + verse_id="mesh", + endpoints=(f"cultnet://127.0.0.1:{server.port}",), + roles=("read-replica",), + )) + peers.upsert(CultMeshPeerCard( + peer_id="closed-peer", + verse_id="mesh", + endpoints=(f"cultnet://127.0.0.1:{closed_port}",), + roles=("read-replica",), + )) + seen: list[tuple[str, bool]] = [] + monitor = CultMeshPeerHealthMonitor( + runtime_id="health-prober", + timeout_seconds=0.2, + on_update=lambda health: seen.append((health.peer_id, health.is_reachable)), + ) + + results = monitor.probe_catalog(peers, verse_id="mesh", roles=["read-replica"]) + finally: + server.stop() + + by_peer = {result.peer_id: result for result in results} + self.assertTrue(by_peer["health-peer"].is_reachable) + self.assertEqual(by_peer["health-peer"].runtime_id, "health-peer") + self.assertEqual(by_peer["health-peer"].runtime_kind, "python") + self.assertEqual(by_peer["health-peer"].display_name, "health-peer") + self.assertEqual(by_peer["health-peer"].supported_document_types, ("mesh.health_note",)) + self.assertIn("cultnet.hello.v0", by_peer["health-peer"].supported_message_versions) + self.assertIn("cultnet.error.v0", by_peer["health-peer"].supported_message_versions) + self.assertEqual(by_peer["health-peer"].supported_mutation_contracts[0]["documentType"], "mesh.health_note") + self.assertIn("shardLog", by_peer["health-peer"].supported_mutation_contracts[0]["operations"]) + self.assertFalse(by_peer["closed-peer"].is_reachable) + self.assertIsNone(by_peer["closed-peer"].runtime_kind) + self.assertEqual(by_peer["closed-peer"].supported_document_types, ()) + self.assertEqual(by_peer["closed-peer"].supported_message_versions, ()) + self.assertEqual(by_peer["closed-peer"].supported_mutation_contracts, ()) + self.assertTrue(by_peer["closed-peer"].error) + self.assertEqual({peer_id for peer_id, _ in seen}, {"health-peer", "closed-peer"}) + self.assertEqual(len(monitor.latest()), 2) + + def test_cultmesh_peer_health_monitor_runs_background_probe_loop(self) -> None: + server = CultMesh.serve_node(CultMesh.create_node(runtime_id="health-loop-peer")) + try: + peers = CultMeshPeerCatalog() + peers.upsert(CultMeshPeerCard( + peer_id="health-loop-peer", + verse_id="mesh", + endpoints=(f"cultnet://127.0.0.1:{server.port}",), + roles=("read-replica",), + )) + seen: list[str] = [] + monitor = CultMeshPeerHealthMonitor( + runtime_id="health-loop-prober", + timeout_seconds=0.2, + interval_seconds=0.05, + on_update=lambda health: seen.append(health.peer_id), + ) + monitor.start(peers, verse_id="mesh", roles=["read-replica"]) + deadline = time.monotonic() + 2.0 + while time.monotonic() < deadline: + latest = monitor.latest() + if latest and latest[0].is_reachable: + break + time.sleep(0.01) + monitor.stop() + finally: + server.stop() + + self.assertTrue(seen) + latest = monitor.latest() + self.assertEqual(latest[0].peer_id, "health-loop-peer") + self.assertTrue(latest[0].is_reachable) + + def test_cultmesh_daemon_entrypoint_serves_framed_cultnet_hello(self) -> None: + with tempfile.TemporaryDirectory() as temp: + ready_path = Path(temp) / "ready.json" + package_src = Path(__file__).resolve().parents[1] / "src" + env = dict(os.environ) + existing_pythonpath = env.get("PYTHONPATH") + env["PYTHONPATH"] = ( + str(package_src) + if not existing_pythonpath + else f"{package_src}{os.pathsep}{existing_pythonpath}" + ) + process = subprocess.Popen( + [ + sys.executable, + "-m", + "cultmesh_py.daemon", + "--runtime-id", + "daemon-test-peer", + "--display-name", + "Daemon Test Peer", + "--port", + "0", + "--ready-file", + str(ready_path), + "--seed-interop-note", + "--seed-shard-id", + "daemon-interop", + "--verse-id", + "daemon-verse", + "--verse-display-name", + "Daemon Verse", + "--role", + "shard-primary", + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + env=env, + ) + try: + deadline = time.monotonic() + 5.0 + while not ready_path.exists() and time.monotonic() < deadline: + if process.poll() is not None: + break + time.sleep(0.05) + if not ready_path.exists(): + _, stderr = process.communicate(timeout=2.0) + self.fail(f"cultmesh daemon did not publish readiness: {stderr}") + ready = json.loads(ready_path.read_text(encoding="utf-8")) + self.assertEqual(ready["schemaVersion"], READY_SCHEMA_VERSION) + self.assertEqual(ready["runtimeId"], "daemon-test-peer") + self.assertEqual(ready["runtimeKind"], "python") + self.assertEqual(ready["displayName"], "Daemon Test Peer") + self.assertEqual(ready["endpoint"], f"cultnet://127.0.0.1:{ready['port']}") + self.assertEqual(ready["shardIds"], ["daemon-interop"]) + self.assertEqual(ready["supportedDocumentTypes"], ["cultcache.interop-note"]) + self.assertIn("cultnet.hello.v0", ready["supportedMessageVersions"]) + self.assertIn("cultnet.document_put_raw.v0", ready["supportedMessageVersions"]) + self.assertNotIn("cultnet.simulation_observation.v0", ready["supportedMessageVersions"]) + self.assertEqual(ready["supportedMutationContracts"][0]["documentType"], "cultcache.interop-note") + self.assertIn("shardLog", ready["supportedMutationContracts"][0]["operations"]) + self.assertEqual(ready["snapshotLimits"], {"maxSnapshotBytes": None, "maxSnapshotDocuments": None}) + self.assertEqual(ready["verses"][0]["verseId"], "daemon-verse") + self.assertEqual(ready["verses"][0]["displayName"], "Daemon Verse") + self.assertEqual(ready["verses"][0]["discoveryEndpoints"], [ready["endpoint"]]) + self.assertEqual(ready["verses"][0]["authorityRuntimeIds"], ["daemon-test-peer"]) + self.assertEqual(ready["peers"][0]["peerId"], "daemon-test-peer") + self.assertEqual(ready["peers"][0]["endpoints"], [ready["endpoint"]]) + self.assertEqual(ready["peers"][0]["roles"], ["shard-primary"]) + self.assertEqual(ready["peers"][0]["shardIds"], ["daemon-interop"]) + + client = CultMesh.create_client("127.0.0.1", int(ready["port"]), timeout_seconds=2.0) + hello_response = client.request( + hello(runtime_id="daemon-test-prober"), + expected_schema_version="cultnet.hello.v0", + ) + + self.assertEqual(hello_response["runtimeId"], "daemon-test-peer") + self.assertEqual(hello_response["displayName"], "Daemon Test Peer") + self.assertIn("cultnet.hello.v0", hello_response["supportedMessageVersions"]) + self.assertIn("cultnet.schema_catalog_request.v0", hello_response["supportedMessageVersions"]) + self.assertNotIn("cultnet.simulation_observation.v0", hello_response["supportedMessageVersions"]) + self.assertEqual(hello_response["supportedDocumentTypes"], ["cultcache.interop-note"]) + self.assertEqual(hello_response["transportProfiles"][0]["transports"][0]["protocol"], "tcp_framed") + self.assertEqual(hello_response["transportProfiles"][0]["transports"][0]["port"], int(ready["port"])) + self.assertEqual(ready["supportedMessageVersions"], hello_response["supportedMessageVersions"]) + self.assertEqual(ready["supportedMutationContracts"], hello_response["supportedMutationContracts"]) + self.assertEqual(ready["transportProfiles"], hello_response["transportProfiles"]) + + schema_response = client.fetch_schema_catalog(schema_ids=["cultcache.interop-note"], include_schema_json=True) + self.assertEqual(schema_response["schemas"][0]["schemaVersion"], INTEROP_SCHEMA_VERSION) + self.assertEqual(schema_response["schemas"][0]["documentType"], "cultcache.interop-note") + + snapshot_response = client.fetch_snapshot_response(schema_ids=["cultcache.interop-note"]) + self.assertEqual(snapshot_response.documents[0].record_key, "note:daemon-test-peer") + note = interop_note_document.decode_payload(snapshot_response.documents[0].payload) + self.assertEqual(note["authorRuntimeId"], "daemon-test-peer") + self.assertIn("daemon", note["tags"]) + + shard_catalog = client.fetch_shard_catalog(schema_ids=["cultcache.interop-note"]) + self.assertEqual(shard_catalog["shards"][0]["shardId"], "daemon-interop") + self.assertEqual(shard_catalog["shards"][0]["schemaIds"], ["cultcache.interop-note"]) + + shard_log = client.fetch_shard_log_response(shard_id="daemon-interop", shard_epoch=1) + self.assertFalse(shard_log.resync_required) + self.assertEqual(shard_log.entries[0].sequence, 1) + self.assertEqual(shard_log.entries[0].change_kind, "added") + self.assertIsNotNone(shard_log.entries[0].raw_document) + assert shard_log.entries[0].raw_document is not None + self.assertEqual(shard_log.entries[0].raw_document.record_key, "note:daemon-test-peer") + + discovery_client = CultMesh.create_verse_discovery_client("127.0.0.1", int(ready["port"]), timeout_seconds=2.0) + verses = discovery_client.fetch_verses(transport_version="cultmesh.v0") + self.assertEqual(verses[0].verse_id, "daemon-verse") + self.assertEqual(verses[0].display_name, "Daemon Verse") + self.assertEqual(verses[0].discovery_endpoints, (ready["endpoint"],)) + self.assertEqual(verses[0].authority_runtime_ids, ("daemon-test-peer",)) + self.assertEqual([verse.to_wire() for verse in verses], ready["verses"]) + + peers = discovery_client.fetch_peers(verse_id="daemon-verse", roles=["shard-primary"]) + self.assertEqual(peers[0].peer_id, "daemon-test-peer") + self.assertEqual(peers[0].endpoints, (ready["endpoint"],)) + self.assertEqual(peers[0].roles, ("shard-primary",)) + self.assertEqual(peers[0].shard_ids, ("daemon-interop",)) + self.assertEqual([peer.to_wire() for peer in peers], ready["peers"]) + + monitor = CultMeshPeerHealthMonitor(runtime_id="daemon-health-prober", timeout_seconds=2.0) + health = monitor.probe_peer(peers[0])[0] + self.assertTrue(health.is_reachable) + self.assertEqual(health.runtime_id, "daemon-test-peer") + self.assertEqual(health.supported_document_types, ("cultcache.interop-note",)) + self.assertEqual(health.supported_mutation_contracts[0]["documentType"], "cultcache.interop-note") + finally: + process.terminate() + try: + process.communicate(timeout=5.0) + except subprocess.TimeoutExpired: + process.kill() + process.communicate(timeout=5.0) + + def test_cultmesh_daemon_restarts_with_persisted_cache_and_shard_log(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + cache_path = root / "mesh.cc" + shard_log_path = root / "shard-log" + env = dict(os.environ) + package_src = Path(__file__).resolve().parents[1] / "src" + existing_pythonpath = env.get("PYTHONPATH") + env["PYTHONPATH"] = ( + str(package_src) + if not existing_pythonpath + else f"{package_src}{os.pathsep}{existing_pythonpath}" + ) + + first_ready = root / "first-ready.json" + first = subprocess.Popen( + [ + sys.executable, + "-m", + "cultmesh_py.daemon", + "--runtime-id", + "durable-daemon-peer", + "--port", + "0", + "--cache-file", + str(cache_path), + "--enable-durable-shard-logs", + "--shard-log-file", + str(shard_log_path), + "--seed-interop-note", + "--seed-shard-id", + "durable-interop", + "--ready-file", + str(first_ready), + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + env=env, + ) + try: + self._wait_for_ready_file(first, first_ready) + finally: + self._terminate_process(first) + + second_ready = root / "second-ready.json" + second = subprocess.Popen( + [ + sys.executable, + "-m", + "cultmesh_py.daemon", + "--runtime-id", + "durable-daemon-peer", + "--port", + "0", + "--cache-file", + str(cache_path), + "--enable-durable-shard-logs", + "--shard-log-file", + str(shard_log_path), + "--register-interop-note", + "--verse-id", + "durable-verse", + "--role", + "shard-primary", + "--ready-file", + str(second_ready), + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + env=env, + ) + try: + ready = self._wait_for_ready_file(second, second_ready) + client = CultMesh.create_client("127.0.0.1", int(ready["port"]), timeout_seconds=2.0) + + snapshot_response = client.fetch_snapshot_response(schema_ids=["cultcache.interop-note"]) + self.assertEqual(snapshot_response.documents[0].record_key, "note:durable-daemon-peer") + note = interop_note_document.decode_payload(snapshot_response.documents[0].payload) + self.assertEqual(note["authorRuntimeId"], "durable-daemon-peer") + + shard_log = client.fetch_shard_log_response(shard_id="durable-interop", shard_epoch=1) + self.assertFalse(shard_log.resync_required) + self.assertEqual(shard_log.entries[0].sequence, 1) + self.assertIsNotNone(shard_log.entries[0].raw_document) + assert shard_log.entries[0].raw_document is not None + self.assertEqual(shard_log.entries[0].raw_document.record_key, "note:durable-daemon-peer") + + discovery_client = CultMesh.create_verse_discovery_client("127.0.0.1", int(ready["port"]), timeout_seconds=2.0) + peers = discovery_client.fetch_peers(verse_id="durable-verse", roles=["shard-primary"]) + self.assertEqual(peers[0].shard_ids, ("durable-interop",)) + finally: + self._terminate_process(second) + + def test_cultmesh_daemon_serves_live_database_subscription_mutations(self) -> None: + with tempfile.TemporaryDirectory() as temp: + ready_path = Path(temp) / "ready.json" + package_src = Path(__file__).resolve().parents[1] / "src" + env = dict(os.environ) + existing_pythonpath = env.get("PYTHONPATH") + env["PYTHONPATH"] = ( + str(package_src) + if not existing_pythonpath + else f"{package_src}{os.pathsep}{existing_pythonpath}" + ) + process = subprocess.Popen( + [ + sys.executable, + "-m", + "cultmesh_py.daemon", + "--runtime-id", + "live-daemon-peer", + "--port", + "0", + "--seed-interop-note", + "--seed-shard-id", + "live-interop", + "--ready-file", + str(ready_path), + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + env=env, + ) + try: + ready = self._wait_for_ready_file(process, ready_path) + client = CultMesh.create_client("127.0.0.1", int(ready["port"]), timeout_seconds=2.0) + live_note = { + "schemaVersion": INTEROP_SCHEMA_VERSION, + "documentId": "note:daemon-live", + "authorRuntimeId": "daemon-live-client", + "title": "daemon live subscription note", + "body": "live subscription put from Python daemon smoke", + "tags": ["daemon", "subscription", "interop"], + } + + with client.subscribe_database( + subscription_id="daemon-live-sub", + schema_ids=["cultcache.interop-note"], + ) as subscription: + snapshot = subscription.read_next_snapshot_response() + subscription.send(document_put_raw( + message_id="daemon-live-put", + key="note:daemon-live", + schema_id="cultcache.interop-note", + stored_at="2026-06-14T00:00:00Z", + payload=interop_note_document.encode_payload(live_note), + source_runtime_id="daemon-live-client", + shard_id="live-interop", + shard_epoch=1, + )) + change = subscription.read_next_change() + + self.assertEqual(snapshot.documents[0].record_key, "note:live-daemon-peer") + self.assertEqual(change.change_kind, "added") + self.assertEqual(change.record_key, "note:daemon-live") + self.assertIsNotNone(change.raw_document) + assert change.raw_document is not None + self.assertEqual( + interop_note_document.decode_payload(change.raw_document.payload), + live_note, + ) + + shard_log = client.fetch_shard_log_response(shard_id="live-interop", shard_epoch=1) + self.assertFalse(shard_log.resync_required) + self.assertEqual([entry.sequence for entry in shard_log.entries], [1, 2]) + self.assertEqual(shard_log.entries[1].change_kind, "added") + self.assertIsNotNone(shard_log.entries[1].raw_document) + assert shard_log.entries[1].raw_document is not None + self.assertEqual(shard_log.entries[1].raw_document.record_key, "note:daemon-live") + finally: + self._terminate_process(process) + + def test_cultmesh_daemon_serves_compacted_shard_log_resync_boundary(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + cache_path = root / "mesh.cc" + shard_log_path = root / "shard-log" + store = CultNetFileShardMutationLogStore(shard_log_path) + node = CultMesh.create_node( + cache_path, + runtime_id="compacted-daemon-peer", + enable_durable_shard_logs=True, + shard_log_path=shard_log_path, + ) + node.database.register_document(interop_note_document) + node.database.put_raw_message( + interop_note_document, + "note:compacted-one", + { + "schemaVersion": INTEROP_SCHEMA_VERSION, + "documentId": "note:compacted-one", + "authorRuntimeId": "compacted-daemon-peer", + "title": "compacted daemon note one", + "body": "first retained cache value", + "tags": ["daemon", "compaction"], + }, + message_id="compacted-put-1", + shard_id="compacted-interop", + shard_epoch=7, + ) + node.database.put_raw_message( + interop_note_document, + "note:compacted-two", + { + "schemaVersion": INTEROP_SCHEMA_VERSION, + "documentId": "note:compacted-two", + "authorRuntimeId": "compacted-daemon-peer", + "title": "compacted daemon note two", + "body": "second retained cache value", + "tags": ["daemon", "compaction"], + }, + message_id="compacted-put-2", + shard_id="compacted-interop", + shard_epoch=7, + ) + store.compact_through("compacted-interop", 1) + + ready_path = root / "ready.json" + package_src = Path(__file__).resolve().parents[1] / "src" + env = dict(os.environ) + existing_pythonpath = env.get("PYTHONPATH") + env["PYTHONPATH"] = ( + str(package_src) + if not existing_pythonpath + else f"{package_src}{os.pathsep}{existing_pythonpath}" + ) + process = subprocess.Popen( + [ + sys.executable, + "-m", + "cultmesh_py.daemon", + "--runtime-id", + "compacted-daemon-peer", + "--port", + "0", + "--cache-file", + str(cache_path), + "--enable-durable-shard-logs", + "--shard-log-file", + str(shard_log_path), + "--register-interop-note", + "--ready-file", + str(ready_path), + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + env=env, + ) + try: + ready = self._wait_for_ready_file(process, ready_path) + client = CultMesh.create_client("127.0.0.1", int(ready["port"]), timeout_seconds=2.0) + + snapshot_response = client.fetch_snapshot_response(schema_ids=["cultcache.interop-note"]) + self.assertEqual( + {document.record_key for document in snapshot_response.documents}, + {"note:compacted-one", "note:compacted-two"}, + ) + + compacted = client.fetch_shard_log_response( + shard_id="compacted-interop", + shard_epoch=7, + after_sequence=0, + ) + retained = client.fetch_shard_log_response( + shard_id="compacted-interop", + shard_epoch=7, + after_sequence=1, + ) + + self.assertTrue(compacted.resync_required) + self.assertEqual(compacted.reason, "compacted_history") + self.assertEqual(compacted.compacted_through, 1) + self.assertEqual(compacted.entries, ()) + self.assertFalse(retained.resync_required) + self.assertEqual([entry.sequence for entry in retained.entries], [2]) + self.assertIsNotNone(retained.entries[0].raw_document) + assert retained.entries[0].raw_document is not None + self.assertEqual(retained.entries[0].raw_document.record_key, "note:compacted-two") + finally: + self._terminate_process(process) + + def test_cultmesh_daemon_accepts_schema_write_forwarder_mutations(self) -> None: + with tempfile.TemporaryDirectory() as temp: + ready_path = Path(temp) / "ready.json" + package_src = Path(__file__).resolve().parents[1] / "src" + env = dict(os.environ) + existing_pythonpath = env.get("PYTHONPATH") + env["PYTHONPATH"] = ( + str(package_src) + if not existing_pythonpath + else f"{package_src}{os.pathsep}{existing_pythonpath}" + ) + process = subprocess.Popen( + [ + sys.executable, + "-m", + "cultmesh_py.daemon", + "--runtime-id", + "forwarder-daemon-peer", + "--port", + "0", + "--register-interop-note", + "--ready-file", + str(ready_path), + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + env=env, + ) + try: + ready = self._wait_for_ready_file(process, ready_path) + endpoint = f"cultnet://127.0.0.1:{ready['port']}" + shard = CultNetShardDescriptor( + shard_id="forwarded-interop", + owner_runtime_id="forwarder-daemon-peer", + epoch=3, + is_primary=False, + schema_ids=("cultcache.interop-note",), + primary_endpoints=(endpoint,), + ) + forwarder: CultNetShardWriteForwarder = CultNetSchemaWriteForwarder(timeout_seconds=2.0) + note = { + "schemaVersion": INTEROP_SCHEMA_VERSION, + "documentId": "note:forwarded-daemon", + "authorRuntimeId": "forwarding-client", + "title": "forwarded daemon note", + "body": "forwarded through the schema write-forwarder", + "tags": ["daemon", "forwarded"], + } + + forwarder.forward_put(shard, document_put_raw( + message_id="forwarded-daemon-put", + key="note:forwarded-daemon", + schema_id="cultcache.interop-note", + stored_at="2026-06-14T00:00:00Z", + payload=interop_note_document.encode_payload(note), + source_runtime_id="forwarding-client", + ).to_wire()) + + client = CultMesh.create_client("127.0.0.1", int(ready["port"]), timeout_seconds=2.0) + self._eventually(lambda: bool(client.fetch_snapshot_response( + schema_ids=["cultcache.interop-note"], + record_keys=["note:forwarded-daemon"], + ).documents)) + snapshot = client.fetch_snapshot_response( + schema_ids=["cultcache.interop-note"], + record_keys=["note:forwarded-daemon"], + ) + self.assertEqual( + interop_note_document.decode_payload(snapshot.documents[0].payload), + note, + ) + + forwarder.forward_delete(shard, document_delete( + message_id="forwarded-daemon-delete", + schema_id="cultcache.interop-note", + record_key="note:forwarded-daemon", + ).to_wire()) + self._eventually(lambda: not client.fetch_snapshot_response( + schema_ids=["cultcache.interop-note"], + record_keys=["note:forwarded-daemon"], + ).documents) + + shard_log = client.fetch_shard_log_response(shard_id="forwarded-interop", shard_epoch=3) + self.assertEqual([entry.change_kind for entry in shard_log.entries], ["added", "removed"]) + self.assertEqual(shard_log.entries[0].put["shardId"], "forwarded-interop") + self.assertEqual(shard_log.entries[0].put["shardEpoch"], 3) + self.assertEqual(shard_log.entries[1].delete["shardId"], "forwarded-interop") + self.assertEqual(shard_log.entries[1].delete["shardEpoch"], 3) + finally: + self._terminate_process(process) + + def test_cultmesh_daemon_enforces_snapshot_document_limit_as_peer_error(self) -> None: + with tempfile.TemporaryDirectory() as temp: + ready_path = Path(temp) / "ready.json" + package_src = Path(__file__).resolve().parents[1] / "src" + env = dict(os.environ) + existing_pythonpath = env.get("PYTHONPATH") + env["PYTHONPATH"] = ( + str(package_src) + if not existing_pythonpath + else f"{package_src}{os.pathsep}{existing_pythonpath}" + ) + process = subprocess.Popen( + [ + sys.executable, + "-m", + "cultmesh_py.daemon", + "--runtime-id", + "limited-daemon-peer", + "--port", + "0", + "--seed-interop-note", + "--max-snapshot-documents", + "0", + "--ready-file", + str(ready_path), + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + env=env, + ) + try: + ready = self._wait_for_ready_file(process, ready_path) + self.assertEqual(ready["snapshotLimits"], {"maxSnapshotBytes": None, "maxSnapshotDocuments": 0}) + client = CultMesh.create_client("127.0.0.1", int(ready["port"]), timeout_seconds=2.0) + + with self.assertRaisesRegex(CultNetPeerError, "Snapshot document limit exceeded") as raised: + client.fetch_snapshot_response(schema_ids=["cultcache.interop-note"]) + + self.assertEqual(raised.exception.response["schemaVersion"], "cultnet.error.v0") + self.assertEqual(raised.exception.response["code"], "snapshot_document_limit_exceeded") + self.assertEqual(raised.exception.response["details"]["documentCount"], 1) + self.assertEqual(raised.exception.response["details"]["maxSnapshotDocuments"], 0) + finally: + self._terminate_process(process) + + def test_cultmesh_daemon_serves_opt_in_simulation_observations(self) -> None: + with tempfile.TemporaryDirectory() as temp: + ready_path = Path(temp) / "ready.json" + package_src = Path(__file__).resolve().parents[1] / "src" + env = dict(os.environ) + existing_pythonpath = env.get("PYTHONPATH") + env["PYTHONPATH"] = ( + str(package_src) + if not existing_pythonpath + else f"{package_src}{os.pathsep}{existing_pythonpath}" + ) + process = subprocess.Popen( + [ + sys.executable, + "-m", + "cultmesh_py.daemon", + "--runtime-id", + "simulation-daemon-peer", + "--port", + "0", + "--enable-simulation-observations", + "--simulation-minimum-witnesses", + "2", + "--simulation-quorum-ratio", + "1.0", + "--verse-id", + "simulation-verse", + "--ready-file", + str(ready_path), + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + env=env, + ) + try: + ready = self._wait_for_ready_file(process, ready_path) + self.assertIn("cultnet.simulation_observation.v0", ready["supportedMessageVersions"]) + self.assertIn("cultnet.simulation_consensus_candidate.v0", ready["supportedMessageVersions"]) + client = CultMesh.create_client("127.0.0.1", int(ready["port"]), timeout_seconds=2.0) + hello_response = client.request( + hello(runtime_id="simulation-daemon-prober"), + expected_schema_version="cultnet.hello.v0", + ) + self.assertEqual(ready["supportedMessageVersions"], hello_response["supportedMessageVersions"]) + discovery_client = CultMesh.create_verse_discovery_client("127.0.0.1", int(ready["port"]), timeout_seconds=2.0) + peers = discovery_client.fetch_peers(verse_id="simulation-verse", roles=["simulation-observer"]) + self.assertEqual(peers[0].peer_id, "simulation-daemon-peer") + self.assertIn("simulation-observer", peers[0].roles) + self.assertIn("read-replica", peers[0].roles) + + claim_hash = compute_simulation_claim_hash("hit", "alice", "bob", "frame:100") + first = client.request( + simulation_observation( + message_id="daemon-sim-1", + witness_runtime_id="watcher-1", + shard_id="arena", + shard_epoch=4, + frame=100, + subject_id="bob", + claim_kind="hit", + claim_hash=claim_hash, + claim_summary="alice shot bob first", + ), + expected_schema_version="cultnet.simulation_consensus_candidate.v0", + ) + second = client.request( + simulation_observation( + message_id="daemon-sim-2", + witness_runtime_id="watcher-2", + shard_id="arena", + shard_epoch=4, + frame=100, + subject_id="bob", + claim_kind="hit", + claim_hash=claim_hash, + claim_summary="alice shot bob first", + ), + expected_schema_version="cultnet.simulation_consensus_candidate.v0", + ) + + self.assertEqual(first["messageId"], "daemon-sim-1") + self.assertFalse(first["hasQuorum"]) + self.assertEqual(first["witnessCount"], 1) + self.assertEqual(second["messageId"], "daemon-sim-2") + self.assertTrue(second["hasQuorum"]) + self.assertEqual(second["witnessCount"], 2) + self.assertEqual(second["claimHash"], claim_hash) + finally: + self._terminate_process(process) + + def _wait_for_ready_file(self, process: subprocess.Popen[str], ready_path: Path) -> dict[str, object]: + deadline = time.monotonic() + 5.0 + while not ready_path.exists() and time.monotonic() < deadline: + if process.poll() is not None: + break + time.sleep(0.05) + if not ready_path.exists(): + _, stderr = process.communicate(timeout=2.0) + self.fail(f"cultmesh daemon did not publish readiness: {stderr}") + return json.loads(ready_path.read_text(encoding="utf-8")) + + def _terminate_process(self, process: subprocess.Popen[str]) -> None: + process.terminate() + try: + process.communicate(timeout=5.0) + except subprocess.TimeoutExpired: + process.kill() + process.communicate(timeout=5.0) + + def test_cultmesh_local_server_serves_node_and_catalogs_over_clients(self) -> None: + document = define_database_entry_type( + "mesh.server_note", + [("body", 0)], + schema_id="mesh.server_note.v1", + ) + node = CultMesh.create_node(runtime_id="mesh-server") + node.database.register_document(document) + node.database.put_raw_message(document, "note:1", {"body": "served"}, shard_id="notes", shard_epoch=1) + observation_hub = CultNetSimulationObservationHub( + CultNetSimulationConsensusOptions(minimum_witnesses=2, quorum_ratio=1.0) + ) + seen_candidates: list[CultNetSimulationConsensusCandidate] = [] + unsubscribe_candidates = observation_hub.watch_candidates(seen_candidates.append) + claim_hash = compute_simulation_claim_hash("hit", "alice", "bob", "frame:100") + verses = CultMeshVerseCatalog() + verses.upsert( + CultMeshVerseDescriptor( + verse_id="server-verse", + display_name="Server Verse", + authority_model="local", + compatibility=CultMeshVerseCompatibility("cultmesh.v0", "rules"), + ) + ) + peers = CultMeshPeerCatalog() + peers.upsert( + CultMeshPeerCard( + peer_id="mesh-server", + verse_id="server-verse", + endpoints=("cultnet://127.0.0.1:0",), + roles=("read-replica",), + ) + ) + + server = CultMesh.serve_node( + node, + verse_catalog=verses, + peer_catalog=peers, + observation_hub=observation_hub, + display_name="Mesh Server", + ) + try: + raw_client = CultMesh.create_client("127.0.0.1", server.port, timeout_seconds=2.0) + hello_response = raw_client.request(hello(runtime_id="probe"), expected_schema_version="cultnet.hello.v0") + schema_response = raw_client.fetch_schema_catalog(schema_ids=["mesh.server_note.v1"], include_schema_json=True) + rudp_client = CultMesh.create_client( + endpoint=f"rudp://127.0.0.1:{server.port}", + timeout_seconds=2.0, + connection_id=server.rudp_connection_id, + runtime_id="rudp-probe", + ) + rudp_hello_response = rudp_client.request( + hello(runtime_id="rudp-probe"), + expected_schema_version="cultnet.hello.v0", + ) + rudp_schema_response = rudp_client.fetch_schema_catalog( + schema_ids=["mesh.server_note.v1"], + include_schema_json=True, + ) + wire_schema_response = raw_client.fetch_schema_catalog(kinds=["wire_message"], include_schema_json=True) + synced_schema_catalog = CultNetSchemaCatalog() + synced_wire_descriptors = raw_client.sync_schema_catalog( + synced_schema_catalog, + kinds=["wire_message"], + include_schema_json=True, + ) + snapshot_response = raw_client.fetch_snapshot(schema_ids=["mesh.server_note.v1"]) + typed_snapshot_response = raw_client.fetch_snapshot_response(schema_ids=["mesh.server_note.v1"]) + shard_catalog = raw_client.fetch_shard_catalog(schema_ids=["mesh.server_note.v1"]) + synced_shard_catalog = CultNetShardCatalog() + synced_shards = raw_client.sync_shard_catalog(synced_shard_catalog, schema_ids=["mesh.server_note.v1"]) + shard_log = raw_client.fetch_shard_log(shard_id="notes", shard_epoch=1) + typed_shard_log = raw_client.fetch_shard_log_response(shard_id="notes", shard_epoch=1) + self.assertEqual(hello_response["transportProfiles"][0]["transports"][0]["protocol"], "tcp_framed") + self.assertEqual(hello_response["transportProfiles"][0]["transports"][0]["port"], server.port) + stale_shard_log = raw_client.fetch_shard_log(shard_id="notes", shard_epoch=0) + discovery_client = CultMesh.create_verse_discovery_client("127.0.0.1", server.port, timeout_seconds=2.0) + fetched_verses = discovery_client.fetch_verses(transport_version="cultmesh.v0") + fetched_peers = discovery_client.fetch_peers(verse_id="server-verse", roles=["read-replica"]) + first_candidate = raw_client.request( + simulation_observation( + message_id="obs-1", + witness_runtime_id="watcher-1", + shard_id="arena", + shard_epoch=4, + frame=100, + subject_id="bob", + claim_kind="hit", + claim_hash=claim_hash, + claim_summary="alice shot bob first", + ), + expected_schema_version="cultnet.simulation_consensus_candidate.v0", + ) + quorum_candidate = raw_client.request( + simulation_observation( + message_id="obs-2", + witness_runtime_id="watcher-2", + shard_id="arena", + shard_epoch=4, + frame=100, + subject_id="bob", + claim_kind="hit", + claim_hash=claim_hash, + claim_summary="alice shot bob first", + ), + expected_schema_version="cultnet.simulation_consensus_candidate.v0", + ) + with raw_client.subscribe_database(subscription_id="sub-server", schema_ids=["mesh.server_note.v1"]) as subscription: + subscription_snapshot = subscription.read_next_snapshot_response() + subscription.send(document_put_raw( + message_id="server-put", + key="note:2", + schema_id="mesh.server_note.v1", + stored_at="2026-06-14T00:00:00Z", + payload=document.encode_payload({"body": "subscribed"}), + shard_id="primary", + shard_epoch=1, + )) + subscription_change = subscription.read_next_change() + subscription.send(document_delete( + message_id="server-delete", + schema_id="mesh.server_note.v1", + record_key="note:2", + shard_id="primary", + shard_epoch=1, + )) + subscription_delete = subscription.read_next_change() + finally: + unsubscribe_candidates() + server.stop() + + self.assertEqual(hello_response["runtimeId"], "mesh-server") + self.assertEqual(hello_response["displayName"], "Mesh Server") + self.assertIn("cultnet.database_subscribe.v0", hello_response["supportedMessageVersions"]) + self.assertIn("cultnet.error.v0", hello_response["supportedMessageVersions"]) + self.assertIn("cultnet.document_put_raw.v0", hello_response["supportedMessageVersions"]) + self.assertIn("cultnet.simulation_observation.v0", hello_response["supportedMessageVersions"]) + self.assertEqual(hello_response["supportedMutationContracts"][0]["documentType"], "mesh.server_note") + self.assertIn("documentDelete", hello_response["supportedMutationContracts"][0]["operations"]) + self.assertIn("shardLog", hello_response["supportedMutationContracts"][0]["operations"]) + advertised_transports = { + transport["protocol"]: transport + for profile in hello_response["transportProfiles"] + for transport in profile["transports"] + } + self.assertEqual(advertised_transports["tcp_framed"]["port"], server.port) + self.assertEqual(advertised_transports["rudp"]["port"], server.port) + self.assertEqual(rudp_hello_response["runtimeId"], "mesh-server") + self.assertEqual(rudp_schema_response["schemas"][0]["schemaId"], "mesh.server_note.v1") + self.assertEqual(schema_response["schemas"][0]["schemaId"], "mesh.server_note.v1") + self.assertIn("schemaJson", schema_response["schemas"][0]) + self.assertIn("cultnet.document_delete.v0", schema_response["schemas"][0]["wireContracts"]) + self.assertIn("cultnet.shard_log_response.v0", schema_response["schemas"][0]["wireContracts"]) + wire_descriptors = {schema["schemaVersion"]: schema for schema in wire_schema_response["schemas"]} + self.assertEqual(wire_descriptors["cultnet.error.v0"]["kind"], "wire_message") + self.assertEqual(wire_descriptors["cultnet.document_put_raw.v0"]["kind"], "wire_message") + self.assertIn("schemaJson", wire_descriptors["cultnet.document_put_raw.v0"]) + self.assertIn("cultmesh.peer_exchange_response.v0", wire_descriptors) + self.assertIn("cultnet.error.v0", {descriptor.schema_version for descriptor in synced_wire_descriptors}) + self.assertIn("cultnet.document_put_raw.v0", {descriptor.schema_version for descriptor in synced_wire_descriptors}) + self.assertIsNotNone(synced_schema_catalog.get("https://github.com/GameCult/cultnet-ts/contracts/cultnet.document-put-raw.schema.json")) + self.assertEqual(snapshot_response["documents"][0]["recordKey"], "note:1") + self.assertEqual(typed_snapshot_response.documents[0].record_key, "note:1") + self.assertEqual(typed_snapshot_response.documents[0].schema_id, "mesh.server_note.v1") + self.assertEqual(shard_catalog["shards"][0]["schemaIds"], ["mesh.server_note.v1"]) + self.assertEqual(shard_catalog["shards"][0]["epoch"], 1) + self.assertEqual(shard_catalog["shards"][0]["shardId"], "notes") + self.assertEqual(synced_shards[0].shard_id, "notes") + self.assertEqual(synced_shards[0].epoch, 1) + self.assertEqual(synced_shard_catalog.get("notes"), synced_shards[0]) + self.assertEqual(shard_log["entries"][0]["changeKind"], "added") + self.assertTrue(stale_shard_log["resyncRequired"]) + self.assertEqual(stale_shard_log["reason"], "stale_epoch") + self.assertEqual(stale_shard_log["shardEpoch"], 1) + self.assertEqual(stale_shard_log["entries"], []) + self.assertEqual(typed_shard_log.last_sequence, 1) + self.assertEqual(typed_shard_log.entries[0].put["document"]["recordKey"], "note:1") + self.assertIsInstance(typed_shard_log.entries[0].raw_document, CultNetRawDocumentRecord) + self.assertEqual(typed_shard_log.entries[0].raw_document.record_key, "note:1") + self.assertEqual(fetched_verses[0].verse_id, "server-verse") + self.assertEqual(fetched_peers[0].peer_id, "mesh-server") + self.assertEqual(first_candidate["messageId"], "obs-1") + self.assertFalse(first_candidate["hasQuorum"]) + self.assertEqual(quorum_candidate["messageId"], "obs-2") + self.assertEqual(quorum_candidate["claimHash"], claim_hash) + self.assertTrue(quorum_candidate["hasQuorum"]) + self.assertEqual(quorum_candidate["witnessCount"], 2) + self.assertEqual([candidate.claim_hash for candidate in seen_candidates], [claim_hash, claim_hash]) + self.assertIsInstance(subscription_snapshot, CultNetRawSnapshotResponse) + self.assertEqual(subscription_snapshot.to_wire()["schemaVersion"], "cultnet.snapshot_response_raw.v0") + self.assertEqual(subscription_change.change_kind, "added") + self.assertEqual(subscription_change.record_key, "note:2") + self.assertEqual(subscription_change.document["recordKey"], "note:2") + self.assertIsInstance(subscription_change.raw_document, CultNetRawDocumentRecord) + self.assertEqual(subscription_change.raw_document.schema_id, "mesh.server_note.v1") + self.assertEqual(subscription_delete.change_kind, "removed") + self.assertEqual(subscription_delete.record_key, "note:2") + self.assertIsNone(subscription_delete.raw_document) + self.assertIsNone(node.database.get(document, "note:2")) + + def test_cultmesh_local_server_resolves_snapshot_and_shard_catalog_schema_aliases(self) -> None: + document = define_database_entry_type( + "mesh.alias_server_note", + [("body", 0)], + schema_id="sha256:mesh-alias-server-note", + schema_name="mesh.alias_server_note", + schema_version="mesh.alias_server_note.v1", + ) + node = CultMesh.create_node(runtime_id="mesh-alias-server") + node.database.register_document(document) + node.database.put_raw_message( + document, + "note:alias", + {"body": "served through alias"}, + shard_id="notes", + shard_epoch=1, + ) + + server = CultMesh.serve_node(node) + try: + raw_client = CultMesh.create_client("127.0.0.1", server.port, timeout_seconds=2.0) + snapshot = raw_client.fetch_snapshot(schema_ids=["mesh.alias_server_note.v1"]) + typed_snapshot = raw_client.fetch_snapshot_response(schema_ids=["mesh.alias_server_note.v1"]) + shard_catalog = raw_client.fetch_shard_catalog(schema_ids=["mesh.alias_server_note.v1"]) + finally: + server.stop() + + local_schema_id = document.catalog_entry().schema_id + self.assertEqual(snapshot["documents"][0]["schemaId"], local_schema_id) + self.assertEqual(snapshot["documents"][0]["recordKey"], "note:alias") + self.assertEqual(typed_snapshot.documents[0].schema_id, local_schema_id) + self.assertEqual(shard_catalog["shards"][0]["schemaIds"], [local_schema_id]) + self.assertEqual(shard_catalog["shards"][0]["shardId"], "notes") + + def test_cultmesh_shard_snapshot_keeps_foreign_schema_alias_membership(self) -> None: + document = define_database_entry_type( + "mesh.alias_shard_note", + [ + ("schema_version", 0), + ("body", 1), + ], + schema_id="sha256:mesh-alias-shard-note", + schema_name="mesh.alias_shard_note", + schema_version="mesh.alias_shard_note.v1", + ) + node = CultMesh.create_node(runtime_id="mesh-alias-shard-server") + node.database.register_document(document) + + node.database.apply_raw_put_message(document_put_raw( + message_id="put-foreign-shard-alias", + key="note:foreign-shard", + schema_id="runtime.generated.mesh.alias_shard_note.ui.99", + stored_at="2026-06-27T00:00:00Z", + payload=document.encode_payload({ + "schema_version": "mesh.alias_shard_note.v1", + "body": "visible through shard snapshot", + }), + shard_id="notes", + shard_epoch=1, + ).to_wire()) + node.database.put_raw_message( + document, + "note:other-shard", + { + "schema_version": "mesh.alias_shard_note.v1", + "body": "hidden by shard filter", + }, + shard_id="other", + shard_epoch=1, + ) + + response = node.database.build_snapshot_response( + schema_ids=["mesh.alias_shard_note.v1"], + shard_id="notes", + shard_epoch=1, + ) + + local_schema_id = document.catalog_entry().schema_id + self.assertEqual(response.shard_id, "notes") + self.assertEqual(response.shard_epoch, 1) + self.assertEqual([(record.schema_id, record.record_key) for record in response.documents], [ + (local_schema_id, "note:foreign-shard"), + ]) + self.assertEqual(node.cache.get_required_envelope(document, "note:foreign-shard").schema_id, local_schema_id) + + node.database.apply_raw_delete_message(document_delete( + message_id="delete-foreign-shard-alias", + schema_id="mesh.alias_shard_note.v1", + record_key="note:foreign-shard", + shard_id="notes", + shard_epoch=1, + ).to_wire()) + deleted_response = node.database.build_snapshot_response( + schema_ids=["mesh.alias_shard_note.v1"], + shard_id="notes", + shard_epoch=1, + ) + self.assertEqual(deleted_response.documents, ()) + + def test_cultmesh_local_server_notifies_schema_alias_subscription_for_raw_put(self) -> None: + document = define_database_entry_type( + "mesh.alias_sub_note", + [ + ("schema_version", 0), + ("body", 1), + ], + schema_id="sha256:mesh-alias-sub-note", + schema_name="mesh.alias_sub_note", + schema_version="mesh.alias_sub_note.v1", + ) + node = CultMesh.create_node(runtime_id="mesh-alias-sub-server") + node.database.register_document(document) + + server = CultMesh.serve_node(node) + try: + raw_client = CultMesh.create_client("127.0.0.1", server.port, timeout_seconds=2.0) + with raw_client.subscribe_database( + subscription_id="sub-alias", + schema_ids=["mesh.alias_sub_note.v1"], + include_snapshot=False, + ) as subscription: + subscription.send(document_put_raw( + message_id="put-foreign-alias", + key="note:alias-sub", + schema_id="runtime.generated.mesh.alias_sub_note.ui.99", + stored_at="2026-06-27T00:00:00Z", + payload=document.encode_payload({ + "schema_version": "mesh.alias_sub_note.v1", + "body": "notified", + }), + )) + change = subscription.read_next_change() + finally: + server.stop() + + local_schema_id = document.catalog_entry().schema_id + self.assertEqual(change.subscription_id, "sub-alias") + self.assertEqual(change.change_kind, "added") + self.assertEqual(change.record_key, "note:alias-sub") + self.assertIsInstance(change.raw_document, CultNetRawDocumentRecord) + self.assertEqual(change.raw_document.schema_id, "runtime.generated.mesh.alias_sub_note.ui.99") + self.assertEqual(node.database.get_required(document, "note:alias-sub")["body"], "notified") + self.assertEqual(node.cache.get_required_envelope(document, "note:alias-sub").schema_id, local_schema_id) + + def test_cultmesh_local_server_notifies_schema_alias_subscription_for_delete(self) -> None: + document = define_database_entry_type( + "mesh.alias_delete_note", + [("body", 0)], + schema_id="sha256:mesh-alias-delete-note", + schema_name="mesh.alias_delete_note", + schema_version="mesh.alias_delete_note.v1", + ) + node = CultMesh.create_node(runtime_id="mesh-alias-delete-server") + node.database.register_document(document) + node.database.put_raw_message( + document, + "note:delete-alias", + {"body": "remove me"}, + shard_id="notes", + shard_epoch=1, + ) + + server = CultMesh.serve_node(node) + try: + raw_client = CultMesh.create_client("127.0.0.1", server.port, timeout_seconds=2.0) + with raw_client.subscribe_database( + subscription_id="sub-delete-alias", + schema_ids=[document.catalog_entry().schema_id], + include_snapshot=False, + ) as subscription: + subscription.send(document_delete( + message_id="delete-foreign-alias", + schema_id="mesh.alias_delete_note.v1", + record_key="note:delete-alias", + shard_id="notes", + shard_epoch=1, + )) + change = subscription.read_next_change() + finally: + server.stop() + + self.assertEqual(change.subscription_id, "sub-delete-alias") + self.assertEqual(change.change_kind, "removed") + self.assertEqual(change.schema_id, "mesh.alias_delete_note.v1") + self.assertEqual(change.record_key, "note:delete-alias") + self.assertIsNone(change.raw_document) + self.assertIsNone(node.database.get(document, "note:delete-alias")) + + def test_cultmesh_local_server_rejects_oversized_snapshot_responses(self) -> None: + document = define_database_entry_type( + "mesh.snapshot_limit_note", + [("body", 0)], + schema_id="mesh.snapshot_limit_note.v1", + ) + node = CultMesh.create_node(runtime_id="mesh-snapshot-limits") + node.database.register_document(document) + node.database.put_raw_message(document, "note:1", {"body": "first"}) + node.database.put_raw_message(document, "note:2", {"body": "second"}) + + server = CultMesh.serve_node(node, max_snapshot_documents=1) + try: + raw_client = CultMesh.create_client("127.0.0.1", server.port, timeout_seconds=2.0) + filtered_snapshot = raw_client.fetch_snapshot(record_keys=["note:1"]) + error = raw_client.request( + snapshot_request(schema_ids=["mesh.snapshot_limit_note.v1"]), + expected_schema_version="cultnet.error.v0", + ) + with self.assertRaisesRegex(CultNetPeerError, "Snapshot document limit exceeded"): + raw_client.fetch_snapshot(schema_ids=["mesh.snapshot_limit_note.v1"]) + finally: + server.stop() + + self.assertEqual(filtered_snapshot["schemaVersion"], "cultnet.snapshot_response_raw.v0") + self.assertEqual([document["recordKey"] for document in filtered_snapshot["documents"]], ["note:1"]) + self.assertEqual(error["schemaVersion"], "cultnet.error.v0") + self.assertIn("Snapshot document limit exceeded", error["error"]) + self.assertEqual(error["code"], "snapshot_document_limit_exceeded") + self.assertEqual(error["details"]["documentCount"], 2) + self.assertEqual(error["details"]["maxSnapshotDocuments"], 1) + + def test_cultmesh_local_server_returns_errors_for_bad_requests(self) -> None: + document = define_database_entry_type( + "mesh.error_note", + [("body", 0)], + schema_id="mesh.error_note.v1", + ) + node = CultMesh.create_node(runtime_id="mesh-error-server") + node.database.register_document(document) + + server = CultMesh.serve_node(node) + try: + raw_client = CultMesh.create_client("127.0.0.1", server.port, timeout_seconds=2.0) + unsupported = raw_client.request( + {"schemaVersion": "cultnet.nope.v0", "messageId": "bad-schema"}, + expected_schema_version="cultnet.error.v0", + ) + malformed_put = raw_client.request( + {"schemaVersion": "cultnet.document_put_raw.v0", "messageId": "bad-put"}, + expected_schema_version="cultnet.error.v0", + ) + malformed_delete = raw_client.request( + {"schemaVersion": "cultnet.document_delete.v0", "messageId": "bad-delete", "schemaId": "mesh.error_note.v1"}, + expected_schema_version="cultnet.error.v0", + ) + finally: + server.stop() + + self.assertEqual(unsupported["messageId"], "bad-schema") + self.assertEqual(unsupported["code"], "unsupported_schema_version") + self.assertEqual(unsupported["details"]["schemaVersion"], "cultnet.nope.v0") + self.assertIn("Unsupported CultNet message schema", unsupported["error"]) + self.assertEqual(malformed_put["messageId"], "bad-put") + self.assertEqual(malformed_put["code"], "malformed_document_put") + self.assertEqual(malformed_delete["messageId"], "bad-delete") + self.assertEqual(malformed_delete["code"], "malformed_document_delete") + + def test_cultmesh_authority_lease_requires_live_matching_lease(self) -> None: + peer = CultMeshPeerCard( + peer_id="voidbot-local", + verse_id="local", + endpoints=("cultmesh://localhost",), + roles=("shard-primary",), + authority_lease_id="lease:voidbot-local", + ) + leases = CultMeshAuthorityLeaseCatalog() + peers = CultMeshPeerCatalog() + peers.upsert(peer) + now = datetime.now(UTC) + self.assertFalse(leases.is_authorized(peer, "shard-primary", at=now)) + self.assertEqual(peers.find_authorized("local", "shard-primary", leases, at=now), []) + + leases.upsert( + CultMeshAuthorityLease( + lease_id="lease:voidbot-local", + verse_id="local", + peer_id="voidbot-local", + roles=("shard-primary", "shard-primary", ""), + valid_from=now - timedelta(seconds=1), + expires_at=now + timedelta(seconds=1), + shard_ids=("shard-a",), + issuer_runtime_id="odin", + signature="signed", + ) + ) + + lease = leases.get("lease:voidbot-local") + self.assertIsNotNone(lease) + assert lease is not None + self.assertEqual(leases.leases, (lease,)) + self.assertEqual(lease.roles, ("shard-primary",)) + self.assertEqual(lease.issuer_runtime_id, "odin") + self.assertEqual(lease.signature, "signed") + self.assertTrue(lease.is_valid_at(now)) + self.assertTrue(lease.covers(peer, "shard-primary", shard_id="shard-a", at=now)) + self.assertFalse(lease.covers(peer, "shard-primary", shard_id="shard-b", at=now)) + self.assertTrue(leases.is_authorized(peer, "shard-primary", shard_id="shard-a", at=now)) + self.assertEqual( + peers.find_authorized("local", "shard-primary", leases, shard_id="shard-a", at=now), + [peer], + ) + self.assertEqual( + peers.first_authorized("local", "shard-primary", leases, shard_id="shard-a", at=now), + peer, + ) + self.assertIsNone(peers.first_authorized("local", "read-replica", leases, at=now)) + self.assertFalse(leases.is_authorized(peer, "shard-primary", shard_id="shard-b", at=now)) + self.assertFalse(leases.is_authorized(peer, "read-replica", at=now)) + self.assertIsNone(leases.get("missing")) + + def test_cultmesh_authority_lease_catalog_can_require_verified_signatures(self) -> None: + peer = CultMeshPeerCard( + peer_id="voidbot-local", + verse_id="local", + endpoints=("cultmesh://localhost",), + roles=("shard-primary",), + shard_ids=("shard-a",), + authority_lease_id="lease:voidbot-local", + ) + now = datetime.now(UTC) + unsigned = CultMeshAuthorityLease( + lease_id="lease:voidbot-local", + verse_id="local", + peer_id="voidbot-local", + roles=("shard-primary",), + valid_from=now - timedelta(seconds=1), + expires_at=now + timedelta(seconds=30), + shard_ids=("shard-a",), + issuer_runtime_id="odin", + ) + verifier = CultMeshHmacAuthorityLeaseVerifier({"odin": b"lease-secret"}) + signed = verifier.issue(unsigned) + strict_leases = CultMesh.create_authority_lease_catalog( + signature_verifier=verifier.verify, + require_verified_signatures=True, + ) + strict_leases.upsert(unsigned) + self.assertFalse(strict_leases.is_verified(unsigned.lease_id)) + self.assertFalse(strict_leases.is_authorized(peer, "shard-primary", shard_id="shard-a", at=now)) + + strict_leases.upsert(signed) + self.assertTrue(strict_leases.is_verified(signed.lease_id)) + self.assertTrue(strict_leases.is_authorized(peer, "shard-primary", shard_id="shard-a", at=now)) + + wrong_key = CultMesh.create_authority_lease_catalog( + signature_verifier=CultMeshHmacAuthorityLeaseVerifier({"odin": b"wrong-secret"}).verify, + require_verified_signatures=True, + ) + wrong_key.upsert(signed) + self.assertFalse(wrong_key.is_verified(signed.lease_id)) + self.assertFalse(wrong_key.is_authorized(peer, "shard-primary", shard_id="shard-a", at=now)) + + tampered = CultMeshAuthorityLease( + lease_id=signed.lease_id, + verse_id=signed.verse_id, + peer_id=signed.peer_id, + roles=("shard-primary", "read-replica"), + valid_from=signed.valid_from, + expires_at=signed.expires_at, + shard_ids=signed.shard_ids, + issuer_runtime_id=signed.issuer_runtime_id, + signature=signed.signature, + ) + strict_leases.upsert(tampered) + self.assertFalse(strict_leases.is_verified(tampered.lease_id)) + self.assertFalse(strict_leases.is_authorized(peer, "shard-primary", shard_id="shard-a", at=now)) + + def test_cultmesh_stream_catalog_negotiates_transport_and_latest_frame(self) -> None: + streams = CultMeshStreamCatalog() + streams.declare( + CultMeshStreamDescriptor( + stream_id="mimir:kiyo-pro", + verse_id="studio", + owner_peer_id="starfire", + kind="video", + label="Kiyo Pro", + clock={"clockDomainId": "starfire-qpc", "confidence": 0.25}, + video={"width": 1920, "height": 1080, "pixelFormat": "YUY2", "framesPerSecond": 30}, + preferred_transports=("shared-d3d12-texture", "shared-memory", "cultcache-page"), + max_in_flight_frames=3, + ) + ) + + negotiation = streams.negotiate( + "mimir:kiyo-pro", + CultMeshStreamConsumerProfile( + peer_id="fensalir", + verse_id="studio", + supported_transports=("shared-d3d12-texture", "cultcache-page"), + accepted_kinds=("video",), + can_import_gpu_handles=True, + max_in_flight_frames=2, + ), + ) + + self.assertEqual(negotiation.transport, "shared-d3d12-texture") + self.assertEqual(negotiation.max_in_flight_frames, 2) + self.assertEqual(negotiation.copy_budget, "zero-copy-target") + + streams.publish_frame( + CultMeshStreamFrameHandle( + stream_id="mimir:kiyo-pro", + sequence=42, + timestamp_ns=1_000_000_000, + duration_ns=33_333_334, + transport="shared-d3d12-texture", + native_handle="0xfeed", + fence_handle="0xbeef", + fence_value=7, + unavoidable_copy_count=0, + ) + ) + self.assertEqual(streams.latest_frame("mimir:kiyo-pro").sequence, 42) + + +if __name__ == "__main__": + unittest.main() diff --git a/packages/cultmesh-py/tools/reactive_performance_probe.py b/packages/cultmesh-py/tools/reactive_performance_probe.py new file mode 100644 index 00000000..86ab9441 --- /dev/null +++ b/packages/cultmesh-py/tools/reactive_performance_probe.py @@ -0,0 +1,145 @@ +from __future__ import annotations + +import argparse +import json +import threading +import time +import tracemalloc +from typing import Any + +from cultcache_py import define_document_type +from cultmesh_py import CultMeshReactiveDocumentOptions, create_node + + +def percentile(ordered: list[float], ratio: float) -> float: + if not ordered: + return 0.0 + index = max(0, min(len(ordered) - 1, int(len(ordered) * ratio + 0.999999) - 1)) + return ordered[index] + + +def measure(document_count: int, idle_seconds: float, active_seconds: float) -> dict[str, Any]: + payload = "x" * (16 * 1024) + changed_document_count = max(1, document_count // 100) + document = define_document_type( + f"gamecult.mesh.python_performance_probe_{document_count}", + encode=lambda value: value, + decode=lambda value: value, + ) + node = create_node(runtime_id=f"python-performance-{document_count}") + node.register_document(document) + reactive_documents = [] + publish_events = [threading.Event() for _ in range(document_count)] + update_started_at = [0.0] * document_count + latencies: list[float] = [] + payload_bytes_published = 0 + publishes = 0 + measurement_lock = threading.Lock() + + for index in range(document_count): + key = f"performance:{index}" + node.put(document, key, {"id": key, "payload": payload, "revision": 1}) + writer = node.authoritative_writer(document, key) + original_write = writer.write + + def measured_write(value: Any, document_index: int = index, write=original_write) -> None: + nonlocal payload_bytes_published, publishes + encoded_bytes = len(document.encode_payload(value)) + latency_ms = (time.perf_counter() - update_started_at[document_index]) * 1000 + with measurement_lock: + payload_bytes_published += encoded_bytes + publishes += 1 + latencies.append(latency_ms) + write(value) + publish_events[document_index].set() + + writer.write = measured_write # type: ignore[method-assign] + reactive_documents.append( + writer.reactive(CultMeshReactiveDocumentOptions(flush_delay_seconds=0.0)) + ) + + tracemalloc.start() + idle_cpu_start = time.process_time() + time.sleep(idle_seconds) + idle_current_bytes, idle_peak_bytes = tracemalloc.get_traced_memory() + idle_cpu_ms = (time.process_time() - idle_cpu_start) * 1000 + tracemalloc.reset_peak() + active_current_start, _ = tracemalloc.get_traced_memory() + active_cpu_start = time.process_time() + active_started_at = time.perf_counter() + frames = 0 + while time.perf_counter() - active_started_at < active_seconds: + for index in range(changed_document_count): + publish_events[index].clear() + update_started_at[index] = time.perf_counter() + reactive_documents[index].update( + lambda draft: draft.__setitem__("revision", draft["revision"] + 1) + ) + for index in range(changed_document_count): + if not publish_events[index].wait(timeout=1.0): + raise RuntimeError(f"Python reactive publication timed out for document {index}") + frames += 1 + remaining = frames / 60 - (time.perf_counter() - active_started_at) + if remaining > 0: + time.sleep(remaining) + + active_elapsed_seconds = time.perf_counter() - active_started_at + active_cpu_ms = (time.process_time() - active_cpu_start) * 1000 + active_current_bytes, active_peak_bytes = tracemalloc.get_traced_memory() + tracemalloc.stop() + for reactive in reactive_documents: + reactive.dispose() + ordered = sorted(latencies) + return { + "documentCount": document_count, + "changedDocumentCount": changed_document_count, + "frames": frames, + "expectedPublishes": frames * changed_document_count, + "actualPublishes": publishes, + "payloadBytesPublished": payload_bytes_published, + "idleTracedCurrentBytes": idle_current_bytes, + "idleTracedPeakBytes": idle_peak_bytes, + "idleCpuMilliseconds": idle_cpu_ms, + "activeTracedRetainedGrowthBytes": active_current_bytes - active_current_start, + "activeTracedPeakGrowthBytes": active_peak_bytes - active_current_start, + "activeCpuMilliseconds": active_cpu_ms, + "publishedPayloadBytesPerSecond": payload_bytes_published / active_elapsed_seconds, + "p50PublishLatencyMilliseconds": percentile(ordered, 0.50), + "p95PublishLatencyMilliseconds": percentile(ordered, 0.95), + "p99PublishLatencyMilliseconds": percentile(ordered, 0.99), + "schedulerThreads": sum( + thread.name == "cultmesh-reactive-scheduler" for thread in threading.enumerate() + ), + } + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--quick", action="store_true") + args = parser.parse_args() + idle_seconds = 0.25 if args.quick else 10.0 + active_seconds = 0.5 if args.quick else 10.0 + results = [measure(count, idle_seconds, active_seconds) for count in (1, 100, 1000)] + print(json.dumps({ + "runtime": "python", + "workload": { + "documentCounts": [1, 100, 1000], + "payloadBytes": 16 * 1024, + "idleSeconds": idle_seconds, + "activeSeconds": active_seconds, + "updateRateHz": 60, + "changedFraction": 0.01, + }, + "results": results, + }, indent=2)) + return 0 if all( + result["actualPublishes"] == result["expectedPublishes"] + and result["p99PublishLatencyMilliseconds"] < 250 + and result["activeTracedPeakGrowthBytes"] < 128 * 1024 * 1024 + and result["schedulerThreads"] == 1 + for result in results + ) else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/packages/gamecult-geometry-rs/Cargo.lock b/packages/cultmesh-rs/Cargo.lock similarity index 53% rename from packages/gamecult-geometry-rs/Cargo.lock rename to packages/cultmesh-rs/Cargo.lock index 986160f0..945bc65b 100644 --- a/packages/gamecult-geometry-rs/Cargo.lock +++ b/packages/cultmesh-rs/Cargo.lock @@ -3,25 +3,55 @@ version = 4 [[package]] -name = "approx" -version = "0.5.1" +name = "aead" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cab112f0a86d568ea0e627cc1d6be74a1e9cd55214684db5561995f6dad897c6" +checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" dependencies = [ - "num-traits", + "crypto-common", + "generic-array", ] [[package]] -name = "assert_type_match" -version = "0.1.1" +name = "aes" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f548ad2c4031f2902e3edc1f29c29e835829437de49562d8eb5dc5584d3a1043" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", + "cfg-if", + "cipher", + "cpufeatures", ] +[[package]] +name = "aes-gcm" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "831010a0f742e1209b3bcea8fab6a8e149051ba6099432c8cb2cc117dec3ead1" +dependencies = [ + "aead", + "aes", + "cipher", + "ctr", + "ghash", + "subtle", +] + +[[package]] +name = "android_system_properties" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc" +dependencies = [ + "libc", +] + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + [[package]] name = "autocfg" version = "1.5.1" @@ -29,119 +59,22 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" [[package]] -name = "bevy_macro_utils" -version = "0.17.3" +name = "base64" +version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62d984f9f8bd0f0d9fb020492a955e641e30e7a425f3588bf346cb3e61fec3c3" -dependencies = [ - "parking_lot", - "proc-macro2", - "quote", - "syn 2.0.119", - "toml_edit", -] +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" [[package]] -name = "bevy_math" -version = "0.17.3" +name = "base64ct" +version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fa74ae5d968749cc073da991757d3c7e3504ac6dbaac5f8c2a54b9d19b0b7ed" -dependencies = [ - "approx", - "bevy_reflect", - "derive_more", - "glam", - "itertools", - "rand", - "rand_distr", - "serde", - "smallvec", - "thiserror", - "variadics_please", -] - -[[package]] -name = "bevy_platform" -version = "0.17.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4691af6d7cfd1b5deb2fc926a43a180a546cdc3fe1e5a013fcee60db9bb2c81f" -dependencies = [ - "critical-section", - "foldhash", - "futures-channel", - "hashbrown 0.16.1", - "js-sys", - "portable-atomic", - "portable-atomic-util", - "serde", - "spin", - "wasm-bindgen", - "wasm-bindgen-futures", -] - -[[package]] -name = "bevy_ptr" -version = "0.17.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17d24d7906c7de556033168b3485de36c59049fbaef0c2c44c715a23e0329b10" - -[[package]] -name = "bevy_reflect" -version = "0.17.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5472b91928c0f3e4e3988c0d036b00719f19520f53a0c3f8c2af72f00e693c5" -dependencies = [ - "assert_type_match", - "bevy_platform", - "bevy_ptr", - "bevy_reflect_derive", - "bevy_utils", - "derive_more", - "disqualified", - "downcast-rs", - "erased-serde", - "foldhash", - "glam", - "serde", - "smol_str", - "thiserror", - "uuid", - "variadics_please", - "wgpu-types", -] - -[[package]] -name = "bevy_reflect_derive" -version = "0.17.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "083784255162fa39960aa3cf3c23af0e515db2daa7f2e796ae34df993f4d3f6c" -dependencies = [ - "bevy_macro_utils", - "indexmap", - "proc-macro2", - "quote", - "syn 2.0.119", - "uuid", -] - -[[package]] -name = "bevy_utils" -version = "0.17.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "789d04f88c764877a4552e07745b402dbc45f5d0545e6d102558f2f1752a1d89" -dependencies = [ - "bevy_platform", - "disqualified", -] +checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" [[package]] name = "bitflags" version = "2.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" -dependencies = [ - "serde_core", -] [[package]] name = "block-buffer" @@ -159,23 +92,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" [[package]] -name = "bytemuck" -version = "1.25.2" +name = "cc" +version = "1.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797" +checksum = "0ad534f4357a5264cce5019c989cf66a4f0dc4e0d1b1d15f8aacec0ff7360273" dependencies = [ - "bytemuck_derive", -] - -[[package]] -name = "bytemuck_derive" -version = "1.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f65693059b6b9c588b9f62fed1cedbf0a8b805631457ea162d68f0de186f3de5" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", + "find-msvc-tools", + "shlex", ] [[package]] @@ -184,6 +107,42 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "iana-time-zone", + "js-sys", + "num-traits", + "serde", + "wasm-bindgen", + "windows-link", +] + +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common", + "inout", +] + +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + [[package]] name = "cpufeatures" version = "0.2.17" @@ -193,12 +152,6 @@ dependencies = [ "libc", ] -[[package]] -name = "critical-section" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b" - [[package]] name = "crypto-common" version = "0.1.7" @@ -206,30 +159,125 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" dependencies = [ "generic-array", + "rand_core 0.6.4", "typenum", ] [[package]] -name = "derive_more" -version = "2.1.1" +name = "ctr" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d751e9e49156b02b44f9c1815bcb94b984cdcc4396ecc32521c739452808b134" +checksum = "0369ee1ad671834580515889b80f2ea915f23b8be8d0daa4bbaf2ac5c7590835" dependencies = [ - "derive_more-impl", + "cipher", ] [[package]] -name = "derive_more-impl" -version = "2.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "799a97264921d8623a957f6c3b9011f3b5492f557bbb7a5a19b7fa6d06ba8dcb" +name = "cultcache-rs" +version = "0.2.0" +dependencies = [ + "anyhow", + "chrono", + "cultcache-rs-derive", + "fs2", + "redb", + "rmp-serde", + "serde", + "serde_bytes", + "serde_json", + "uuid", + "windows-sys 0.59.0", +] + +[[package]] +name = "cultcache-rs-derive" +version = "0.1.0" dependencies = [ "proc-macro2", "quote", - "rustc_version", "syn 2.0.119", ] +[[package]] +name = "cultmesh-rs" +version = "0.1.0" +dependencies = [ + "anyhow", + "cultcache-rs", + "cultnet-rs", + "pretty_assertions", + "rmp-serde", + "serde", + "tempfile", +] + +[[package]] +name = "cultnet-rs" +version = "0.1.0" +dependencies = [ + "aes-gcm", + "anyhow", + "base64", + "chrono", + "cultcache-rs", + "ed25519-dalek", + "hmac", + "rand", + "rmp-serde", + "rmpv", + "serde", + "serde_bytes", + "serde_json", + "sha2", + "socket2", + "uuid", + "windows-sys 0.61.2", + "zeroize", +] + +[[package]] +name = "curve25519-dalek" +version = "4.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" +dependencies = [ + "cfg-if", + "cpufeatures", + "curve25519-dalek-derive", + "digest", + "fiat-crypto", + "rustc_version", + "subtle", + "zeroize", +] + +[[package]] +name = "curve25519-dalek-derive" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "der" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" +dependencies = [ + "const-oid", + "zeroize", +] + +[[package]] +name = "diff" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" + [[package]] name = "digest" version = "0.10.7" @@ -238,75 +286,88 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", "crypto-common", + "subtle", ] [[package]] -name = "disqualified" -version = "1.0.0" +name = "ed25519" +version = "2.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9c272297e804878a2a4b707cfcfc6d2328b5bb936944613b4fdf2b9269afdfd" - -[[package]] -name = "downcast-rs" -version = "2.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "117240f60069e65410b3ae1bb213295bd828f707b5bec6596a1afc8793ce0cbc" - -[[package]] -name = "either" -version = "1.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" - -[[package]] -name = "equivalent" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" - -[[package]] -name = "erased-serde" -version = "0.4.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2add8a07dd6a8d93ff627029c51de145e12686fbc36ecb298ac22e74cf02dec" +checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53" dependencies = [ + "pkcs8", + "signature", +] + +[[package]] +name = "ed25519-dalek" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70e796c081cee67dc755e1a36a0a172b897fab85fc3f6bc48307991f64e4eca9" +dependencies = [ + "curve25519-dalek", + "ed25519", "serde", - "serde_core", - "typeid", + "sha2", + "subtle", + "zeroize", ] [[package]] -name = "foldhash" -version = "0.2.0" +name = "errno" +version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] [[package]] -name = "futures-channel" -version = "0.3.33" +name = "fastrand" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "262590f4fe6afeb0bc83be1daa64e52657fe185690a958af7f3ad0e92085c5ae" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "fiat-crypto" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" + +[[package]] +name = "find-msvc-tools" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d45db016d36b838f563236e9193d0ee6ce38f3f68b6c94e914b4929c96bbb890" + +[[package]] +name = "fs2" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213" dependencies = [ - "futures-core", + "libc", + "winapi", ] [[package]] name = "futures-core" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cd50c473c80f6d7c3670a752354b8e569b1a7cbfdc0419ec88e5edad85e0dc7" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" [[package]] name = "futures-task" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b231ed28831efb4a61a08580c4bc233ec56bc009f4cd8f52da2c3cb97df0c109" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" [[package]] name = "futures-util" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a77a90a256fce34da66415271e30f94ee91c57b04b8a2c042d9cf3220179deaa" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" dependencies = [ "futures-core", "futures-task", @@ -314,17 +375,6 @@ dependencies = [ "slab", ] -[[package]] -name = "gamecult-geometry" -version = "0.1.0" -dependencies = [ - "approx", - "bevy_math", - "rmp-serde", - "serde", - "sha2", -] - [[package]] name = "generic-array" version = "0.14.7" @@ -335,6 +385,17 @@ dependencies = [ "version_check", ] +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + [[package]] name = "getrandom" version = "0.3.4" @@ -359,57 +420,68 @@ dependencies = [ ] [[package]] -name = "glam" -version = "0.30.10" +name = "ghash" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19fc433e8437a212d1b6f1e68c7824af3aed907da60afa994e7f542d18d12aa9" +checksum = "f0d8a4362ccb29cb0b265253fb0a2728f592895ee6854fd9bc13f2ffda266ff1" dependencies = [ - "bytemuck", - "rand", - "serde_core", + "opaque-debug", + "polyval", ] [[package]] -name = "hashbrown" -version = "0.16.1" +name = "hmac" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" dependencies = [ - "equivalent", - "serde", - "serde_core", + "digest", ] [[package]] -name = "hashbrown" -version = "0.17.1" +name = "iana-time-zone" +version = "0.1.65" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" - -[[package]] -name = "indexmap" -version = "2.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" dependencies = [ - "equivalent", - "hashbrown 0.17.1", + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", ] [[package]] -name = "itertools" -version = "0.14.0" +name = "iana-time-zone-haiku" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" dependencies = [ - "either", + "cc", ] +[[package]] +name = "inout" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +dependencies = [ + "generic-array", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + [[package]] name = "js-sys" -version = "0.3.103" +version = "0.3.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53b44bfcdb3f8d5837a46dae1ca9660a837176eee74a28b229bc626816589102" +checksum = "0e0c1080212aad755ea003d18543e8768dd432c48819efd73a7bf1e39b7a5a3a" dependencies = [ "cfg-if", "futures-util", @@ -423,25 +495,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] -name = "libm" -version = "0.2.16" +name = "linux-raw-sys" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" - -[[package]] -name = "lock_api" -version = "0.4.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" -dependencies = [ - "scopeguard", -] +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" [[package]] name = "log" -version = "0.4.33" +version = "0.4.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" [[package]] name = "memchr" @@ -456,7 +519,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", - "libm", ] [[package]] @@ -466,27 +528,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] -name = "parking_lot" -version = "0.12.5" +name = "opaque-debug" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" -dependencies = [ - "lock_api", - "parking_lot_core", -] - -[[package]] -name = "parking_lot_core" -version = "0.9.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" -dependencies = [ - "cfg-if", - "libc", - "redox_syscall", - "smallvec", - "windows-link", -] +checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" [[package]] name = "pin-project-lite" @@ -495,18 +540,25 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" [[package]] -name = "portable-atomic" -version = "1.14.0" +name = "pkcs8" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d20d5497ef88037a52ff98267d066e7f11fcc5e99bbfbd58a42336193aacec3" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der", + "spki", +] [[package]] -name = "portable-atomic-util" -version = "0.2.7" +name = "polyval" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2a106d1259c23fac8e543272398ae0e3c0b8d33c88ed73d0cc71b0f1d902618" +checksum = "9d1fe60d06143b2430aa532c94cfe9e29783047f06c0d7fd359a9a51b729fa25" dependencies = [ - "portable-atomic", + "cfg-if", + "cpufeatures", + "opaque-debug", + "universal-hash", ] [[package]] @@ -518,6 +570,16 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "pretty_assertions" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ae130e2f271fbc2ac3a40fb1d07180839cdbbe443c7a27e1e3c13c5cac0116d" +dependencies = [ + "diff", + "yansi", +] + [[package]] name = "proc-macro2" version = "1.0.107" @@ -555,7 +617,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" dependencies = [ "rand_chacha", - "rand_core", + "rand_core 0.9.5", ] [[package]] @@ -565,7 +627,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" dependencies = [ "ppv-lite86", - "rand_core", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", ] [[package]] @@ -578,22 +649,12 @@ dependencies = [ ] [[package]] -name = "rand_distr" -version = "0.5.1" +name = "redb" +version = "4.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a8615d50dcf34fa31f7ab52692afec947c4dd0ab803cc87cb3b0b4570ff7463" +checksum = "de6c3b63e007e90ce536ec2ae4690826136a20ec8dbbbb400daef1bb999d2e36" dependencies = [ - "num-traits", - "rand", -] - -[[package]] -name = "redox_syscall" -version = "0.5.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" -dependencies = [ - "bitflags", + "libc", ] [[package]] @@ -615,6 +676,17 @@ dependencies = [ "serde", ] +[[package]] +name = "rmpv" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a4e1d4b9b938a26d2996af33229f0ca0956c652c1375067f0b45291c1df8417" +dependencies = [ + "rmp", + "serde", + "serde_bytes", +] + [[package]] name = "rustc_version" version = "0.4.1" @@ -624,18 +696,25 @@ dependencies = [ "semver", ] +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + [[package]] name = "rustversion" version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" -[[package]] -name = "scopeguard" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" - [[package]] name = "semver" version = "1.0.28" @@ -652,6 +731,16 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde_bytes" +version = "0.11.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5d440709e79d88e51ac01c4b72fc6cb7314017bb7da9eeff678aa94c10e3ea8" +dependencies = [ + "serde", + "serde_core", +] + [[package]] name = "serde_core" version = "1.0.229" @@ -669,7 +758,20 @@ checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn 3.0.3", + "syn 3.0.4", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", ] [[package]] @@ -683,6 +785,21 @@ dependencies = [ "digest", ] +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "signature" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" +dependencies = [ + "rand_core 0.6.4", +] + [[package]] name = "slab" version = "0.4.12" @@ -690,29 +807,31 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" [[package]] -name = "smallvec" -version = "1.15.2" +name = "socket2" +version = "0.5.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" - -[[package]] -name = "smol_str" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd538fb6910ac1099850255cf94a94df6551fbdd602454387d0adb2d1ca6dead" +checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678" dependencies = [ - "serde", + "libc", + "windows-sys 0.52.0", ] [[package]] -name = "spin" -version = "0.10.1" +name = "spki" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "023a211cb3138dbc438680b32560ad89f699977624c9f8dbb95a47d5b4c07dd3" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" dependencies = [ - "portable-atomic", + "base64ct", + "der", ] +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + [[package]] name = "syn" version = "2.0.119" @@ -726,9 +845,9 @@ dependencies = [ [[package]] name = "syn" -version = "3.0.3" +version = "3.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +checksum = "e6275cddf4610d1775e6d1fe9469b2e77d0f39fd98fb7450901b821e0c53649f" dependencies = [ "proc-macro2", "quote", @@ -736,61 +855,18 @@ dependencies = [ ] [[package]] -name = "thiserror" -version = "2.0.19" +name = "tempfile" +version = "3.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09a43598840e33d5b0331f38c5e30d13bb11c11210a4b58f0d9b18a5a5eefcd9" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ - "thiserror-impl", + "fastrand", + "getrandom 0.4.3", + "once_cell", + "rustix", + "windows-sys 0.61.2", ] -[[package]] -name = "thiserror-impl" -version = "2.0.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43cbfe0cf76104d42a574802844187e84a305e531ed54455f11fbde0f10541cd" -dependencies = [ - "proc-macro2", - "quote", - "syn 3.0.3", -] - -[[package]] -name = "toml_datetime" -version = "0.7.5+spec-1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92e1cfed4a3038bc5a127e35a2d360f145e1f4b971b551a2ba5fd7aedf7e1347" -dependencies = [ - "serde_core", -] - -[[package]] -name = "toml_edit" -version = "0.23.10+spec-1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84c8b9f757e028cee9fa244aea147aab2a9ec09d5325a9b01e0a49730c2b5269" -dependencies = [ - "indexmap", - "toml_datetime", - "toml_parser", - "winnow 0.7.15", -] - -[[package]] -name = "toml_parser" -version = "1.1.2+spec-1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526" -dependencies = [ - "winnow 1.0.4", -] - -[[package]] -name = "typeid" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc7d623258602320d5c55d1bc22793b57daff0ec7efc270ea7d55ce1d5f5471c" - [[package]] name = "typenum" version = "1.20.1" @@ -804,26 +880,24 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" [[package]] -name = "uuid" -version = "1.24.0" +name = "universal-hash" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf3923a6f5c4c6382e0b653c4117f48d631ea17f38ed86e2a828e6f7412f5239" +checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" dependencies = [ - "getrandom 0.4.3", - "js-sys", - "serde_core", - "wasm-bindgen", + "crypto-common", + "subtle", ] [[package]] -name = "variadics_please" -version = "1.1.0" +name = "uuid" +version = "1.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41b6d82be61465f97d42bd1d15bf20f3b0a3a0905018f38f9d6f6962055b0b5c" +checksum = "b5772d71c9be8a8a6ac2117d949c5b224c1b72241bb611d9a3012edcf8af7812" dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", + "getrandom 0.4.3", + "js-sys", + "wasm-bindgen", ] [[package]] @@ -832,6 +906,12 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + [[package]] name = "wasip2" version = "1.0.4+wasi-0.2.12" @@ -843,9 +923,9 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.126" +version = "0.2.127" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b067c0c11094aef6b7a801c1e34a26affafdf3d051dba08456b868789aaf9a4" +checksum = "1b70935747edd64d89de3efa29d73789b806c15798f8e7dca4d8ac356b50ce70" dependencies = [ "cfg-if", "once_cell", @@ -854,21 +934,11 @@ dependencies = [ "wasm-bindgen-shared", ] -[[package]] -name = "wasm-bindgen-futures" -version = "0.4.76" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c62df1340f32221cb9c54d6a27b030e3dba64361d4a95bed55f9aacb44da291d" -dependencies = [ - "js-sys", - "wasm-bindgen", -] - [[package]] name = "wasm-bindgen-macro" -version = "0.2.126" +version = "0.2.127" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "167ce5e579f6bcf889c4f7175a8a5a585de84e8ff93976ce393efa5f2837aab1" +checksum = "77775f8f3f7217702089053b94958f8f54061a3f663417df76e19cbdcca29bc1" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -876,9 +946,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.126" +version = "0.2.127" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3997c7839262f4ef12cf90b818d6340c18e80f263f1a94bf157d0ec4420380e" +checksum = "e11d33f857dc2fb11b8bc75aee111aa9cbeb12cd9f25efd3d4c2a3dd4e235284" dependencies = [ "bumpalo", "proc-macro2", @@ -889,36 +959,68 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.126" +version = "0.2.127" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc1b4cb0cc549fcf58d7dfc081778139b3d283a081644e833e84682ad71cea24" +checksum = "7ef64dbcc55df09c7e5a46182d181c2cfa3e925f3da937ea764728b4bbb9dcbf" dependencies = [ "unicode-ident", ] [[package]] -name = "web-sys" -version = "0.3.103" +name = "winapi" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8622dcb61c0bcc9fffa6938bed81210af2da9a7e4a1a834b2e37a59b6dfb6141" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" dependencies = [ - "js-sys", - "wasm-bindgen", + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", ] [[package]] -name = "wgpu-types" -version = "26.0.0" +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eca7a8d8af57c18f57d393601a1fb159ace8b2328f1b6b5f80893f7d672c9ae2" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" dependencies = [ - "bitflags", - "bytemuck", - "js-sys", - "log", - "serde", - "thiserror", - "web-sys", + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", ] [[package]] @@ -928,19 +1030,113 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" [[package]] -name = "winnow" -version = "0.7.15" +name = "windows-result" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" dependencies = [ - "memchr", + "windows-link", ] [[package]] -name = "winnow" -version = "1.0.4" +name = "windows-strings" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "wit-bindgen" @@ -949,21 +1145,39 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" [[package]] -name = "zerocopy" -version = "0.8.55" +name = "yansi" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5a105cd7b140f6eeec8acff2ea38135d3cab283ada58540f629fe51e46696eb" +checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" + +[[package]] +name = "zerocopy" +version = "0.8.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "556764e583adb45a9f8d413c2a147fa7e8d821e48e12b14fd560b607998b75eb" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.55" +version = "0.8.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fe976fb70c78cd64cccfe3a6fc142244e8a77b70959b30faf9d0ac37ee228eb" +checksum = "f2ab42fc20575779bd240faa45f94a74256f755c0fa9e89f0ede20d91d0cdfc1" dependencies = [ "proc-macro2", "quote", "syn 2.0.119", ] + +[[package]] +name = "zeroize" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/packages/cultmesh-rs/Cargo.toml b/packages/cultmesh-rs/Cargo.toml new file mode 100644 index 00000000..820050c3 --- /dev/null +++ b/packages/cultmesh-rs/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "cultmesh-rs" +version = "0.1.0" +edition = "2024" +license = "MIT" +publish = false + +[lib] +name = "cultmesh_rs" +path = "src/lib.rs" + +[dependencies] +anyhow = "1" +cultcache-rs = { path = "../cultcache-rs" } +cultnet-rs = { path = "../cultnet-rs" } +serde = { version = "1", features = ["derive"] } + +[dev-dependencies] +pretty_assertions = "1.4.1" +rmp-serde = "1" +tempfile = "3" diff --git a/packages/cultmesh-rs/src/lib.rs b/packages/cultmesh-rs/src/lib.rs new file mode 100644 index 00000000..8cfe8cec --- /dev/null +++ b/packages/cultmesh-rs/src/lib.rs @@ -0,0 +1,1609 @@ +use anyhow::{Context, Result}; +use cultcache_rs::CultCache; +use cultcache_rs::CultCacheEnvelope; +use cultcache_rs::CultSoaTable; +use cultcache_rs::DatabaseEntry; +use cultcache_rs::SingleFileMessagePackBackingStore; +use cultcache_rs::SoaDocument; +use cultnet_rs::CultNetDocumentPutOptions; +use cultnet_rs::CultNetDocumentRegistry; +use cultnet_rs::CultNetRudpSocketTransportConnection; +use cultnet_rs::CultNetRudpSocketTransportOptions; +use cultnet_rs::CultNetWireContract; +use cultnet_rs::decode_cultnet_message_from_slice; +use cultnet_rs::encode_cultnet_message_to_vec; +use serde::Serialize; +use std::collections::BTreeMap; +use std::collections::hash_map::RandomState; +use std::env; +use std::hash::{BuildHasher, Hasher}; +use std::net::SocketAddr; +use std::net::UdpSocket; +use std::path::Path; +use std::thread; +use std::time::Duration; +use std::time::Instant; + +mod rudp_document_server; + +pub use rudp_document_server::*; + +pub const CULTMESH_RUDP_DOCUMENT_CATALOG_CONNECTION_ID: u32 = 0x0d1d_0002; + +pub trait CultMeshDocumentSet: Clone + Send + Sync + 'static { + fn register_cache(&self, cache: &mut CultCache) -> Result<()>; + fn register_documents(&self, registry: &mut CultNetDocumentRegistry) -> Result<()>; +} + +#[macro_export] +macro_rules! cultmesh_documents { + ($name:ident { $($entry:ty => $schema_version:expr),* $(,)? }) => { + #[derive(Clone, Copy, Debug, Default)] + pub struct $name; + + impl $crate::CultMeshDocumentSet for $name { + fn register_cache( + &self, + cache: &mut cultcache_rs::CultCache, + ) -> ::anyhow::Result<()> { + $( + cache.register_entry_type::<$entry>()?; + )* + Ok(()) + } + + fn register_documents( + &self, + registry: &mut cultnet_rs::CultNetDocumentRegistry, + ) -> ::anyhow::Result<()> { + $( + registry.register( + cultnet_rs::CultNetDocumentBinding::for_entry_with_schema_id::<$entry>( + $schema_version.to_string(), + $schema_version.to_string(), + ), + ); + )* + Ok(()) + } + } + }; +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CultMeshNodeOptions { + pub runtime_id: String, + pub pull_on_start: bool, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CultMeshRudpDocumentPublishOptions { + pub target: SocketAddr, + pub runtime_id: String, + pub connect_timeout: Duration, + pub flush_timeout: Duration, + pub poll_interval: Duration, + pub resend_delay_ms: u64, + pub source_agent_id: Option, + pub source_role: Option, + pub tags: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CultMeshRudpSnapshotOptions { + pub target: SocketAddr, + pub runtime_id: String, + /// Exact correlation id for the SnapshotRequest. Raw callers must supply + /// this; `CultMeshNode::pull_rudp_catalog_snapshot` generates one when it + /// owns the request. + pub message_id: String, + pub connect_timeout: Duration, + pub response_timeout: Duration, + pub poll_interval: Duration, + pub resend_delay_ms: u64, + pub schema_ids: Option>, + pub record_keys: Option>, +} + +impl CultMeshRudpSnapshotOptions { + pub fn odin(target: SocketAddr, runtime_id: impl Into) -> Self { + Self { + target, + runtime_id: runtime_id.into(), + ..Self::default() + } + } +} + +impl Default for CultMeshRudpSnapshotOptions { + fn default() -> Self { + Self { + target: SocketAddr::from(([0, 0, 0, 0], 0)), + runtime_id: "cultmesh-rudp-snapshot-client".to_string(), + message_id: String::new(), + connect_timeout: Duration::from_secs(3), + response_timeout: Duration::from_secs(3), + poll_interval: Duration::from_millis(10), + resend_delay_ms: 50, + schema_ids: None, + record_keys: None, + } + } +} + +impl CultMeshRudpDocumentPublishOptions { + pub fn odin(target: SocketAddr, runtime_id: impl Into) -> Self { + Self { + target, + runtime_id: runtime_id.into(), + ..Self::default() + } + } +} + +impl Default for CultMeshRudpDocumentPublishOptions { + fn default() -> Self { + Self { + target: SocketAddr::from(([0, 0, 0, 0], 0)), + runtime_id: "cultmesh-rudp-document-publisher".to_string(), + connect_timeout: Duration::from_secs(3), + flush_timeout: Duration::from_millis(300), + poll_interval: Duration::from_millis(10), + resend_delay_ms: 50, + source_agent_id: None, + source_role: None, + tags: Vec::new(), + } + } +} + +impl Default for CultMeshNodeOptions { + fn default() -> Self { + Self { + runtime_id: "cultmesh-local".to_string(), + pull_on_start: true, + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CultMeshRudpEndpoint { + pub host: String, + pub port: u16, + pub addr: SocketAddr, +} + +pub struct CultMeshNode { + runtime_id: String, + cache: CultCache, + documents: CultNetDocumentRegistry, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum CultMeshStreamKind { + Audio, + Video, + Tensor, + Bytes, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum CultMeshStreamBodyTransport { + SharedMemory, + SharedD3d12Texture, + SharedD3d11Texture, + DmaBuf, + IoSurface, + AHardwareBuffer, + CultCachePage, + InlineBytes, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum CultMeshStreamAccess { + Read, + Write, + ReadWrite, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum CultMeshStreamCopyBudget { + ZeroCopyTarget, + OneCopyFallback, + OpaqueRuntime, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct CultMeshStreamClock { + pub clock_domain_id: String, + pub source_id: Option, + pub sample_rate: i32, + pub offset_to_verse_time_ns: i64, + pub confidence: f64, + pub evidence_kind: Option, +} + +impl CultMeshStreamClock { + pub fn new(clock_domain_id: impl Into) -> Result { + let clock_domain_id = require_non_empty(clock_domain_id.into(), "clock_domain_id")?; + Ok(Self { + clock_domain_id, + source_id: None, + sample_rate: 0, + offset_to_verse_time_ns: 0, + confidence: 0.0, + evidence_kind: None, + }) + } + + pub fn source_id(mut self, source_id: impl Into) -> Result { + self.source_id = Some(require_non_empty(source_id.into(), "source_id")?); + Ok(self) + } + + pub fn confidence(mut self, confidence: f64) -> Self { + self.confidence = confidence; + self + } + + pub fn evidence_kind(mut self, evidence_kind: impl Into) -> Result { + self.evidence_kind = Some(require_non_empty(evidence_kind.into(), "evidence_kind")?); + Ok(self) + } +} + +#[derive(Clone, Debug, PartialEq)] +pub struct CultMeshStreamDescriptor { + pub stream_id: String, + pub verse_id: String, + pub owner_peer_id: String, + pub kind: CultMeshStreamKind, + pub clock: CultMeshStreamClock, + pub preferred_transports: Vec, + pub label: Option, + pub required_access: CultMeshStreamAccess, + pub max_in_flight_frames: u32, + pub metadata_schema_id: Option, +} + +impl CultMeshStreamDescriptor { + pub fn new( + stream_id: impl Into, + verse_id: impl Into, + owner_peer_id: impl Into, + kind: CultMeshStreamKind, + clock: CultMeshStreamClock, + preferred_transports: Vec, + ) -> Result { + if preferred_transports.is_empty() { + anyhow::bail!("preferred_transports must not be empty"); + } + Ok(Self { + stream_id: require_non_empty(stream_id.into(), "stream_id")?, + verse_id: require_non_empty(verse_id.into(), "verse_id")?, + owner_peer_id: require_non_empty(owner_peer_id.into(), "owner_peer_id")?, + kind, + clock, + preferred_transports, + label: None, + required_access: CultMeshStreamAccess::Read, + max_in_flight_frames: 0, + metadata_schema_id: None, + }) + } + + pub fn label(mut self, label: impl Into) -> Result { + self.label = Some(require_non_empty(label.into(), "label")?); + Ok(self) + } + + pub fn max_in_flight_frames(mut self, max_in_flight_frames: u32) -> Self { + self.max_in_flight_frames = max_in_flight_frames; + self + } + + pub fn metadata_schema_id(mut self, metadata_schema_id: impl Into) -> Result { + self.metadata_schema_id = Some(require_non_empty( + metadata_schema_id.into(), + "metadata_schema_id", + )?); + Ok(self) + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CultMeshStreamConsumerProfile { + pub peer_id: String, + pub verse_id: String, + pub supported_transports: Vec, + pub accepted_kinds: Vec, + pub max_in_flight_frames: u32, +} + +impl CultMeshStreamConsumerProfile { + pub fn new( + peer_id: impl Into, + verse_id: impl Into, + supported_transports: Vec, + ) -> Result { + if supported_transports.is_empty() { + anyhow::bail!("supported_transports must not be empty"); + } + Ok(Self { + peer_id: require_non_empty(peer_id.into(), "peer_id")?, + verse_id: require_non_empty(verse_id.into(), "verse_id")?, + supported_transports, + accepted_kinds: Vec::new(), + max_in_flight_frames: 0, + }) + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CultMeshStreamNegotiation { + pub stream_id: String, + pub producer_peer_id: String, + pub consumer_peer_id: String, + pub transport: CultMeshStreamBodyTransport, + pub access: CultMeshStreamAccess, + pub max_in_flight_frames: u32, + pub copy_budget: CultMeshStreamCopyBudget, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CultMeshStreamFrameHandle { + pub stream_id: String, + pub sequence: u64, + pub timestamp_ns: i64, + pub duration_ns: i64, + pub transport: CultMeshStreamBodyTransport, + pub byte_length: usize, + pub resource_key: Option, + pub unavoidable_copy_count: u32, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CultMeshSharedMemoryFrameRingStats { + pub stream_id: String, + pub slot_count: usize, + pub slot_byte_length: usize, + pub published_frames: u64, + pub dropped_frames: u64, + pub blocked_writes: u64, + pub latest_sequence: u64, + pub unavoidable_copy_count: u64, +} + +pub struct CultMeshFrameReadLease<'a> { + pub handle: CultMeshStreamFrameHandle, + bytes: &'a [u8], +} + +impl CultMeshFrameReadLease<'_> { + pub fn bytes(&self) -> &[u8] { + self.bytes + } +} + +#[derive(Clone, Debug)] +pub struct CultMeshSharedMemoryFrameRing { + stream_id: String, + slot_byte_length: usize, + slots: Vec)>>, + write_cursor: usize, + latest_slot: Option, + next_sequence: u64, + published_frames: u64, + dropped_frames: u64, + blocked_writes: u64, + unavoidable_copy_count: u64, +} + +impl CultMeshSharedMemoryFrameRing { + pub fn new( + stream_id: impl Into, + slot_count: usize, + slot_byte_length: usize, + ) -> Result { + let stream_id = require_non_empty(stream_id.into(), "stream_id")?; + if slot_count == 0 { + anyhow::bail!("slot_count must be greater than zero"); + } + if slot_byte_length == 0 { + anyhow::bail!("slot_byte_length must be greater than zero"); + } + Ok(Self { + stream_id, + slot_byte_length, + slots: vec![None; slot_count], + write_cursor: 0, + latest_slot: None, + next_sequence: 0, + published_frames: 0, + dropped_frames: 0, + blocked_writes: 0, + unavoidable_copy_count: 0, + }) + } + + pub fn try_publish_copy( + &mut self, + bytes: &[u8], + timestamp_ns: i64, + duration_ns: i64, + ) -> Result> { + if bytes.len() > self.slot_byte_length { + anyhow::bail!("frame byte length exceeds slot size"); + } + if self.slots.is_empty() { + self.blocked_writes += 1; + return Ok(None); + } + + let slot_index = self.write_cursor; + if self.slots[slot_index].is_some() { + self.dropped_frames += 1; + } + let handle = CultMeshStreamFrameHandle { + stream_id: self.stream_id.clone(), + sequence: self.next_sequence, + timestamp_ns, + duration_ns, + transport: CultMeshStreamBodyTransport::SharedMemory, + byte_length: bytes.len(), + resource_key: Some(format!("{}:slot:{slot_index}", self.stream_id)), + unavoidable_copy_count: 1, + }; + self.slots[slot_index] = Some((handle.clone(), bytes.to_vec())); + self.latest_slot = Some(slot_index); + self.write_cursor = (slot_index + 1) % self.slots.len(); + self.next_sequence = self.next_sequence.saturating_add(1); + self.published_frames = self.published_frames.saturating_add(1); + self.unavoidable_copy_count = self.unavoidable_copy_count.saturating_add(1); + Ok(Some(handle)) + } + + pub fn try_acquire_latest_read(&self) -> Option> { + let slot_index = self.latest_slot?; + let (handle, bytes) = self.slots[slot_index].as_ref()?; + Some(CultMeshFrameReadLease { + handle: handle.clone(), + bytes, + }) + } + + pub fn stats(&self) -> CultMeshSharedMemoryFrameRingStats { + let latest_sequence = self + .latest_slot + .and_then(|slot| self.slots[slot].as_ref().map(|(handle, _)| handle.sequence)) + .unwrap_or(0); + CultMeshSharedMemoryFrameRingStats { + stream_id: self.stream_id.clone(), + slot_count: self.slots.len(), + slot_byte_length: self.slot_byte_length, + published_frames: self.published_frames, + dropped_frames: self.dropped_frames, + blocked_writes: self.blocked_writes, + latest_sequence, + unavoidable_copy_count: self.unavoidable_copy_count, + } + } +} + +#[derive(Clone, Debug, Default)] +pub struct CultMeshStreamCatalog { + streams: BTreeMap, + latest_frames: BTreeMap, + rings: BTreeMap, +} + +impl CultMeshStreamCatalog { + pub fn declare(&mut self, stream: CultMeshStreamDescriptor) { + self.streams.insert(stream.stream_id.clone(), stream); + } + + pub fn get(&self, stream_id: &str) -> Option<&CultMeshStreamDescriptor> { + self.streams.get(stream_id) + } + + pub fn streams(&self) -> Vec<&CultMeshStreamDescriptor> { + self.streams.values().collect() + } + + pub fn find( + &self, + verse_id: &str, + kind: Option, + ) -> Vec<&CultMeshStreamDescriptor> { + self.streams + .values() + .filter(|stream| stream.verse_id == verse_id && kind.is_none_or(|k| stream.kind == k)) + .collect() + } + + pub fn negotiate( + &self, + stream_id: &str, + consumer: &CultMeshStreamConsumerProfile, + ) -> Result { + let stream = self + .streams + .get(stream_id) + .ok_or_else(|| anyhow::anyhow!("unknown CultMesh stream '{stream_id}'"))?; + if stream.verse_id != consumer.verse_id { + anyhow::bail!("stream and consumer must belong to the same Verse"); + } + if !consumer.accepted_kinds.is_empty() && !consumer.accepted_kinds.contains(&stream.kind) { + anyhow::bail!("consumer does not accept stream kind"); + } + let transport = stream + .preferred_transports + .iter() + .copied() + .find(|candidate| consumer.supported_transports.contains(candidate)) + .ok_or_else(|| { + anyhow::anyhow!("stream and consumer have no compatible body transport") + })?; + let producer_max = if stream.max_in_flight_frames == 0 { + u32::MAX + } else { + stream.max_in_flight_frames + }; + let consumer_max = if consumer.max_in_flight_frames == 0 { + u32::MAX + } else { + consumer.max_in_flight_frames + }; + Ok(CultMeshStreamNegotiation { + stream_id: stream.stream_id.clone(), + producer_peer_id: stream.owner_peer_id.clone(), + consumer_peer_id: consumer.peer_id.clone(), + transport, + access: stream.required_access, + max_in_flight_frames: producer_max.min(consumer_max), + copy_budget: copy_budget_for(transport), + }) + } + + pub fn create_shared_memory_ring( + &mut self, + stream_id: &str, + slot_count: usize, + slot_byte_length: usize, + ) -> Result<&mut CultMeshSharedMemoryFrameRing> { + if !self.streams.contains_key(stream_id) { + anyhow::bail!("unknown CultMesh stream '{stream_id}'"); + } + self.rings.insert( + stream_id.to_string(), + CultMeshSharedMemoryFrameRing::new(stream_id, slot_count, slot_byte_length)?, + ); + Ok(self + .rings + .get_mut(stream_id) + .expect("ring was inserted for stream")) + } + + pub fn ring(&self, stream_id: &str) -> Option<&CultMeshSharedMemoryFrameRing> { + self.rings.get(stream_id) + } + + pub fn ring_mut(&mut self, stream_id: &str) -> Option<&mut CultMeshSharedMemoryFrameRing> { + self.rings.get_mut(stream_id) + } + + pub fn publish_frame(&mut self, handle: CultMeshStreamFrameHandle) -> Result<()> { + if !self.streams.contains_key(&handle.stream_id) { + anyhow::bail!("unknown CultMesh stream '{}'", handle.stream_id); + } + self.latest_frames.insert(handle.stream_id.clone(), handle); + Ok(()) + } + + pub fn latest_frame(&self, stream_id: &str) -> Option<&CultMeshStreamFrameHandle> { + self.latest_frames.get(stream_id) + } +} + +fn copy_budget_for(transport: CultMeshStreamBodyTransport) -> CultMeshStreamCopyBudget { + match transport { + CultMeshStreamBodyTransport::SharedMemory + | CultMeshStreamBodyTransport::SharedD3d12Texture + | CultMeshStreamBodyTransport::SharedD3d11Texture + | CultMeshStreamBodyTransport::DmaBuf + | CultMeshStreamBodyTransport::IoSurface + | CultMeshStreamBodyTransport::AHardwareBuffer => CultMeshStreamCopyBudget::ZeroCopyTarget, + CultMeshStreamBodyTransport::CultCachePage => CultMeshStreamCopyBudget::OneCopyFallback, + CultMeshStreamBodyTransport::InlineBytes => CultMeshStreamCopyBudget::OpaqueRuntime, + } +} + +fn require_non_empty(value: String, name: &str) -> Result { + if value.trim().is_empty() { + anyhow::bail!("{name} must be non-empty"); + } + Ok(value) +} + +impl CultMeshNode { + pub fn runtime_id(&self) -> &str { + &self.runtime_id + } + + pub fn cache(&self) -> &CultCache { + &self.cache + } + + pub fn documents(&self) -> &CultNetDocumentRegistry { + &self.documents + } + + pub fn get(&self, key: &str) -> Result> { + self.cache.get::(key) + } + + pub fn get_required(&self, key: &str) -> Result { + self.cache.get_required::(key) + } + + pub fn get_all_with_keys(&self) -> Result> { + self.cache.get_all_with_keys::() + } + + pub fn soa(&self) -> Result> { + self.cache.soa::() + } + + pub fn put(&mut self, key: impl Into, value: &T) -> Result { + self.cache.put(key, value) + } + + pub fn pull_all_backing_stores(&mut self) -> Result<()> { + self.cache.pull_all_backing_stores() + } + + pub fn delete(&mut self, key: &str) -> Result { + self.cache.delete::(key) + } + + pub fn flush(&mut self) -> Result<()> { + Ok(()) + } + + pub fn publish_document_to_rudp_catalog( + &self, + key: impl Into, + value: &T, + options: CultMeshRudpDocumentPublishOptions, + ) -> Result<()> + where + T: DatabaseEntry + Serialize, + { + let message = self.create_rudp_document_message(key, value, &options)?; + publish_cultnet_message_to_rudp_catalog(&message, options) + } + + pub fn create_rudp_document_message( + &self, + key: impl Into, + value: &T, + options: &CultMeshRudpDocumentPublishOptions, + ) -> Result + where + T: DatabaseEntry + Serialize, + { + let key = key.into(); + self.documents.create_raw_document_put_message( + format!("{}:{}:{}", options.runtime_id, T::TYPE, key), + key, + value, + CultNetDocumentPutOptions { + source_runtime_id: Some(options.runtime_id.clone()), + source_agent_id: options.source_agent_id.clone(), + source_role: options.source_role.clone(), + tags: if options.tags.is_empty() { + None + } else { + Some(options.tags.clone()) + }, + ..CultNetDocumentPutOptions::default() + }, + ) + } + + pub fn pull_rudp_catalog_snapshot( + &mut self, + mut options: CultMeshRudpSnapshotOptions, + ) -> Result { + if options.message_id.trim().is_empty() { + options.message_id = format!( + "{}:snapshot:{}:{}", + options.runtime_id, + unix_millis(), + fresh_rudp_connection_epoch() + ); + } + let response = request_raw_snapshot_from_rudp_catalog(options)?; + let cultnet_rs::CultNetMessage::SnapshotResponseRaw { documents, .. } = response else { + anyhow::bail!("expected cultnet.snapshot_response_raw.v0"); + }; + let mut applied = 0usize; + for document in documents { + let Some(document_type) = + registered_document_type_for_schema(&self.documents, &document.schema_id) + else { + continue; + }; + self.cache.put_raw_envelope(CultCacheEnvelope { + key: document.record_key, + r#type: document_type, + payload: document.payload, + stored_at: document.stored_at, + schema_id: Some(document.schema_id), + })?; + applied += 1; + } + Ok(applied) + } +} + +fn registered_document_type_for_schema( + documents: &CultNetDocumentRegistry, + schema_id: &str, +) -> Option { + let normalized = strip_schema_version_suffix(schema_id); + if normalized != schema_id { + return Some(normalized.to_string()); + } + documents + .binding_by_schema_id(schema_id) + .map(|binding| binding.document_type.clone()) + .or_else(|| documents.binding(schema_id).map(|_| schema_id.to_string())) +} + +fn strip_schema_version_suffix(value: &str) -> &str { + let Some((prefix, suffix)) = value.rsplit_once(".v") else { + return value; + }; + if suffix.chars().all(|character| character.is_ascii_digit()) { + prefix + } else { + value + } +} + +/// Request a raw CultNet snapshot without importing it into a local cache. +/// +/// The returned value is the exact validated `SnapshotResponseRaw`, including +/// its correlation `message_id`. Each call binds a fresh source port, chooses a +/// fresh connection epoch, and succeeds only after the reliable request is +/// acknowledged. +pub fn request_raw_snapshot_from_rudp_catalog( + options: CultMeshRudpSnapshotOptions, +) -> Result { + if options.runtime_id.trim().is_empty() { + anyhow::bail!("runtime_id must be non-empty"); + } + if options.message_id.trim().is_empty() { + anyhow::bail!("raw CultMesh RUDP snapshot requires a message_id"); + } + let message_id = options.message_id.clone(); + let mut client = connect_rudp_client( + options.target, + &options.runtime_id, + options.connect_timeout, + options.poll_interval, + options.resend_delay_ms, + "cultmesh-rudp-snapshot-client", + )?; + + let request = cultnet_rs::CultNetMessage::SnapshotRequest { + message_id: message_id.clone(), + schema_ids: options.schema_ids, + record_keys: options.record_keys, + }; + let payload = encode_cultnet_message_to_vec(&request, CultNetWireContract::CultNetSchemaV0)?; + let receipt = client.send_reliable("schema", payload)?; + let response_deadline = Instant::now() + options.response_timeout; + let mut snapshot_response = None; + while Instant::now() < response_deadline { + if let Some(frame) = client.receive_once()? { + if frame.channel_id == "schema" { + let message = decode_cultnet_message_from_slice( + &frame.payload, + CultNetWireContract::CultNetSchemaV0, + )?; + match &message { + cultnet_rs::CultNetMessage::SnapshotResponseRaw { + message_id: received, + .. + } if received == &message_id => { + snapshot_response = Some(message); + } + cultnet_rs::CultNetMessage::Error { error } => { + let _ = client.disconnect(b"snapshot-error".to_vec()); + anyhow::bail!("CultMesh RUDP snapshot failed: {error}"); + } + _ => {} + } + } + } + match client.reliable_send_status(&receipt) { + cultnet_rs::CultNetRudpReliableSendStatus::Acknowledged + if snapshot_response.is_some() => + { + let _ = client.disconnect(b"snapshot-complete".to_vec()); + return Ok(snapshot_response.expect("response was checked as present")); + } + cultnet_rs::CultNetRudpReliableSendStatus::Invalidated => { + anyhow::bail!("CultMesh RUDP snapshot request receipt was invalidated") + } + _ => {} + } + client.poll_resends()?; + thread::sleep(options.poll_interval); + } + let _ = client.disconnect(b"snapshot-timeout".to_vec()); + if snapshot_response.is_some() { + anyhow::bail!( + "timed out waiting for reliable acknowledgement of CultMesh RUDP snapshot request to {}", + options.target + ); + } + anyhow::bail!( + "timed out waiting for CultMesh RUDP snapshot from {}", + options.target + ) +} + +fn unix_millis() -> u128 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_millis() +} + +/// Publish one raw CultNet schema message to a RUDP catalog. +/// +/// Each call owns a fresh connection epoch and returns only after the catalog +/// acknowledges reliable application admission. +pub fn publish_cultnet_message_to_rudp_catalog( + message: &cultnet_rs::CultNetMessage, + options: CultMeshRudpDocumentPublishOptions, +) -> Result<()> { + if options.runtime_id.trim().is_empty() { + anyhow::bail!("runtime_id must be non-empty"); + } + let mut client = connect_rudp_client( + options.target, + &options.runtime_id, + options.connect_timeout, + options.poll_interval, + options.resend_delay_ms, + "cultmesh-rudp-document-publisher", + )?; + + let payload = encode_cultnet_message_to_vec(message, CultNetWireContract::CultNetSchemaV0)?; + let receipt = client.send_reliable("schema", payload)?; + let flush_deadline = Instant::now() + options.flush_timeout; + while Instant::now() < flush_deadline { + let _ = client.receive_once()?; + match client.reliable_send_status(&receipt) { + cultnet_rs::CultNetRudpReliableSendStatus::Acknowledged => { + let _ = client.disconnect(b"document-acknowledged".to_vec()); + return Ok(()); + } + cultnet_rs::CultNetRudpReliableSendStatus::Invalidated => { + anyhow::bail!("CultMesh RUDP document receipt was invalidated") + } + cultnet_rs::CultNetRudpReliableSendStatus::Pending => {} + } + client.poll_resends()?; + thread::sleep(options.poll_interval); + } + let _ = client.disconnect(b"document-unacknowledged".to_vec()); + anyhow::bail!( + "timed out waiting for reliable acknowledgement from CultMesh RUDP catalog {}", + options.target + ) +} + +/// Publish a batch of raw CultNet schema messages to a RUDP catalog. +/// +/// Unlike [`publish_cultnet_message_to_rudp_catalog`], which polls for +/// per-message application-level acknowledgement, this sends every message on +/// one connection and waits only for a transport-level `flush_reliable`. It +/// trades per-receipt confirmation for batch throughput; callers that need a +/// stronger delivery guarantee should use the singular publish path instead. +pub fn publish_cultnet_messages_to_rudp_catalog( + messages: &[cultnet_rs::CultNetMessage], + options: CultMeshRudpDocumentPublishOptions, +) -> Result<()> { + if options.runtime_id.trim().is_empty() { + anyhow::bail!("runtime_id must be non-empty"); + } + let mut client = connect_rudp_client( + options.target, + &options.runtime_id, + options.connect_timeout, + options.poll_interval, + options.resend_delay_ms, + "cultmesh-rudp-document-publisher", + )?; + + for message in messages { + let payload = encode_cultnet_message_to_vec(message, CultNetWireContract::CultNetSchemaV0)?; + client.send("schema", payload)?; + } + client.flush_reliable(options.flush_timeout) +} + +fn connect_rudp_client( + target: SocketAddr, + runtime_id: &str, + connect_timeout: Duration, + poll_interval: Duration, + resend_delay_ms: u64, + transport_id: &str, +) -> Result { + let bind_addr = if target.is_ipv4() { + SocketAddr::from(([0, 0, 0, 0], 0)) + } else { + SocketAddr::from(([0, 0, 0, 0, 0, 0, 0, 0], 0)) + }; + let socket = UdpSocket::bind(bind_addr)?; + socket.set_read_timeout(Some(poll_interval))?; + let mut client = + CultNetRudpSocketTransportConnection::new(CultNetRudpSocketTransportOptions { + media_delivery: None, + runtime_id: runtime_id.to_string(), + socket, + mode: cultnet_rs::CultNetRudpSocketMode::Client, + remote_addr: Some(target), + connection_id: fresh_rudp_connection_epoch(), + initial_sequence: 1, + resend_delay_ms, + transport_id: Some(transport_id.to_string()), + max_payload_bytes: None, + max_fragment_bytes: Some(1200), + max_pending_reliable_packets: Some(1024), + media_reliable_expire_after_ms: None, + reconnect_policy: None, + })?; + client.connect(Vec::new())?; + let connect_deadline = Instant::now() + connect_timeout; + while !client.connected() && Instant::now() < connect_deadline { + let _ = client.receive_once()?; + client.poll_resends()?; + thread::sleep(poll_interval); + } + if !client.connected() { + anyhow::bail!("timed out connecting CultMesh RUDP client {runtime_id} to {target}"); + } + Ok(client) +} + +fn fresh_rudp_connection_epoch() -> u32 { + let mut hasher = RandomState::new().build_hasher(); + hasher.write_u128(unix_millis()); + let epoch = hasher.finish() as u32; + if epoch == 0 { 1 } else { epoch } +} + +#[derive(Clone, Copy, Debug, Default)] +pub struct CultMesh; + +impl CultMesh { + pub fn resolve_rudp_endpoint(endpoint: &str) -> Result { + let text = require_non_empty(endpoint.trim().to_string(), "endpoint")?; + if text.starts_with("rudp://") { + return parse_rudp_socket_addr(&text); + } + if text.starts_with("cultmesh://") { + let resolved = default_cultmesh_rudp_endpoint_resolver(&text)?.ok_or_else(|| { + anyhow::anyhow!("CultMesh URI {text} did not resolve to a RUDP endpoint") + })?; + return parse_rudp_socket_addr(&resolved); + } + parse_socket_addr(&text) + } + + pub fn create_node( + store_path: impl AsRef, + documents: D, + options: CultMeshNodeOptions, + ) -> Result + where + D: CultMeshDocumentSet, + { + let mut cache = CultCache::new(); + documents.register_cache(&mut cache)?; + cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(store_path.as_ref()))?; + if options.pull_on_start { + cache.pull_all_backing_stores()?; + } + + let mut registry = CultNetDocumentRegistry::new(); + documents.register_documents(&mut registry)?; + + Ok(CultMeshNode { + runtime_id: options.runtime_id, + cache, + documents: registry, + }) + } + + pub fn start_node( + store_path: impl AsRef, + documents: D, + options: CultMeshNodeOptions, + ) -> Result + where + D: CultMeshDocumentSet, + { + Self::create_node(store_path, documents, options) + } + + pub fn create_stream_catalog() -> CultMeshStreamCatalog { + CultMeshStreamCatalog::default() + } +} + +pub fn create_node( + store_path: impl AsRef, + documents: D, + options: CultMeshNodeOptions, +) -> Result +where + D: CultMeshDocumentSet, +{ + CultMesh::create_node(store_path, documents, options) +} + +pub fn start_node( + store_path: impl AsRef, + documents: D, + options: CultMeshNodeOptions, +) -> Result +where + D: CultMeshDocumentSet, +{ + CultMesh::start_node(store_path, documents, options) +} + +pub fn create_stream_catalog() -> CultMeshStreamCatalog { + CultMesh::create_stream_catalog() +} + +pub fn resolve_rudp_endpoint(endpoint: &str) -> Result { + CultMesh::resolve_rudp_endpoint(endpoint) +} + +fn parse_rudp_socket_addr(value: &str) -> Result { + let text = value.trim().strip_prefix("rudp://").unwrap_or(value.trim()); + parse_socket_addr(text) +} + +fn parse_socket_addr(value: &str) -> Result { + value + .parse() + .with_context(|| format!("RUDP endpoint must be a socket address, got {value:?}")) +} + +fn default_cultmesh_rudp_endpoint_resolver(uri: &str) -> Result> { + let authority = cultmesh_uri_authority(uri)?; + let slug = authority + .chars() + .map(|ch| { + if ch.is_ascii_alphanumeric() { + ch.to_ascii_uppercase() + } else { + '_' + } + }) + .collect::() + .trim_matches('_') + .to_string(); + if slug.is_empty() { + return Ok(None); + } + + for key in [ + format!("CULTMESH_URI_{slug}_RUDP"), + format!("{slug}_CULTMESH_RUDP_ENDPOINT"), + ] { + if let Ok(value) = env::var(&key) { + let text = value.trim(); + if !text.is_empty() { + return Ok(Some(if text.starts_with("rudp://") { + text.to_string() + } else { + format!("rudp://{text}") + })); + } + } + } + Ok(None) +} + +fn cultmesh_uri_authority(uri: &str) -> Result { + let rest = uri + .strip_prefix("cultmesh://") + .ok_or_else(|| anyhow::anyhow!("CultMesh URI must start with cultmesh://"))?; + let authority = rest.split(['/', '?', '#']).next().unwrap_or("").trim(); + if authority.is_empty() { + anyhow::bail!("CultMesh URI must include an authority"); + } + Ok(authority.to_string()) +} + +#[cfg(test)] +mod tests { + use super::*; + use pretty_assertions::assert_eq; + + #[derive(Clone, Debug, PartialEq, Eq, cultcache_rs::DatabaseEntry)] + #[cultcache(type = "cultmesh.test.note", schema = "CultMeshTestNote")] + struct Note { + #[cultcache(key = 0)] + body: String, + } + + cultmesh_documents!(TestDocuments { + Note => "cultmesh.test.note.v0", + }); + + #[test] + fn node_round_trips_registered_documents() -> Result<()> { + let temp = tempfile::tempdir()?; + let store_path = temp.path().join("cultmesh.cc"); + let note = Note { + body: "blessed circuit".to_string(), + }; + + let mut node = CultMesh::create_node( + &store_path, + TestDocuments, + CultMeshNodeOptions { + runtime_id: "test-runtime".to_string(), + ..CultMeshNodeOptions::default() + }, + )?; + assert_eq!(node.runtime_id(), "test-runtime"); + node.put("note", ¬e)?; + node.flush()?; + + let reloaded = CultMesh::create_node(&store_path, TestDocuments, Default::default())?; + assert_eq!(reloaded.get_required::("note")?, note); + assert!( + reloaded + .documents() + .binding_by_schema_id("cultmesh.test.note.v0") + .is_some() + ); + Ok(()) + } + + #[test] + fn node_publishes_registered_document_to_rudp_catalog() -> Result<()> { + let temp = tempfile::tempdir()?; + let store_path = temp.path().join("cultmesh.cc"); + let note = Note { + body: "respect odin once".to_string(), + }; + let node = CultMesh::create_node( + &store_path, + TestDocuments, + CultMeshNodeOptions { + runtime_id: "muninn-test".to_string(), + ..CultMeshNodeOptions::default() + }, + )?; + + let socket = UdpSocket::bind("127.0.0.1:0")?; + socket.set_read_timeout(Some(Duration::from_millis(10)))?; + let target = socket.local_addr()?; + let (sender, receiver) = std::sync::mpsc::channel(); + let delivered = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); + let delivered_by_sink = delivered.clone(); + let server = thread::spawn(move || -> Result<()> { + let mut server = CultMeshRudpDocumentServer::new( + socket, + move |receipt: CultMeshRudpRawDocumentReceipt| { + sender + .send(( + receipt.document.schema_id, + receipt.document.record_key, + receipt.document.source_runtime_id, + )) + .ok(); + delivered_by_sink.store(true, std::sync::atomic::Ordering::SeqCst); + Ok(()) + }, + |_query: &CultMeshRudpSnapshotQuery| Ok(Vec::new()), + CultMeshSystemClock::default(), + CultMeshRudpDocumentServerOptions::default(), + )?; + let deadline = Instant::now() + Duration::from_secs(2); + while Instant::now() < deadline { + server.poll_once()?; + if delivered.load(std::sync::atomic::Ordering::SeqCst) { + return Ok(()); + } + thread::sleep(Duration::from_millis(1)); + } + anyhow::bail!("timed out waiting for RUDP document put") + }); + + node.publish_document_to_rudp_catalog( + "note", + ¬e, + CultMeshRudpDocumentPublishOptions::odin(target, "muninn-test"), + )?; + let received = receiver.recv_timeout(Duration::from_secs(2))?; + server.join().expect("server thread should not panic")?; + + assert_eq!(received.0, "cultmesh.test.note.v0"); + assert_eq!(received.1, "note"); + assert_eq!(received.2.as_deref(), Some("muninn-test")); + Ok(()) + } + + #[test] + fn document_publish_fails_when_catalog_is_unreachable() -> Result<()> { + let temp = tempfile::tempdir()?; + let node = CultMesh::create_node( + temp.path().join("cultmesh.cc"), + TestDocuments, + Default::default(), + )?; + let unused = UdpSocket::bind("127.0.0.1:0")?; + let target = unused.local_addr()?; + drop(unused); + + let error = node + .publish_document_to_rudp_catalog( + "note", + &Note { + body: "no false success".into(), + }, + CultMeshRudpDocumentPublishOptions { + target, + connect_timeout: Duration::from_millis(30), + poll_interval: Duration::from_millis(2), + ..CultMeshRudpDocumentPublishOptions::default() + }, + ) + .expect_err("unreachable catalog must fail closed"); + assert!(error.to_string().contains("timed out connecting")); + Ok(()) + } + + #[test] + fn document_publish_requires_application_acknowledgement() -> Result<()> { + let temp = tempfile::tempdir()?; + let node = CultMesh::create_node( + temp.path().join("cultmesh.cc"), + TestDocuments, + Default::default(), + )?; + let socket = UdpSocket::bind("127.0.0.1:0")?; + let target = socket.local_addr()?; + let stop = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); + let server_stop = stop.clone(); + let server = thread::spawn(move || -> Result<()> { + let mut server = CultMeshRudpDocumentServer::new( + socket, + |_receipt: CultMeshRudpRawDocumentReceipt| { + anyhow::bail!("injected durable sink failure") + }, + |_query: &CultMeshRudpSnapshotQuery| Ok(Vec::new()), + CultMeshSystemClock::default(), + CultMeshRudpDocumentServerOptions::default(), + )?; + while !server_stop.load(std::sync::atomic::Ordering::SeqCst) { + let _ = server.poll_once(); + thread::sleep(Duration::from_millis(1)); + } + Ok(()) + }); + + let error = node + .publish_document_to_rudp_catalog( + "note", + &Note { + body: "still not accepted".into(), + }, + CultMeshRudpDocumentPublishOptions { + target, + connect_timeout: Duration::from_millis(100), + flush_timeout: Duration::from_millis(50), + poll_interval: Duration::from_millis(2), + resend_delay_ms: 5, + ..CultMeshRudpDocumentPublishOptions::default() + }, + ) + .expect_err("unacknowledged document must fail closed"); + stop.store(true, std::sync::atomic::Ordering::SeqCst); + server.join().expect("server thread should not panic")?; + assert!(error.to_string().contains("reliable acknowledgement")); + Ok(()) + } + + #[test] + fn raw_snapshot_preserves_caller_challenge_and_raw_response() -> Result<()> { + let expected = cultnet_rs::CultNetRawDocumentRecord { + schema_id: "cultmesh.test.raw.v0".into(), + record_key: "route-proof".into(), + stored_at: "2026-09-03T00:00:00Z".into(), + payload_encoding: cultnet_rs::CultNetRawPayloadEncoding::Messagepack, + payload: vec![0xc4, 0x03, 0, 0xff, 1], + source_runtime_id: Some("odin-test".into()), + source_agent_id: None, + source_role: None, + tags: None, + }; + let socket = UdpSocket::bind("127.0.0.1:0")?; + let target = socket.local_addr()?; + let challenge = "9aa63ad8-83b5-4cc4-ae4a-91d361d5f845".to_string(); + let server_document = expected.clone(); + let server_challenge = challenge.clone(); + let stop = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); + let server_stop = stop.clone(); + let server = thread::spawn(move || -> Result<()> { + let mut server = CultMeshRudpDocumentServer::new( + socket, + |_receipt: CultMeshRudpRawDocumentReceipt| Ok(()), + move |query: &CultMeshRudpSnapshotQuery| { + if query.message_id != server_challenge { + anyhow::bail!("snapshot challenge changed") + } + Ok(vec![server_document.clone()]) + }, + CultMeshSystemClock::default(), + CultMeshRudpDocumentServerOptions::default(), + )?; + while !server_stop.load(std::sync::atomic::Ordering::SeqCst) { + server.poll_once()?; + thread::sleep(Duration::from_millis(1)); + } + Ok(()) + }); + + let response = request_raw_snapshot_from_rudp_catalog(CultMeshRudpSnapshotOptions { + target, + runtime_id: "route-proof-test".into(), + message_id: challenge.clone(), + connect_timeout: Duration::from_millis(100), + response_timeout: Duration::from_millis(500), + poll_interval: Duration::from_millis(2), + resend_delay_ms: 5, + schema_ids: Some(vec!["cultmesh.test.raw.v0".into()]), + record_keys: Some(vec!["route-proof".into()]), + }); + stop.store(true, std::sync::atomic::Ordering::SeqCst); + server.join().expect("server thread should not panic")?; + let response = response?; + assert_eq!( + response, + cultnet_rs::CultNetMessage::SnapshotResponseRaw { + message_id: challenge, + documents: vec![expected], + } + ); + Ok(()) + } + + #[test] + fn raw_snapshot_rejects_response_without_request_acknowledgement() -> Result<()> { + let socket = UdpSocket::bind("127.0.0.1:0")?; + socket.set_read_timeout(Some(Duration::from_millis(200)))?; + let target = socket.local_addr()?; + let server = thread::spawn(move || -> Result<()> { + let mut wire = vec![0_u8; 65_535]; + let (received, peer) = socket.recv_from(&mut wire)?; + let connect = cultnet_rs::decode_rudp_packet(&wire[..received])?; + let mut session = + cultnet_rs::CultNetRudpSession::new(cultnet_rs::CultNetRudpSessionOptions { + connection_id: connect.connection_id, + initial_sequence: 1, + resend_delay_ms: 5, + max_pending_reliable_packets: Some(64), + }); + let accept = session.accept_connect(&connect, 1, Vec::new())?; + socket.send_to(&cultnet_rs::encode_rudp_packet(&accept)?, peer)?; + + loop { + let (received, received_peer) = socket.recv_from(&mut wire)?; + if received_peer != peer { + continue; + } + let packet = cultnet_rs::decode_rudp_packet(&wire[..received])?; + let result = session.receive(&packet, 2)?; + let Some(frame) = result.delivered.into_iter().next() else { + continue; + }; + let request = decode_cultnet_message_from_slice( + &frame.payload, + CultNetWireContract::CultNetSchemaV0, + )?; + let cultnet_rs::CultNetMessage::SnapshotRequest { message_id, .. } = request else { + anyhow::bail!("expected snapshot request") + }; + let payload = encode_cultnet_message_to_vec( + &cultnet_rs::CultNetMessage::SnapshotResponseRaw { + message_id, + documents: Vec::new(), + }, + CultNetWireContract::CultNetSchemaV0, + )?; + let mut packets = session.send_many( + "schema", + payload, + cultnet_rs::CultNetRudpSendOptions { + reliable: true, + ordered: true, + sequenced: false, + now_ms: 2, + reliable_expire_after_ms: None, + }, + Some(1200), + )?; + for packet in &mut packets { + packet.ack = 0; + packet.ack_mask = 0; + socket.send_to(&cultnet_rs::encode_rudp_packet(packet)?, peer)?; + } + return Ok(()); + } + }); + + let error = request_raw_snapshot_from_rudp_catalog(CultMeshRudpSnapshotOptions { + target, + runtime_id: "unacknowledged-snapshot-test".into(), + message_id: "b7987efb-69c5-4c42-9355-b6e09535ad21".into(), + connect_timeout: Duration::from_millis(100), + response_timeout: Duration::from_millis(50), + poll_interval: Duration::from_millis(2), + resend_delay_ms: 5, + schema_ids: None, + record_keys: None, + }) + .expect_err("response without request ACK must fail closed"); + server.join().expect("server thread should not panic")?; + assert!(error.to_string().contains("reliable acknowledgement")); + Ok(()) + } + + #[test] + fn resolves_rudp_endpoint_from_cultmesh_uri_bootstrap_env() -> Result<()> { + let previous = env::var("CULTMESH_URI_ODIN_RUDP").ok(); + unsafe { + env::set_var("CULTMESH_URI_ODIN_RUDP", "127.0.0.1:17871"); + } + let result = CultMesh::resolve_rudp_endpoint("cultmesh://odin/rendezvous/provider-catalog"); + unsafe { + if let Some(value) = previous { + env::set_var("CULTMESH_URI_ODIN_RUDP", value); + } else { + env::remove_var("CULTMESH_URI_ODIN_RUDP"); + } + } + + assert_eq!(result?, "127.0.0.1:17871".parse()?); + Ok(()) + } + + #[test] + fn rejects_unresolved_cultmesh_rudp_endpoint() { + let previous = env::var("CULTMESH_URI_MISSING_ODIN_RUDP").ok(); + unsafe { + env::remove_var("CULTMESH_URI_MISSING_ODIN_RUDP"); + } + let error = + CultMesh::resolve_rudp_endpoint("cultmesh://missing-odin/rendezvous/provider-catalog") + .expect_err("unresolved CultMesh URI should fail"); + unsafe { + if let Some(value) = previous { + env::set_var("CULTMESH_URI_MISSING_ODIN_RUDP", value); + } + } + assert!(error.to_string().contains("did not resolve")); + } + + #[test] + fn raw_snapshot_application_uses_registered_document_type() -> Result<()> { + let temp = tempfile::tempdir()?; + let store_path = temp.path().join("cultmesh.cc"); + let mut node = CultMesh::create_node( + &store_path, + TestDocuments, + CultMeshNodeOptions { + runtime_id: "snapshot-apply-test".to_string(), + ..CultMeshNodeOptions::default() + }, + )?; + let payload = rmp_serde::to_vec(&Note { + body: "catalog truth".to_string(), + })?; + + node.cache + .put_raw_envelope(cultcache_rs::CultCacheEnvelope { + key: "note".to_string(), + r#type: node + .documents + .binding_by_schema_id("cultmesh.test.note.v0") + .expect("test binding") + .document_type + .clone(), + payload, + stored_at: "2026-07-03T00:00:00Z".to_string(), + schema_id: Some("cultmesh.test.note.v0".to_string()), + })?; + node.flush()?; + + let reloaded = CultMesh::create_node(&store_path, TestDocuments, Default::default())?; + assert_eq!(reloaded.get_required::("note")?.body, "catalog truth"); + assert_eq!(reloaded.cache().snapshot()[0].r#type, Note::TYPE); + Ok(()) + } + + #[test] + fn stream_catalog_negotiates_and_publishes_latest_shared_memory_frame() -> Result<()> { + let mut catalog = CultMesh::create_stream_catalog(); + let stream = CultMeshStreamDescriptor::new( + "muninn:nightwing:move-evidence", + "mimir-live", + "muninn:nightwing", + CultMeshStreamKind::Bytes, + CultMeshStreamClock::new("muninn:nightwing:clock")? + .source_id("muninn:nightwing:move-evidence")? + .confidence(1.0) + .evidence_kind("muninn-move-evidence")?, + vec![ + CultMeshStreamBodyTransport::SharedMemory, + CultMeshStreamBodyTransport::CultCachePage, + ], + )? + .label("Muninn Move evidence")? + .max_in_flight_frames(4) + .metadata_schema_id("mimir.muninn_move_evidence_stream_frame.v1")?; + catalog.declare(stream); + + let consumer = CultMeshStreamConsumerProfile::new( + "mimir:starfire", + "mimir-live", + vec![CultMeshStreamBodyTransport::SharedMemory], + )?; + let negotiation = catalog.negotiate("muninn:nightwing:move-evidence", &consumer)?; + + assert_eq!( + negotiation.transport, + CultMeshStreamBodyTransport::SharedMemory + ); + assert_eq!( + negotiation.copy_budget, + CultMeshStreamCopyBudget::ZeroCopyTarget + ); + + let handle = { + let ring = + catalog.create_shared_memory_ring("muninn:nightwing:move-evidence", 4, 128)?; + ring.try_publish_copy(&[1, 2, 3], 42, 0)? + .expect("frame should publish") + }; + catalog.publish_frame(handle.clone())?; + + assert_eq!( + catalog.latest_frame("muninn:nightwing:move-evidence"), + Some(&handle) + ); + let lease = catalog + .ring("muninn:nightwing:move-evidence") + .and_then(CultMeshSharedMemoryFrameRing::try_acquire_latest_read) + .expect("latest frame should be readable"); + assert_eq!(lease.handle.sequence, 0); + assert_eq!(lease.bytes(), &[1, 2, 3]); + Ok(()) + } +} diff --git a/packages/cultmesh-rs/src/rudp_document_server.rs b/packages/cultmesh-rs/src/rudp_document_server.rs new file mode 100644 index 00000000..bebeb6c3 --- /dev/null +++ b/packages/cultmesh-rs/src/rudp_document_server.rs @@ -0,0 +1,647 @@ +use anyhow::{Result, anyhow}; +use cultnet_rs::{ + CultNetMessage, CultNetRawDocumentRecord, CultNetRudpPacket, CultNetRudpPacketType, + CultNetRudpSendOptions, CultNetRudpSession, CultNetRudpSessionOptions, CultNetWireContract, + decode_cultnet_message_from_slice, decode_rudp_packet, encode_cultnet_message_to_vec, + encode_rudp_packet, +}; +use std::collections::BTreeMap; +use std::io::ErrorKind; +use std::net::{SocketAddr, UdpSocket}; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; + +const MAX_UDP_DATAGRAM_BYTES: usize = 65_535; + +/// The transport identity of one remote CultNet RUDP session. +/// +/// Connection identifiers are scoped to a remote socket address. Two clients +/// may therefore use the same connection identifier without sharing ordering, +/// acknowledgement, resend, or fragment state. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub struct CultMeshRudpSessionKey { + pub remote_addr: SocketAddr, + pub connection_id: u32, +} + +/// A raw document plus the receipt fact owned by the receiving process. +/// +/// `document.payload` is the exact byte vector decoded from the CultNet raw +/// record. The trusted receipt time lives beside the sender-owned record so the +/// transport never rewrites its metadata or payload. +#[derive(Clone, Debug, PartialEq)] +pub struct CultMeshRudpRawDocumentReceipt { + pub session: CultMeshRudpSessionKey, + pub message_id: String, + pub transport_sequence: u32, + pub received_at_unix_millis: u64, + pub document: CultNetRawDocumentRecord, +} + +/// The caller-visible query for a read-only raw snapshot. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CultMeshRudpSnapshotQuery { + pub session: CultMeshRudpSessionKey, + pub message_id: String, + pub requested_at_unix_millis: u64, + pub schema_ids: Option>, + pub record_keys: Option>, +} + +/// Caller-owned admission/persistence port for received raw documents. +pub trait CultMeshRudpRawDocumentSink { + fn accept_raw_document(&mut self, receipt: CultMeshRudpRawDocumentReceipt) -> Result<()>; +} + +impl CultMeshRudpRawDocumentSink for F +where + F: FnMut(CultMeshRudpRawDocumentReceipt) -> Result<()>, +{ + fn accept_raw_document(&mut self, receipt: CultMeshRudpRawDocumentReceipt) -> Result<()> { + self(receipt) + } +} + +/// Caller-owned catalog port for serving raw snapshot requests. +/// +/// The caller decides which records the requester may see. CultMesh preserves +/// those records as raw CultNet documents and does not interpret their schemas. +pub trait CultMeshRudpSnapshotSource { + fn raw_snapshot( + &mut self, + query: &CultMeshRudpSnapshotQuery, + ) -> Result>; +} + +impl CultMeshRudpSnapshotSource for F +where + F: FnMut(&CultMeshRudpSnapshotQuery) -> Result>, +{ + fn raw_snapshot( + &mut self, + query: &CultMeshRudpSnapshotQuery, + ) -> Result> { + self(query) + } +} + +/// Trusted local clocks for receipt facts and transport maintenance. +/// +/// Receipt time is wall time. Expiry and resend decisions use the monotonic +/// value, so a wall-clock correction cannot resurrect or pin a session. +pub trait CultMeshRudpServerClock { + fn now_unix_millis(&self) -> u64; + fn now_monotonic_millis(&self) -> u64; +} + +#[derive(Clone, Debug)] +pub struct CultMeshSystemClock { + monotonic_origin: Instant, +} + +impl Default for CultMeshSystemClock { + fn default() -> Self { + Self { + monotonic_origin: Instant::now(), + } + } +} + +impl CultMeshRudpServerClock for CultMeshSystemClock { + fn now_unix_millis(&self) -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_millis() + .try_into() + .unwrap_or(u64::MAX) + } + + fn now_monotonic_millis(&self) -> u64 { + self.monotonic_origin + .elapsed() + .as_millis() + .try_into() + .unwrap_or(u64::MAX) + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CultMeshRudpDocumentServerOptions { + pub max_sessions: usize, + pub session_idle_timeout: Duration, + pub session_max_lifetime: Duration, + /// Conservative payload admission budget across all live sessions. + /// + /// Admitted ingress and queued snapshot-response bytes are charged until + /// the session ends even when CultNet releases them sooner. Together with + /// CultNet's per-session receive caps, this makes retained payload memory + /// finite without duplicating its fragment/ordering machinery here. + pub max_admitted_payload_bytes: usize, + pub max_admitted_payload_bytes_per_session: usize, + pub max_snapshot_response_bytes: usize, + pub max_snapshot_documents: usize, + pub resend_delay: Duration, + pub max_pending_reliable_packets_per_session: usize, + pub max_fragment_bytes: usize, +} + +impl Default for CultMeshRudpDocumentServerOptions { + fn default() -> Self { + Self { + max_sessions: 64, + session_idle_timeout: Duration::from_secs(30), + session_max_lifetime: Duration::from_secs(15 * 60), + max_admitted_payload_bytes: 32 * 1024 * 1024, + max_admitted_payload_bytes_per_session: 4 * 1024 * 1024, + max_snapshot_response_bytes: 1024 * 1024, + max_snapshot_documents: 4096, + resend_delay: Duration::from_millis(50), + max_pending_reliable_packets_per_session: 1024, + max_fragment_bytes: 1200, + } + } +} + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct CultMeshRudpMaintenance { + pub sessions_expired: usize, + pub packets_resent: usize, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum CultMeshRudpApplicationOperation { + DocumentPutRaw, + SnapshotRequest, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CultMeshRudpApplicationRejection { + pub session: CultMeshRudpSessionKey, + pub operation: CultMeshRudpApplicationOperation, + pub message_id: String, + pub reason: String, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum CultMeshRudpPollOutcome { + Idle, + Handled, + ApplicationRejected(CultMeshRudpApplicationRejection), +} + +struct SessionEntry { + session: CultNetRudpSession, + created_at_monotonic_millis: u64, + last_activity_monotonic_millis: u64, + admitted_payload_bytes: usize, +} + +/// A synchronous, multi-session CultNet RUDP raw-document server. +/// +/// The server owns exactly one UDP socket. Call `poll_once` from an existing +/// daemon loop; no executor or background thread is created here. +pub struct CultMeshRudpDocumentServer { + socket: UdpSocket, + sessions: BTreeMap, + sink: S, + snapshot_source: Q, + clock: C, + options: CultMeshRudpDocumentServerOptions, +} + +impl CultMeshRudpDocumentServer +where + S: CultMeshRudpRawDocumentSink, + Q: CultMeshRudpSnapshotSource, + C: CultMeshRudpServerClock, +{ + pub fn new( + socket: UdpSocket, + sink: S, + snapshot_source: Q, + clock: C, + options: CultMeshRudpDocumentServerOptions, + ) -> Result { + validate_options(&options)?; + socket.local_addr().map_err(|error| { + anyhow!("CultMesh RUDP server requires a bound UDP socket: {error}") + })?; + socket.set_nonblocking(true)?; + Ok(Self { + socket, + sessions: BTreeMap::new(), + sink, + snapshot_source, + clock, + options, + }) + } + + pub fn local_addr(&self) -> Result { + Ok(self.socket.local_addr()?) + } + + pub fn session_count(&self) -> usize { + self.sessions.len() + } + + /// Poll transport maintenance and receive at most one UDP datagram. + /// + /// Application rejection invalidates only the responsible peer session and + /// is returned as data so a daemon can log it and keep serving. Local + /// socket and server-state failures remain errors. + pub fn poll_once(&mut self) -> Result { + self.maintain()?; + let mut wire = vec![0_u8; MAX_UDP_DATAGRAM_BYTES]; + let (received, remote_addr) = match self.socket.recv_from(&mut wire) { + Ok(value) => value, + Err(error) if error.kind() == ErrorKind::WouldBlock => { + return Ok(CultMeshRudpPollOutcome::Idle); + } + Err(error) => return Err(error.into()), + }; + wire.truncate(received); + + let packet = match decode_rudp_packet(&wire) { + Ok(packet) => packet, + Err(_) => return Ok(CultMeshRudpPollOutcome::Handled), + }; + let now_unix = self.clock.now_unix_millis(); + let now_monotonic = self.clock.now_monotonic_millis(); + let key = CultMeshRudpSessionKey { + remote_addr, + connection_id: packet.connection_id, + }; + + if packet.packet_type == CultNetRudpPacketType::Connect { + self.accept_connection(key, &packet, now_monotonic)?; + return Ok(CultMeshRudpPollOutcome::Handled); + } + + if !self.sessions.contains_key(&key) { + return Ok(CultMeshRudpPollOutcome::Handled); + } + if packet.packet_type == CultNetRudpPacketType::Data + && packet.channel_id == "schema" + && !packet.reliable + { + return Ok(CultMeshRudpPollOutcome::Handled); + } + let payload_charge = if packet.packet_type == CultNetRudpPacketType::Data { + packet.payload.len() + } else { + 0 + }; + if !self.payload_budget_allows(key, payload_charge) { + return Ok(CultMeshRudpPollOutcome::Handled); + } + + let result = { + let entry = self + .sessions + .get_mut(&key) + .expect("checked session must remain present"); + entry.last_activity_monotonic_millis = now_monotonic; + entry.admitted_payload_bytes = + entry.admitted_payload_bytes.saturating_add(payload_charge); + entry.session.receive(&packet, now_monotonic) + }; + let result = match result { + Ok(result) => result, + Err(_) => { + self.sessions.remove(&key); + return Ok(CultMeshRudpPollOutcome::Handled); + } + }; + + if let Some(reply) = result.reply { + self.send_packet(key.remote_addr, &reply)?; + } + + if result.disconnected { + self.sessions.remove(&key); + return Ok(CultMeshRudpPollOutcome::Handled); + } + + for frame in result.delivered { + if frame.channel_id != "schema" { + continue; + } + let message = match decode_cultnet_message_from_slice( + &frame.payload, + CultNetWireContract::CultNetSchemaV0, + ) { + Ok(message) => message, + Err(_) => continue, + }; + if let Some(rejection) = self.deliver_application_message( + key, + frame.sequence, + message, + now_unix, + now_monotonic, + )? { + self.sessions.remove(&key); + return Ok(CultMeshRudpPollOutcome::ApplicationRejected(rejection)); + } + } + + if packet.reliable { + let ack = self + .sessions + .get_mut(&key) + .ok_or_else(|| anyhow!("CultMesh RUDP session disappeared before ACK"))? + .session + .create_ack(); + self.send_packet(key.remote_addr, &ack)?; + } + Ok(CultMeshRudpPollOutcome::Handled) + } + + /// Expire idle sessions and resend reliable packets whose deadline passed. + pub fn maintain(&mut self) -> Result { + let now = self.clock.now_monotonic_millis(); + let idle_timeout_ms = duration_millis(self.options.session_idle_timeout); + let lifetime_ms = duration_millis(self.options.session_max_lifetime); + let before = self.sessions.len(); + self.sessions.retain(|_, entry| { + now.saturating_sub(entry.last_activity_monotonic_millis) <= idle_timeout_ms + && now.saturating_sub(entry.created_at_monotonic_millis) <= lifetime_ms + }); + let expired = before - self.sessions.len(); + + let mut resends = Vec::new(); + for (key, entry) in &mut self.sessions { + for packet in entry.session.due_resends(now) { + resends.push((key.remote_addr, packet)); + } + } + for (remote_addr, packet) in &resends { + self.send_packet(*remote_addr, packet)?; + } + Ok(CultMeshRudpMaintenance { + sessions_expired: expired, + packets_resent: resends.len(), + }) + } + + fn accept_connection( + &mut self, + key: CultMeshRudpSessionKey, + packet: &CultNetRudpPacket, + now: u64, + ) -> Result<()> { + if !self.sessions.contains_key(&key) { + if self.sessions.len() >= self.options.max_sessions { + return Ok(()); + } + self.sessions.insert( + key, + SessionEntry { + session: CultNetRudpSession::new(CultNetRudpSessionOptions { + connection_id: key.connection_id, + initial_sequence: 1, + resend_delay_ms: duration_millis(self.options.resend_delay), + max_pending_reliable_packets: Some( + self.options.max_pending_reliable_packets_per_session, + ), + }), + created_at_monotonic_millis: now, + last_activity_monotonic_millis: now, + admitted_payload_bytes: 0, + }, + ); + } + + let accept = { + let entry = self + .sessions + .get_mut(&key) + .expect("accepted session must be present"); + // The same key is a retransmitted Connect for the existing epoch. + // A genuinely fresh client incarnation must choose a fresh id. + entry.last_activity_monotonic_millis = now; + entry.session.accept_connect(packet, now, Vec::new())? + }; + self.send_packet(key.remote_addr, &accept) + } + + fn deliver_application_message( + &mut self, + key: CultMeshRudpSessionKey, + transport_sequence: u32, + message: CultNetMessage, + now_unix: u64, + now: u64, + ) -> Result> { + match message { + CultNetMessage::DocumentPutRaw { + message_id, + document, + } => { + let receipt = CultMeshRudpRawDocumentReceipt { + session: key, + message_id: message_id.clone(), + transport_sequence, + received_at_unix_millis: now_unix, + document, + }; + if let Err(error) = self.sink.accept_raw_document(receipt) { + return Ok(Some(CultMeshRudpApplicationRejection { + session: key, + operation: CultMeshRudpApplicationOperation::DocumentPutRaw, + message_id, + reason: format!("{error:#}"), + })); + } + } + CultNetMessage::SnapshotRequest { + message_id, + schema_ids, + record_keys, + } => { + let query = CultMeshRudpSnapshotQuery { + session: key, + message_id: message_id.clone(), + requested_at_unix_millis: now_unix, + schema_ids, + record_keys, + }; + let documents = match self.snapshot_source.raw_snapshot(&query) { + Ok(documents) => documents, + Err(error) => { + return Ok(Some(CultMeshRudpApplicationRejection { + session: key, + operation: CultMeshRudpApplicationOperation::SnapshotRequest, + message_id, + reason: format!("{error:#}"), + })); + } + }; + if documents.len() > self.options.max_snapshot_documents { + return Ok(Some(CultMeshRudpApplicationRejection { + session: key, + operation: CultMeshRudpApplicationOperation::SnapshotRequest, + message_id, + reason: format!( + "snapshot returned {} documents; limit is {}", + documents.len(), + self.options.max_snapshot_documents + ), + })); + } + let response = CultNetMessage::SnapshotResponseRaw { + message_id: message_id.clone(), + documents, + }; + let payload = match encode_cultnet_message_to_vec( + &response, + CultNetWireContract::CultNetSchemaV0, + ) { + Ok(payload) => payload, + Err(error) => { + return Ok(Some(CultMeshRudpApplicationRejection { + session: key, + operation: CultMeshRudpApplicationOperation::SnapshotRequest, + message_id, + reason: format!("snapshot response could not be encoded: {error:#}"), + })); + } + }; + if payload.len() > self.options.max_snapshot_response_bytes { + return Ok(Some(CultMeshRudpApplicationRejection { + session: key, + operation: CultMeshRudpApplicationOperation::SnapshotRequest, + message_id, + reason: format!( + "snapshot response is {} bytes; limit is {}", + payload.len(), + self.options.max_snapshot_response_bytes + ), + })); + } + if !self.payload_budget_allows(key, payload.len()) { + return Ok(Some(CultMeshRudpApplicationRejection { + session: key, + operation: CultMeshRudpApplicationOperation::SnapshotRequest, + message_id, + reason: "retained payload budget is full".into(), + })); + } + let payload_bytes = payload.len(); + let entry = self + .sessions + .get_mut(&key) + .ok_or_else(|| anyhow!("CultMesh RUDP session disappeared before response"))?; + let packets = match entry.session.send_many( + "schema", + payload, + CultNetRudpSendOptions { + reliable: true, + ordered: true, + sequenced: false, + now_ms: now, + reliable_expire_after_ms: None, + }, + Some(self.options.max_fragment_bytes), + ) { + Ok(packets) => packets, + Err(error) => { + return Ok(Some(CultMeshRudpApplicationRejection { + session: key, + operation: CultMeshRudpApplicationOperation::SnapshotRequest, + message_id, + reason: format!("snapshot response could not be queued: {error:#}"), + })); + } + }; + entry.admitted_payload_bytes = + entry.admitted_payload_bytes.saturating_add(payload_bytes); + for packet in &packets { + self.send_packet(key.remote_addr, packet)?; + } + } + _ => {} + } + Ok(None) + } + + fn payload_budget_allows(&self, key: CultMeshRudpSessionKey, bytes: usize) -> bool { + let Some(session_bytes) = self + .sessions + .get(&key) + .map(|entry| entry.admitted_payload_bytes) + else { + return false; + }; + if session_bytes + .checked_add(bytes) + .is_none_or(|total| total > self.options.max_admitted_payload_bytes_per_session) + { + return false; + } + self.sessions + .values() + .try_fold(0_usize, |total, entry| { + total.checked_add(entry.admitted_payload_bytes) + }) + .and_then(|total| total.checked_add(bytes)) + .is_some_and(|total| total <= self.options.max_admitted_payload_bytes) + } + + fn send_packet(&mut self, remote_addr: SocketAddr, packet: &CultNetRudpPacket) -> Result<()> { + let wire = encode_rudp_packet(packet)?; + self.socket.send_to(&wire, remote_addr)?; + Ok(()) + } +} + +fn validate_options(options: &CultMeshRudpDocumentServerOptions) -> Result<()> { + if options.max_sessions == 0 { + return Err(anyhow!("max_sessions must be greater than zero")); + } + if options.session_idle_timeout.is_zero() { + return Err(anyhow!("session_idle_timeout must be greater than zero")); + } + if options.session_max_lifetime.is_zero() { + return Err(anyhow!("session_max_lifetime must be greater than zero")); + } + if options.max_admitted_payload_bytes == 0 { + return Err(anyhow!( + "max_admitted_payload_bytes must be greater than zero" + )); + } + if options.max_admitted_payload_bytes_per_session == 0 + || options.max_admitted_payload_bytes_per_session > options.max_admitted_payload_bytes + { + return Err(anyhow!( + "per-session admitted payload limit must be non-zero and no larger than the server limit" + )); + } + if options.max_snapshot_response_bytes == 0 + || options.max_snapshot_response_bytes > options.max_admitted_payload_bytes_per_session + { + return Err(anyhow!( + "snapshot response limit must be non-zero and fit in one session's payload budget" + )); + } + if options.max_snapshot_documents == 0 { + return Err(anyhow!("max_snapshot_documents must be greater than zero")); + } + if options.resend_delay.is_zero() { + return Err(anyhow!("resend_delay must be greater than zero")); + } + if options.max_pending_reliable_packets_per_session == 0 { + return Err(anyhow!( + "max_pending_reliable_packets_per_session must be greater than zero" + )); + } + if options.max_fragment_bytes == 0 { + return Err(anyhow!("max_fragment_bytes must be greater than zero")); + } + Ok(()) +} + +fn duration_millis(duration: Duration) -> u64 { + duration.as_millis().try_into().unwrap_or(u64::MAX) +} diff --git a/packages/cultmesh-rs/tests/rudp_document_server.rs b/packages/cultmesh-rs/tests/rudp_document_server.rs new file mode 100644 index 00000000..df14b4ea --- /dev/null +++ b/packages/cultmesh-rs/tests/rudp_document_server.rs @@ -0,0 +1,555 @@ +use anyhow::Result; +use cultmesh_rs::{ + CultMeshRudpApplicationOperation, CultMeshRudpDocumentServer, + CultMeshRudpDocumentServerOptions, CultMeshRudpPollOutcome, CultMeshRudpRawDocumentReceipt, + CultMeshRudpRawDocumentSink, CultMeshRudpServerClock, CultMeshRudpSnapshotQuery, + CultMeshRudpSnapshotSource, +}; +use cultnet_rs::{ + CultNetMessage, CultNetRawDocumentRecord, CultNetRawPayloadEncoding, + CultNetRudpReliableSendReceipt, CultNetRudpReliableSendStatus, CultNetRudpSocketMode, + CultNetRudpSocketTransportConnection, CultNetRudpSocketTransportOptions, CultNetWireContract, + decode_cultnet_message_from_slice, encode_cultnet_message_to_vec, +}; +use pretty_assertions::assert_eq; +use std::net::{SocketAddr, UdpSocket}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Mutex}; +use std::thread; +use std::time::Duration; + +#[derive(Default)] +struct SinkState { + receipts: Vec, + failures_remaining: usize, +} + +#[derive(Clone, Default)] +struct Sink(Arc>); + +impl Sink { + fn fail_once() -> Self { + Self(Arc::new(Mutex::new(SinkState { + failures_remaining: 1, + ..Default::default() + }))) + } +} + +impl CultMeshRudpRawDocumentSink for Sink { + fn accept_raw_document(&mut self, receipt: CultMeshRudpRawDocumentReceipt) -> Result<()> { + let mut state = self.0.lock().unwrap(); + if state.failures_remaining > 0 { + state.failures_remaining -= 1; + anyhow::bail!("injected sink failure"); + } + state.receipts.push(receipt); + Ok(()) + } +} + +#[derive(Default)] +struct SourceState { + documents: Vec, + failures_remaining: usize, + calls: usize, +} + +#[derive(Clone, Default)] +struct Source(Arc>); + +impl Source { + fn documents(documents: Vec) -> Self { + Self(Arc::new(Mutex::new(SourceState { + documents, + ..Default::default() + }))) + } + + fn fail_once(documents: Vec) -> Self { + Self(Arc::new(Mutex::new(SourceState { + documents, + failures_remaining: 1, + calls: 0, + }))) + } +} + +impl CultMeshRudpSnapshotSource for Source { + fn raw_snapshot( + &mut self, + _: &CultMeshRudpSnapshotQuery, + ) -> Result> { + let mut state = self.0.lock().unwrap(); + state.calls += 1; + if state.failures_remaining > 0 { + state.failures_remaining -= 1; + anyhow::bail!("injected source failure"); + } + Ok(state.documents.clone()) + } +} + +#[derive(Clone)] +struct Clock { + unix: Arc, + monotonic: Arc, +} + +impl Clock { + fn new(now: u64) -> Self { + Self { + unix: Arc::new(AtomicU64::new(now)), + monotonic: Arc::new(AtomicU64::new(now)), + } + } + + fn set(&self, now: u64) { + self.unix.store(now, Ordering::SeqCst); + self.monotonic.store(now, Ordering::SeqCst); + } + + fn set_unix(&self, now: u64) { + self.unix.store(now, Ordering::SeqCst); + } + + fn set_monotonic(&self, now: u64) { + self.monotonic.store(now, Ordering::SeqCst); + } +} + +impl CultMeshRudpServerClock for Clock { + fn now_unix_millis(&self) -> u64 { + self.unix.load(Ordering::SeqCst) + } + + fn now_monotonic_millis(&self) -> u64 { + self.monotonic.load(Ordering::SeqCst) + } +} + +type Server = CultMeshRudpDocumentServer; + +fn document(key: &str, payload: Vec) -> CultNetRawDocumentRecord { + CultNetRawDocumentRecord { + schema_id: "test.raw.v1".into(), + record_key: key.into(), + stored_at: "2026-09-03T00:00:00Z".into(), + payload_encoding: CultNetRawPayloadEncoding::Messagepack, + payload, + source_runtime_id: Some(format!("runtime-{key}")), + source_agent_id: None, + source_role: None, + tags: None, + } +} + +fn server( + options: CultMeshRudpDocumentServerOptions, + clock: Clock, + sink: Sink, + source: Source, +) -> Result { + CultMeshRudpDocumentServer::new( + UdpSocket::bind("127.0.0.1:0")?, + sink, + source, + clock, + options, + ) +} + +fn client(target: SocketAddr, id: u32) -> Result { + let socket = UdpSocket::bind("127.0.0.1:0")?; + socket.set_nonblocking(true)?; + CultNetRudpSocketTransportConnection::new(CultNetRudpSocketTransportOptions { + media_delivery: None, + runtime_id: "test-client".into(), + socket, + mode: CultNetRudpSocketMode::Client, + remote_addr: Some(target), + connection_id: id, + initial_sequence: 1, + resend_delay_ms: 10, + transport_id: None, + max_payload_bytes: None, + max_fragment_bytes: Some(1200), + max_pending_reliable_packets: Some(64), + media_reliable_expire_after_ms: None, + reconnect_policy: None, + }) +} + +fn connect( + server: &mut Server, + clients: &mut [&mut CultNetRudpSocketTransportConnection], +) -> Result<()> { + for client in &mut *clients { + client.connect(Vec::new())?; + } + for _ in 0..500 { + server.poll_once()?; + for client in &mut *clients { + client.receive_once()?; + } + if clients.iter().all(|client| client.connected()) { + for _ in 0..clients.len() { + server.poll_once()?; + } + return Ok(()); + } + thread::sleep(Duration::from_millis(1)); + } + anyhow::bail!("client connection timed out") +} + +fn send(client: &mut CultNetRudpSocketTransportConnection, message: &CultNetMessage) -> Result<()> { + client.send( + "schema", + encode_cultnet_message_to_vec(message, CultNetWireContract::CultNetSchemaV0)?, + ) +} + +fn send_reliable( + client: &mut CultNetRudpSocketTransportConnection, + message: &CultNetMessage, +) -> Result { + client.send_reliable( + "schema", + encode_cultnet_message_to_vec(message, CultNetWireContract::CultNetSchemaV0)?, + ) +} + +#[test] +fn same_connection_id_is_peer_scoped_and_raw_bytes_are_untouched() -> Result<()> { + let clock = Clock::new(42_000); + let sink = Sink::default(); + let mut server = server(Default::default(), clock, sink.clone(), Source::default())?; + let mut a = client(server.local_addr()?, 77)?; + let mut b = client(server.local_addr()?, 77)?; + connect(&mut server, &mut [&mut a, &mut b])?; + + let a_bytes = vec![0xc4, 0x04, 0x00, 0xff, 0x80, 0x01]; + let b_bytes = vec![0x92, 0x01, 0xc4, 0x02, 0x00, 0xff]; + send( + &mut a, + &CultNetMessage::DocumentPutRaw { + message_id: "a".into(), + document: document("a", a_bytes.clone()), + }, + )?; + send( + &mut b, + &CultNetMessage::DocumentPutRaw { + message_id: "b".into(), + document: document("b", b_bytes.clone()), + }, + )?; + for _ in 0..500 { + server.poll_once()?; + a.receive_once()?; + b.receive_once()?; + if sink.0.lock().unwrap().receipts.len() == 2 { + break; + } + } + + let receipts = sink.0.lock().unwrap(); + assert_eq!(server.session_count(), 2); + assert_eq!(receipts.receipts.len(), 2); + assert_ne!( + receipts.receipts[0].session.remote_addr, + receipts.receipts[1].session.remote_addr + ); + assert_eq!( + receipts + .receipts + .iter() + .find(|r| r.message_id == "a") + .unwrap() + .document + .payload, + a_bytes + ); + assert_eq!( + receipts + .receipts + .iter() + .find(|r| r.message_id == "b") + .unwrap() + .document + .payload, + b_bytes + ); + assert!( + receipts + .receipts + .iter() + .all(|r| r.received_at_unix_millis == 42_000) + ); + Ok(()) +} + +#[test] +fn duplicate_connect_preserves_epoch_and_fresh_epoch_is_separate() -> Result<()> { + let clock = Clock::new(45_000); + let sink = Sink::default(); + let mut server = server(Default::default(), clock, sink.clone(), Source::default())?; + let target = server.local_addr()?; + let mut incumbent = client(target, 91)?; + connect(&mut server, &mut [&mut incumbent])?; + + incumbent.connect(Vec::new())?; + server.poll_once()?; + incumbent.receive_once()?; + assert_eq!(server.session_count(), 1); + + send( + &mut incumbent, + &CultNetMessage::DocumentPutRaw { + message_id: "after-duplicate".into(), + document: document("incumbent", vec![1, 2, 3]), + }, + )?; + for _ in 0..20 { + server.poll_once()?; + if sink.0.lock().unwrap().receipts.len() == 1 { + break; + } + } + assert_eq!(sink.0.lock().unwrap().receipts.len(), 1); + + let mut fresh_epoch = client(target, 92)?; + connect(&mut server, &mut [&mut fresh_epoch])?; + assert_eq!(server.session_count(), 2); + Ok(()) +} + +#[test] +fn application_rejection_is_nonfatal_peer_scoped_and_unacknowledged() -> Result<()> { + let clock = Clock::new(48_000); + let sink = Sink::fail_once(); + let source = Source::fail_once(vec![document("snapshot", vec![4, 5, 6])]); + let mut server = server( + Default::default(), + clock.clone(), + sink.clone(), + source.clone(), + )?; + let target = server.local_addr()?; + + let mut publisher = client(target, 101)?; + let mut snapshot_client = client(target, 102)?; + let mut survivor = client(target, 103)?; + connect( + &mut server, + &mut [&mut publisher, &mut snapshot_client, &mut survivor], + )?; + let publish_receipt = send_reliable( + &mut publisher, + &CultNetMessage::DocumentPutRaw { + message_id: "rejected-put".into(), + document: document("rejected", vec![0, 255, 1]), + }, + )?; + let CultMeshRudpPollOutcome::ApplicationRejected(rejection) = server.poll_once()? else { + panic!("sink rejection must be returned as a nonfatal poll outcome"); + }; + assert_eq!(rejection.session.connection_id, 101); + assert_eq!( + rejection.operation, + CultMeshRudpApplicationOperation::DocumentPutRaw + ); + assert_eq!(rejection.message_id, "rejected-put"); + assert!(rejection.reason.contains("injected sink failure")); + assert_eq!(server.session_count(), 2); + publisher.receive_once()?; + assert_eq!( + publisher.reliable_send_status(&publish_receipt), + CultNetRudpReliableSendStatus::Pending + ); + + let snapshot_receipt = send_reliable( + &mut snapshot_client, + &CultNetMessage::SnapshotRequest { + message_id: "rejected-snapshot".into(), + schema_ids: None, + record_keys: None, + }, + )?; + let CultMeshRudpPollOutcome::ApplicationRejected(rejection) = server.poll_once()? else { + panic!("source rejection must be returned as a nonfatal poll outcome"); + }; + assert_eq!(rejection.session.connection_id, 102); + assert_eq!( + rejection.operation, + CultMeshRudpApplicationOperation::SnapshotRequest + ); + assert_eq!(rejection.message_id, "rejected-snapshot"); + assert!(rejection.reason.contains("injected source failure")); + assert_eq!(server.session_count(), 1); + snapshot_client.receive_once()?; + assert_eq!( + snapshot_client.reliable_send_status(&snapshot_receipt), + CultNetRudpReliableSendStatus::Pending + ); + + let survivor_receipt = send_reliable( + &mut survivor, + &CultNetMessage::DocumentPutRaw { + message_id: "accepted-after-rejections".into(), + document: document("survivor", vec![9, 8, 7]), + }, + )?; + assert_eq!(server.poll_once()?, CultMeshRudpPollOutcome::Handled); + for _ in 0..20 { + survivor.receive_once()?; + if survivor.reliable_send_status(&survivor_receipt) + == CultNetRudpReliableSendStatus::Acknowledged + { + break; + } + } + assert_eq!( + survivor.reliable_send_status(&survivor_receipt), + CultNetRudpReliableSendStatus::Acknowledged + ); + assert_eq!(sink.0.lock().unwrap().receipts.len(), 1); + assert_eq!(source.0.lock().unwrap().calls, 1); + Ok(()) +} + +#[test] +fn snapshot_response_resends_until_acknowledged() -> Result<()> { + let clock = Clock::new(51_000); + let expected = document("catalog", vec![0x81, 0xa1, 0x78, 0x2a]); + let source = Source::documents(vec![expected.clone()]); + let options = CultMeshRudpDocumentServerOptions { + resend_delay: Duration::from_millis(10), + ..Default::default() + }; + let mut server = server(options, clock.clone(), Sink::default(), source)?; + let mut client = client(server.local_addr()?, 88)?; + connect(&mut server, &mut [&mut client])?; + send( + &mut client, + &CultNetMessage::SnapshotRequest { + message_id: "snapshot".into(), + schema_ids: Some(vec!["test.raw.v1".into()]), + record_keys: Some(vec!["catalog".into()]), + }, + )?; + server.poll_once()?; + + clock.set(51_011); + assert_eq!(server.maintain()?.packets_resent, 1); + let response = loop { + if let Some(frame) = client.receive_once()? { + break decode_cultnet_message_from_slice( + &frame.payload, + CultNetWireContract::CultNetSchemaV0, + )?; + } + }; + assert_eq!( + response, + CultNetMessage::SnapshotResponseRaw { + message_id: "snapshot".into(), + documents: vec![expected], + } + ); + + for _ in 0..20 { + server.poll_once()?; + } + clock.set(51_022); + assert_eq!(server.maintain()?.packets_resent, 0); + Ok(()) +} + +#[test] +fn payload_and_snapshot_output_budgets_fail_closed() -> Result<()> { + let clock = Clock::new(55_000); + let sink = Sink::default(); + let message = CultNetMessage::DocumentPutRaw { + message_id: "budget".into(), + document: document("budget", vec![7; 64]), + }; + let encoded = encode_cultnet_message_to_vec(&message, CultNetWireContract::CultNetSchemaV0)?; + let options = CultMeshRudpDocumentServerOptions { + max_admitted_payload_bytes: encoded.len(), + max_admitted_payload_bytes_per_session: encoded.len(), + max_snapshot_response_bytes: encoded.len(), + ..Default::default() + }; + let mut budget_server = server(options, clock, sink.clone(), Source::default())?; + let target = budget_server.local_addr()?; + let mut first = client(target, 111)?; + let mut second = client(target, 112)?; + connect(&mut budget_server, &mut [&mut first, &mut second])?; + send(&mut first, &message)?; + budget_server.poll_once()?; + send(&mut second, &message)?; + budget_server.poll_once()?; + assert_eq!(sink.0.lock().unwrap().receipts.len(), 1); + + let oversized = Source::documents(vec![document("oversized", vec![8; 512])]); + let options = CultMeshRudpDocumentServerOptions { + max_snapshot_response_bytes: 128, + ..Default::default() + }; + let mut snapshot_server = server(options, Clock::new(56_000), Sink::default(), oversized)?; + let mut snapshot_client = client(snapshot_server.local_addr()?, 113)?; + connect(&mut snapshot_server, &mut [&mut snapshot_client])?; + let receipt = send_reliable( + &mut snapshot_client, + &CultNetMessage::SnapshotRequest { + message_id: "too-large".into(), + schema_ids: None, + record_keys: None, + }, + )?; + let CultMeshRudpPollOutcome::ApplicationRejected(rejection) = snapshot_server.poll_once()? + else { + panic!("oversized snapshot must reject only its session"); + }; + assert_eq!( + rejection.operation, + CultMeshRudpApplicationOperation::SnapshotRequest + ); + assert_eq!(rejection.message_id, "too-large"); + assert!(rejection.reason.contains("snapshot response is")); + assert_eq!(snapshot_server.session_count(), 0); + snapshot_client.receive_once()?; + assert_eq!( + snapshot_client.reliable_send_status(&receipt), + CultNetRudpReliableSendStatus::Pending + ); + Ok(()) +} + +#[test] +fn session_cap_and_expiry_use_monotonic_time() -> Result<()> { + let clock = Clock::new(60_000); + let options = CultMeshRudpDocumentServerOptions { + max_sessions: 1, + session_idle_timeout: Duration::from_secs(1), + session_max_lifetime: Duration::from_millis(100), + ..Default::default() + }; + let mut server = server(options, clock.clone(), Sink::default(), Source::default())?; + let target = server.local_addr()?; + let mut admitted = client(target, 1)?; + connect(&mut server, &mut [&mut admitted])?; + let mut rejected = client(target, 1)?; + rejected.connect(Vec::new())?; + server.poll_once()?; + assert_eq!(server.session_count(), 1); + assert!(!rejected.connected()); + + clock.set_unix(1); + clock.set_monotonic(60_101); + assert_eq!(server.maintain()?.sessions_expired, 1); + assert_eq!(server.session_count(), 0); + Ok(()) +} diff --git a/packages/cultmesh-ts/README.md b/packages/cultmesh-ts/README.md index 9c4dfb01..d3fe565b 100644 --- a/packages/cultmesh-ts/README.md +++ b/packages/cultmesh-ts/README.md @@ -99,11 +99,13 @@ const station = await CultMesh.startNode(statePath, { const stock = station.document(stationStockUiDocument, "station:starbridge:stock"); const current = await stock.latest(); -await stock.set(updatedStockFromUi); +await stock.authoritativeWriter().write(updatedStockFromUi); -const reactive = station.reactiveDocument(stationStockUiDocument, "station:starbridge:stock"); +const reactive = stock.authoritativeWriter().reactive(); await reactive.ready; -reactive.current.availableMissiles -= 4; +reactive.update(draft => { + draft.availableMissiles -= 4; +}); ``` State pointers are the same kind of managed surface for UI and tools. They can @@ -418,7 +420,8 @@ Use `node.syncDocumentFromPeerSnapshot(...)` or local node from the remote snapshot. The helper requests the raw snapshot, applies it through the node's document registry, and returns the requested typed definition. Same-schema aliases stay local after the call: the caller can -keep using `node.document(aliasDefinition, key)` or `node.reactiveDocument(...)` +keep using `node.document(aliasDefinition, key)` and explicitly select +`authoritativeWriter()`, `predictionWriter()`, or `observe()` without re-threading the RUDP peer. The same typed-definition overload is available for @@ -432,7 +435,7 @@ sync came from local persistence or a remote CultNet snapshot. Use `node.syncDocumentFromPublication(...)` or `CultMesh.syncDocumentFromPublication(...)` for the same source-agnostic shape when the publication should hydrate a local node before the caller continues -through `node.document(...)` or `node.reactiveDocument(...)`. +through `node.document(...)` and its explicit writer or observer. ## Streaming Mode diff --git a/packages/cultmesh-ts/package.json b/packages/cultmesh-ts/package.json index a1cc77c4..0ca8afb3 100644 --- a/packages/cultmesh-ts/package.json +++ b/packages/cultmesh-ts/package.json @@ -43,8 +43,8 @@ }, "dependencies": { "@msgpack/msgpack": "^3.1.2", - "cultcache-ts": "^0.13.5", - "cultnet-ts": "^0.1.1" + "@gamecult/cultcache-ts": "^0.14.0", + "cultnet-ts": "^0.2.0" }, "devDependencies": { "@types/node": "^24.0.0", diff --git a/packages/cultmesh-ts/src/index.ts b/packages/cultmesh-ts/src/index.ts index aa84f5f9..e9fbb8be 100644 --- a/packages/cultmesh-ts/src/index.ts +++ b/packages/cultmesh-ts/src/index.ts @@ -8,7 +8,7 @@ import { type CacheBackingStore, type CultCacheEnvelope, type CultCacheDocumentValue, -} from "cultcache-ts"; +} from "@gamecult/cultcache-ts"; import { CultNetDocumentRegistry, CultNetRudpSession, @@ -232,6 +232,7 @@ export interface CultMeshReactiveDocumentOptions { readonly context?: CultMeshQueryContext | string; readonly watch?: boolean; readonly debounceMs?: number; + readonly replaceDirtyCurrentOnCanonicalSnapshot?: boolean; readonly onError?: (error: unknown) => void; } @@ -248,15 +249,30 @@ export interface CultMeshReactiveDocumentReconciliation { - readonly current: TDocument; + readonly writer: CultMeshDocumentWriter; + readonly snapshot: TDocument; + readonly dirty: boolean; readonly reconciliation: CultMeshReactiveDocumentReconciliation | undefined; readonly ready: Promise; readonly disposed: boolean; + update(update: (draft: TDocument) => void | TDocument): TDocument; + replaceLocal(value: TDocument): TDocument; + flush(): Promise; clearReconciliation(): void; refresh(): Promise; dispose(): void; } +export interface CultMeshObservedDocument { + readonly current: TDocument; + readonly ready: Promise; + readonly disposed: boolean; + refresh(): Promise; + dispose(): void; +} + +export type CultMeshDocumentWriteKind = "authoritative-replacement" | "prediction"; + export interface CultMeshNativeSliceColumn { readonly name: string; readonly valueType: string; @@ -284,6 +300,13 @@ export interface CultMeshStateBindingDescriptor { readonly sourceId?: string; readonly schemaId?: string; readonly routeHint: CultMeshRouteHint; + readonly bindingName?: string; + readonly documentId?: string; + readonly fieldPath?: string; + readonly valueKind?: "string" | "number" | "boolean" | "choice" | "string-list"; + readonly accessMode?: "read" | "write" | "read-write" | "local-draft"; + readonly authority?: string; + readonly writeCommand?: string; } export interface CultMeshStateBindingRecord { @@ -293,6 +316,13 @@ export interface CultMeshStateBindingRecord { readonly schemaId: string; readonly routeKind: string; readonly routeDescription: string; + readonly bindingName: string; + readonly documentId: string; + readonly fieldPath: string; + readonly valueKind: string; + readonly accessMode: string; + readonly authority: string; + readonly writeCommand: string; } export interface CultMeshOperationBindingDescriptor { @@ -842,10 +872,6 @@ export class CultMeshDocumentHandle { return this.submitPredictionDocument !== undefined; } - public get canSet(): boolean { - return this.canSubmitPrediction || this.canReplace; - } - public latest(context: CultMeshQueryContext | string = "local"): Promise { return this.snapshotDocument( this.resolveContext(typeof context === "string" ? cultMeshQueryContext(context) : context), @@ -932,63 +958,44 @@ export class CultMeshDocumentHandle { return this.submitPredictionDocument(this.resolveContext(context), value); } - public set(value: TDocument): Promise; - public set(context: CultMeshQueryContext | string, value: TDocument): Promise; - public set( - contextOrValue: CultMeshQueryContext | string | TDocument, - maybeValue?: TDocument, - ): Promise { - const hasContext = - typeof contextOrValue === "string" || isCultMeshQueryContext(contextOrValue); - const context = hasContext - ? typeof contextOrValue === "string" - ? cultMeshQueryContext(contextOrValue) - : contextOrValue as CultMeshQueryContext - : cultMeshQueryContext("local"); - const value = hasContext ? maybeValue : contextOrValue as TDocument; - if (value === undefined) { - throw new Error(`Document '${this.documentId}' requires a value.`); + public authoritativeWriter( + context: CultMeshQueryContext | string = "local", + ): CultMeshDocumentWriter { + if (!this.replaceDocument) { + throw new Error(`Document '${this.documentId}' does not support authoritative replacement.`); } - - const resolved = this.resolveContext(context); - if (this.submitPredictionDocument) { - return this.submitPredictionDocument(resolved, value); - } - if (this.replaceDocument) { - return this.replaceDocument(resolved, value); - } - - throw new Error(`Document '${this.documentId}' does not support mutation.`); + const resolved = this.resolveContext( + typeof context === "string" ? cultMeshQueryContext(context) : context, + ); + return new CultMeshDocumentWriter( + this as unknown as CultMeshDocumentHandle, + "authoritative-replacement", + resolved, + value => this.replaceDocument!(resolved, value as unknown as TDocument), + ); } - public update(update: CultMeshDocumentUpdater): Promise; - public update( - context: CultMeshQueryContext | string, - update: CultMeshDocumentUpdater, - ): Promise; - public async update( - contextOrUpdate: CultMeshQueryContext | string | CultMeshDocumentUpdater, - maybeUpdate?: CultMeshDocumentUpdater, - ): Promise { - const hasContext = - typeof contextOrUpdate === "string" || isCultMeshQueryContext(contextOrUpdate); - const context = hasContext - ? contextOrUpdate as CultMeshQueryContext | string - : "local"; - const update = hasContext ? maybeUpdate : contextOrUpdate as CultMeshDocumentUpdater; - if (!update) { - throw new Error(`Document '${this.documentId}' requires an update function.`); + public predictionWriter( + context: CultMeshQueryContext | string = "local", + ): CultMeshDocumentWriter { + if (!this.submitPredictionDocument) { + throw new Error(`Document '${this.documentId}' does not support prediction submission.`); } - - const next = await update(await this.latest(context)); - await this.set(context, next); - return next; + const resolved = this.resolveContext( + typeof context === "string" ? cultMeshQueryContext(context) : context, + ); + return new CultMeshDocumentWriter( + this as unknown as CultMeshDocumentHandle, + "prediction", + resolved, + value => this.submitPredictionDocument!(resolved, value as unknown as TDocument), + ); } - public reactive( + public observe( options: CultMeshReactiveDocumentOptions = {}, - ): CultMeshReactiveDocument { - return new CultMeshReactiveDocumentHandle( + ): CultMeshObservedDocument { + return new CultMeshObservedDocumentHandle( this as unknown as CultMeshDocumentHandle, options, ); @@ -1041,22 +1048,40 @@ export class CultMeshDocumentHandle { } } -class CultMeshReactiveDocumentHandle - implements CultMeshReactiveDocument { +export class CultMeshDocumentWriter { + public constructor( + public readonly document: CultMeshDocumentHandle, + public readonly kind: CultMeshDocumentWriteKind, + public readonly context: CultMeshQueryContext, + private readonly writeDocument: (value: TDocument) => Promise, + ) {} + + public write(value: TDocument): Promise { + return this.writeDocument(cloneCultMeshDocument(value)); + } + + public async update(update: CultMeshDocumentUpdater): Promise { + const next = await update(cloneCultMeshDocument(await this.document.latest(this.context))); + await this.write(next); + return next; + } + + public reactive( + options: Omit = {}, + ): CultMeshReactiveDocument { + return new CultMeshReactiveDocumentHandle(this, options); + } +} + +class CultMeshObservedDocumentHandle + implements CultMeshObservedDocument { readonly #document: CultMeshDocumentHandle; readonly #context: CultMeshQueryContext | string; readonly #onError: (error: unknown) => void; - readonly #debounceMs: number; - #current: TDocument; + #canonicalVersion = 0; + #current = {} as TDocument; #disposed = false; - #applyingRemote = false; - #pendingSet: Promise = Promise.resolve(); - #flushScheduled = false; - #flushTimer: ReturnType | undefined; #unsubscribe: CultMeshUnsubscribe | undefined; - #lastPredicted: TDocument | undefined; - #reconciliation: CultMeshReactiveDocumentReconciliation | undefined; - #reconciliationVersion = 0; public readonly ready: Promise; @@ -1066,19 +1091,17 @@ class CultMeshReactiveDocumentHandle ) { this.#document = document; this.#context = options.context ?? "local"; - this.#debounceMs = options.debounceMs ?? 0; this.#onError = options.onError ?? ((error) => { queueMicrotask(() => { throw error; }); }); - this.#current = this.createProxy({} as TDocument); this.ready = this.refresh(); - if (options.watch !== false) { try { this.#unsubscribe = document.watch(this.#context, value => { - this.applySnapshot(value); + this.#current = cloneCultMeshDocument(value); + this.#canonicalVersion++; }); } catch (error) { this.#onError(error); @@ -1087,9 +1110,100 @@ class CultMeshReactiveDocumentHandle } public get current(): TDocument { + this.throwIfDisposed(); return this.#current; } + public get disposed(): boolean { + return this.#disposed; + } + + public async refresh(): Promise { + this.throwIfDisposed(); + const versionBeforeRead = this.#canonicalVersion; + const latest = cloneCultMeshDocument(await this.#document.latest(this.#context)); + if (versionBeforeRead === this.#canonicalVersion) { + this.#current = latest; + this.#canonicalVersion++; + } + return this.#current; + } + + public dispose(): void { + if (this.#disposed) return; + this.#disposed = true; + this.#unsubscribe?.(); + this.#unsubscribe = undefined; + } + + private throwIfDisposed(): void { + if (this.#disposed) { + throw new Error(`Observed document '${this.#document.documentId}' is disposed.`); + } + } +} + +class CultMeshReactiveDocumentHandle + implements CultMeshReactiveDocument { + readonly #writer: CultMeshDocumentWriter; + readonly #document: CultMeshDocumentHandle; + readonly #onError: (error: unknown) => void; + readonly #debounceMs: number; + readonly #replaceDirtyCurrentOnCanonicalSnapshot: boolean; + #current = {} as TDocument; + #disposed = false; + #pendingSet: Promise = Promise.resolve(); + #flushScheduled = false; + #cancelScheduledFlush: (() => void) | undefined; + #unsubscribe: CultMeshUnsubscribe | undefined; + #dirty = false; + #lastPredicted: TDocument | undefined; + #reconciliation: CultMeshReactiveDocumentReconciliation | undefined; + #reconciliationVersion = 0; + #canonicalVersion = 0; + + public readonly ready: Promise; + + public constructor( + writer: CultMeshDocumentWriter, + options: Omit = {}, + ) { + this.#writer = writer; + this.#document = writer.document; + this.#debounceMs = options.debounceMs ?? 0; + this.#replaceDirtyCurrentOnCanonicalSnapshot = + options.replaceDirtyCurrentOnCanonicalSnapshot ?? false; + this.#onError = options.onError ?? ((error) => { + queueMicrotask(() => { + throw error; + }); + }); + this.ready = this.refresh(); + + if (options.watch !== false) { + try { + this.#unsubscribe = this.#document.watch(this.#writer.context, value => { + this.applySnapshot(value); + }); + } catch (error) { + this.#onError(error); + } + } + } + + public get writer(): CultMeshDocumentWriter { + return this.#writer; + } + + public get snapshot(): TDocument { + this.throwIfDisposed(); + return cloneCultMeshDocument(this.#current); + } + + public get dirty(): boolean { + return this.#dirty; + } + public get reconciliation(): CultMeshReactiveDocumentReconciliation | undefined { return this.#reconciliation; } @@ -1099,9 +1213,37 @@ class CultMeshReactiveDocumentHandle } public async refresh(): Promise { - const latest = await this.#document.latest(this.#context); - this.applySnapshot(latest); - return this.#current; + this.throwIfDisposed(); + const versionBeforeRead = this.#canonicalVersion; + const latest = await this.#document.latest(this.#writer.context); + if (versionBeforeRead === this.#canonicalVersion) { + this.applySnapshot(latest); + } + return this.snapshot; + } + + public update(update: (draft: TDocument) => void | TDocument): TDocument { + this.throwIfDisposed(); + const draft = cloneCultMeshDocument(this.#current); + const returned = update(draft); + this.#current = cloneCultMeshDocument(returned ?? draft); + this.#dirty = true; + this.queueSet(); + return this.snapshot; + } + + public replaceLocal(value: TDocument): TDocument { + this.throwIfDisposed(); + this.#current = cloneCultMeshDocument(value); + this.#dirty = true; + this.queueSet(); + return this.snapshot; + } + + public async flush(): Promise { + this.throwIfDisposed(); + this.flushNow(); + await this.#pendingSet; } public clearReconciliation(): void { @@ -1110,10 +1252,8 @@ class CultMeshReactiveDocumentHandle public dispose(): void { this.#disposed = true; - if (this.#flushTimer !== undefined) { - clearTimeout(this.#flushTimer); - this.#flushTimer = undefined; - } + this.#cancelScheduledFlush?.(); + this.#cancelScheduledFlush = undefined; this.#flushScheduled = false; this.#unsubscribe?.(); this.#unsubscribe = undefined; @@ -1121,18 +1261,13 @@ class CultMeshReactiveDocumentHandle private applySnapshot(value: TDocument): void { const canonical = cloneCultMeshDocument(value); + this.#canonicalVersion++; this.captureReconciliation(canonical); - this.#applyingRemote = true; - try { - for (const key of Reflect.ownKeys(this.#current)) { - if (!Reflect.has(canonical, key)) { - Reflect.deleteProperty(this.#current, key); - } - } - Object.assign(this.#current, canonical); - } finally { - this.#applyingRemote = false; + if (this.#dirty && !this.#replaceDirtyCurrentOnCanonicalSnapshot) { + return; } + this.#current = canonical; + this.#dirty = false; } private captureReconciliation(canonical: TDocument): void { @@ -1158,27 +1293,6 @@ class CultMeshReactiveDocumentHandle this.#lastPredicted = undefined; } - private createProxy(seed: TDocument): TDocument { - return new Proxy(seed, { - set: (target, property, value) => { - const changed = !Object.is(Reflect.get(target, property), value); - const ok = Reflect.set(target, property, value); - if (ok && changed && !this.#applyingRemote) { - this.queueSet(); - } - return ok; - }, - deleteProperty: (target, property) => { - const hadProperty = Reflect.has(target, property); - const ok = Reflect.deleteProperty(target, property); - if (ok && hadProperty && !this.#applyingRemote) { - this.queueSet(); - } - return ok; - }, - }); - } - private queueSet(): void { if (this.#disposed) { return; @@ -1188,36 +1302,56 @@ class CultMeshReactiveDocumentHandle } this.#flushScheduled = true; - const flush = () => { - this.#flushScheduled = false; - this.#flushTimer = undefined; - if (this.#disposed) { - return; - } - - const snapshot = cloneCultMeshDocument(this.#current); - this.#lastPredicted = snapshot; - this.#pendingSet = this.#pendingSet - .catch(() => undefined) - .then(() => this.#document.set(this.#context, snapshot)) - .catch(error => this.#onError(error)); - }; + const flush = () => this.flushNow(); const debounceMs = this.#debounceMs; if (debounceMs > 0) { - this.#flushTimer = setTimeout(flush, debounceMs); + const timer = setTimeout(flush, debounceMs); + this.#cancelScheduledFlush = () => clearTimeout(timer); return; } const requestFrame = (globalThis as { requestAnimationFrame?: (callback: FrameRequestCallback) => number; + cancelAnimationFrame?: (handle: number) => void; }).requestAnimationFrame; if (typeof requestFrame === "function") { - requestFrame(() => flush()); + const frame = requestFrame(() => flush()); + const cancelFrame = (globalThis as { + cancelAnimationFrame?: (handle: number) => void; + }).cancelAnimationFrame; + this.#cancelScheduledFlush = typeof cancelFrame === "function" + ? () => cancelFrame(frame) + : undefined; return; } - this.#flushTimer = setTimeout(flush, 0); + let cancelled = false; + queueMicrotask(() => { + if (!cancelled) flush(); + }); + this.#cancelScheduledFlush = () => { cancelled = true; }; + } + + private flushNow(): void { + if (!this.#flushScheduled && !this.#dirty) return; + this.#flushScheduled = false; + this.#cancelScheduledFlush?.(); + this.#cancelScheduledFlush = undefined; + if (this.#disposed || !this.#dirty) return; + const snapshot = cloneCultMeshDocument(this.#current); + this.#dirty = false; + this.#lastPredicted = snapshot; + this.#pendingSet = this.#pendingSet + .catch(() => undefined) + .then(() => this.#writer.write(snapshot)) + .catch(error => this.#onError(error)); + } + + private throwIfDisposed(): void { + if (this.#disposed) { + throw new Error(`Reactive document '${this.#document.documentId}' is disposed.`); + } } } @@ -1295,10 +1429,6 @@ export class CultMeshBoundDocumentHandle { return this.document.canSubmitPrediction; } - public get canSet(): boolean { - return this.document.canSet; - } - public latest(): Promise { return this.document.latest(cultMeshQueryContextFromVerse(this.verse)); } @@ -1319,18 +1449,18 @@ export class CultMeshBoundDocumentHandle { return this.document.submitPrediction(cultMeshQueryContextFromVerse(this.verse), value); } - public set(value: TDocument): Promise { - return this.document.set(cultMeshQueryContextFromVerse(this.verse), value); + public authoritativeWriter(): CultMeshDocumentWriter { + return this.document.authoritativeWriter(cultMeshQueryContextFromVerse(this.verse)); } - public update(update: CultMeshDocumentUpdater): Promise { - return this.document.update(cultMeshQueryContextFromVerse(this.verse), update); + public predictionWriter(): CultMeshDocumentWriter { + return this.document.predictionWriter(cultMeshQueryContextFromVerse(this.verse)); } - public reactive( + public observe( options: Omit = {}, - ): CultMeshReactiveDocument { - return this.document.reactive({ + ): CultMeshObservedDocument { + return this.document.observe({ ...options, context: cultMeshQueryContextFromVerse(this.verse), }); @@ -1451,10 +1581,6 @@ export class CultMeshDocumentCatalog { return this.tryDocument(schema)?.canSubmitPrediction ?? false; } - public canSet(schema: CultMeshDocumentSchemaDescriptor): boolean { - return this.tryDocument(schema)?.canSet ?? false; - } - public submitPrediction( schema: CultMeshDocumentSchemaDescriptor, value: TDocument, @@ -1482,60 +1608,18 @@ export class CultMeshDocumentCatalog { return this.document(schema, options).submitPrediction(context, value); } - public set( + public authoritativeWriter( schema: CultMeshDocumentSchemaDescriptor, - value: TDocument, - options?: { parse?: (value: unknown) => TDocument; context?: CultMeshQueryContext | string }, - ): Promise; - public set( - schema: CultMeshDocumentSchemaDescriptor, - context: CultMeshQueryContext | string, - value: TDocument, - options?: { parse?: (value: unknown) => TDocument }, - ): Promise; - public set( - schema: CultMeshDocumentSchemaDescriptor, - contextOrValue: CultMeshQueryContext | string | TDocument, - valueOrOptions?: TDocument | { parse?: (value: unknown) => TDocument; context?: CultMeshQueryContext | string }, - maybeOptions: { parse?: (value: unknown) => TDocument } = {}, - ): Promise { - const hasContext = - typeof contextOrValue === "string" || isCultMeshQueryContext(contextOrValue); - const context = hasContext - ? contextOrValue as CultMeshQueryContext | string - : (valueOrOptions as { context?: CultMeshQueryContext | string } | undefined)?.context ?? "local"; - const value = hasContext ? valueOrOptions as TDocument : contextOrValue as TDocument; - const options = hasContext ? maybeOptions : valueOrOptions as { parse?: (value: unknown) => TDocument } | undefined; - return this.document(schema, options).set(context, value); + options: { parse?: (value: unknown) => TDocument; context?: CultMeshQueryContext | string } = {}, + ): CultMeshDocumentWriter { + return this.document(schema, options).authoritativeWriter(options.context ?? "local"); } - public update( + public predictionWriter( schema: CultMeshDocumentSchemaDescriptor, - update: CultMeshDocumentUpdater, - options?: { parse?: (value: unknown) => TDocument; context?: CultMeshQueryContext | string }, - ): Promise; - public update( - schema: CultMeshDocumentSchemaDescriptor, - context: CultMeshQueryContext | string, - update: CultMeshDocumentUpdater, - options?: { parse?: (value: unknown) => TDocument }, - ): Promise; - public update( - schema: CultMeshDocumentSchemaDescriptor, - contextOrUpdate: CultMeshQueryContext | string | CultMeshDocumentUpdater, - updateOrOptions?: CultMeshDocumentUpdater | { parse?: (value: unknown) => TDocument; context?: CultMeshQueryContext | string }, - maybeOptions: { parse?: (value: unknown) => TDocument } = {}, - ): Promise { - const hasContext = - typeof contextOrUpdate === "string" || isCultMeshQueryContext(contextOrUpdate); - const context = hasContext - ? contextOrUpdate as CultMeshQueryContext | string - : (updateOrOptions as { context?: CultMeshQueryContext | string } | undefined)?.context ?? "local"; - const update = hasContext - ? updateOrOptions as CultMeshDocumentUpdater - : contextOrUpdate as CultMeshDocumentUpdater; - const options = hasContext ? maybeOptions : updateOrOptions as { parse?: (value: unknown) => TDocument } | undefined; - return this.document(schema, options).update(context, update); + options: { parse?: (value: unknown) => TDocument; context?: CultMeshQueryContext | string } = {}, + ): CultMeshDocumentWriter { + return this.document(schema, options).predictionWriter(options.context ?? "local"); } public watch( @@ -3281,6 +3365,15 @@ export function cultMeshDescribeStateRefResolver( }; } +function stateBindingMetadata( + options: Pick, +): Partial { + return Object.fromEntries( + Object.entries(options).filter(([, value]) => value !== undefined), + ) as Partial; +} + export function cultMeshStateBinding( targetProp: string, pointer: @@ -3294,6 +3387,13 @@ export function cultMeshStateBinding( sourceId?: string; schemaId?: string; routeHint?: CultMeshRouteHint; + bindingName?: string; + documentId?: string; + fieldPath?: string; + valueKind?: CultMeshStateBindingDescriptor["valueKind"]; + accessMode?: CultMeshStateBindingDescriptor["accessMode"]; + authority?: string; + writeCommand?: string; } = {}, ): CultMeshStateBindingDescriptor { requireNonEmpty(targetProp, "targetProp"); @@ -3305,6 +3405,7 @@ export function cultMeshStateBinding( sourceId: options.sourceId, schemaId: options.schemaId, routeHint: options.routeHint ?? cultMeshRouteHint(), + ...stateBindingMetadata(options), }; } @@ -3316,6 +3417,7 @@ export function cultMeshStateBinding( sourceId: options.sourceId ?? source?.sourceId, schemaId: options.schemaId ?? source?.schemaId, routeHint: options.routeHint ?? pointer.routeHint ?? cultMeshRouteHint(), + ...stateBindingMetadata(options), }; } @@ -3329,6 +3431,13 @@ export function cultMeshStateBindingRecord( schemaId?: string, routeKind?: string, routeDescription?: string, + bindingName?: string, + documentId?: string, + fieldPath?: string, + valueKind?: string, + accessMode?: string, + authority?: string, + writeCommand?: string, ): CultMeshStateBindingRecord; export function cultMeshStateBindingRecord( bindingOrTargetProp?: Partial | string, @@ -3337,6 +3446,13 @@ export function cultMeshStateBindingRecord( schemaId?: string, routeKind?: string, routeDescription?: string, + bindingName?: string, + documentId?: string, + fieldPath?: string, + valueKind?: string, + accessMode?: string, + authority?: string, + writeCommand?: string, ): CultMeshStateBindingRecord { if (typeof bindingOrTargetProp === "object" || bindingOrTargetProp === undefined) { const binding = bindingOrTargetProp; @@ -3348,6 +3464,13 @@ export function cultMeshStateBindingRecord( schemaId: binding?.schemaId ?? "", routeKind: route.kind, routeDescription: route.description, + bindingName: binding?.bindingName ?? "", + documentId: binding?.documentId ?? "", + fieldPath: binding?.fieldPath ?? "", + valueKind: binding?.valueKind ?? "", + accessMode: binding?.accessMode ?? "", + authority: binding?.authority ?? "", + writeCommand: binding?.writeCommand ?? "", }; } @@ -3358,6 +3481,13 @@ export function cultMeshStateBindingRecord( schemaId: schemaId ?? "", routeKind: routeKind ?? "", routeDescription: routeDescription ?? "", + bindingName: bindingName ?? "", + documentId: documentId ?? "", + fieldPath: fieldPath ?? "", + valueKind: valueKind ?? "", + accessMode: accessMode ?? "", + authority: authority ?? "", + writeCommand: writeCommand ?? "", }; } @@ -3381,6 +3511,15 @@ export function cultMeshStateBindingFromRecord( }, options.fallbackRouteHint, ), + ...stateBindingMetadata({ + bindingName: record.bindingName || undefined, + documentId: record.documentId || undefined, + fieldPath: record.fieldPath || undefined, + valueKind: (record.valueKind || undefined) as CultMeshStateBindingDescriptor["valueKind"], + accessMode: (record.accessMode || undefined) as CultMeshStateBindingDescriptor["accessMode"], + authority: record.authority || undefined, + writeCommand: record.writeCommand || undefined, + }), }; } @@ -3927,29 +4066,6 @@ export class CultMeshNode { ); } - public reactiveDocument( - definition: TDefinition, - key: string, - options: { - context?: CultMeshQueryContext | string; - watch?: boolean; - debounceMs?: number; - onError?: (error: unknown) => void; - documentId?: string; - routeHint?: CultMeshRouteHint; - pollMs?: number; - } = {}, - ): CultMeshReactiveDocument & object> { - const { - documentId, - routeHint, - pollMs, - ...reactiveOptions - } = options; - return this.document(definition, key, { documentId, routeHint, pollMs }) - .reactive & object>(reactiveOptions); - } - public async syncDocumentFromPeerSnapshot( peer: CultNetPeer | (() => CultNetPeer | Promise), definition: TDefinition, @@ -5005,23 +5121,6 @@ export class CultMesh { return cultMeshGlobalDocumentFromCache(cache, definition, options); } - public static reactiveDocument( - node: CultMeshNode, - definition: TDefinition, - key: string, - options: { - context?: CultMeshQueryContext | string; - watch?: boolean; - debounceMs?: number; - onError?: (error: unknown) => void; - documentId?: string; - routeHint?: CultMeshRouteHint; - pollMs?: number; - } = {}, - ): CultMeshReactiveDocument & object> { - return node.reactiveDocument(definition, key, options); - } - public static syncDocumentFromPeerSnapshot( node: CultMeshNode, peer: CultNetPeer | (() => CultNetPeer | Promise), @@ -5248,6 +5347,13 @@ export class CultMesh { sourceId?: string; schemaId?: string; routeHint?: CultMeshRouteHint; + bindingName?: string; + documentId?: string; + fieldPath?: string; + valueKind?: CultMeshStateBindingDescriptor["valueKind"]; + accessMode?: CultMeshStateBindingDescriptor["accessMode"]; + authority?: string; + writeCommand?: string; } = {}, ): CultMeshStateBindingDescriptor { return cultMeshStateBinding(targetProp, pointer, options); @@ -5263,6 +5369,13 @@ export class CultMesh { schemaId?: string, routeKind?: string, routeDescription?: string, + bindingName?: string, + documentId?: string, + fieldPath?: string, + valueKind?: string, + accessMode?: string, + authority?: string, + writeCommand?: string, ): CultMeshStateBindingRecord; public static stateBindingRecord( bindingOrTargetProp?: Partial | string, @@ -5271,6 +5384,13 @@ export class CultMesh { schemaId?: string, routeKind?: string, routeDescription?: string, + bindingName?: string, + documentId?: string, + fieldPath?: string, + valueKind?: string, + accessMode?: string, + authority?: string, + writeCommand?: string, ): CultMeshStateBindingRecord { return typeof bindingOrTargetProp === "object" || bindingOrTargetProp === undefined ? cultMeshStateBindingRecord(bindingOrTargetProp) @@ -5281,6 +5401,13 @@ export class CultMesh { schemaId, routeKind, routeDescription, + bindingName, + documentId, + fieldPath, + valueKind, + accessMode, + authority, + writeCommand, ); } @@ -5550,7 +5677,7 @@ export class CultMesh { const documents = new CultNetDocumentRegistry(options.bindings); for (const definition of options.documents ?? []) { - cache.registerDocumentType(definition); + await cache.registerDocumentType(definition); if (!documents.get(definition.type)) { documents.register(defineCultNetDocumentBinding({ definition })); } @@ -5689,6 +5816,9 @@ export class CultMesh { if (result.reply) { socket.send(encodeRudpPacket(result.reply), record.remote.port, record.remote.address); } + for (const ready of result.readyToSend ?? []) { + socket.send(encodeRudpPacket(ready), record.remote.port, record.remote.address); + } for (const frame of result.delivered) { if (frame.channelId !== "schema") { continue; @@ -5875,6 +6005,37 @@ export class CultMesh { }; } + public static resolveRudpEndpoint( + endpoint: string, + environment: Record = process.env, + ): CultMeshRudpEndpoint { + requireNonEmpty(endpoint, "endpoint"); + const text = endpoint.trim(); + if (text.toLowerCase().startsWith("rudp://")) { + return CultMesh.parseRudpEndpoint(text); + } + if (!text.toLowerCase().startsWith("cultmesh://")) { + return CultMesh.parseRudpEndpoint(`rudp://${text}`); + } + + const uri = new URL(text); + const authority = uri.hostname.trim(); + if (!authority) throw new Error("CultMesh URI must include an authority."); + const slug = authority + .replace(/[^a-z0-9]+/gi, "_") + .replace(/^_+|_+$/g, "") + .toUpperCase(); + for (const key of [`CULTMESH_URI_${slug}_RUDP`, `${slug}_CULTMESH_RUDP_ENDPOINT`]) { + const resolved = environment[key]?.trim(); + if (resolved) { + return CultMesh.parseRudpEndpoint( + resolved.toLowerCase().startsWith("rudp://") ? resolved : `rudp://${resolved}`, + ); + } + } + throw new Error(`CultMesh URI ${text} did not resolve to a RUDP endpoint.`); + } + public static parseRudpEndpoint(endpoint: string): CultMeshRudpEndpoint { requireNonEmpty(endpoint, "endpoint"); const parsed = new URL(endpoint); @@ -5920,7 +6081,7 @@ export class CultMesh { tags: options.tags, }, )); - await new Promise((resolve) => setTimeout(resolve, Math.max(0, options.flushTimeoutMs ?? 150))); + await peer.flush(options.flushTimeoutMs ?? 30_000); } finally { peer.close(); } diff --git a/packages/cultmesh-ts/test/cultmesh.test.ts b/packages/cultmesh-ts/test/cultmesh.test.ts index 213cc4c4..16d51e1e 100644 --- a/packages/cultmesh-ts/test/cultmesh.test.ts +++ b/packages/cultmesh-ts/test/cultmesh.test.ts @@ -5,7 +5,7 @@ import test from "node:test"; import assert from "node:assert/strict"; import { encode } from "@msgpack/msgpack"; import { z } from "zod"; -import { defineDocumentType } from "cultcache-ts"; +import { defineDocumentType } from "@gamecult/cultcache-ts"; import { CultNetDocumentRegistry, CultNetPeer, @@ -114,7 +114,6 @@ test("CultMesh TS document handles hide local cache plumbing behind typed reacti assert.equal(document.documentId, "cultmesh.note:note:1"); assert.equal(document.canReplace, true); assert.equal(document.canSubmitPrediction, false); - assert.equal(document.canSet, true); assert.equal((await bound.latest()).body, "initial"); await bound.replace({ @@ -131,13 +130,12 @@ test("CultMesh TS document handles hide local cache plumbing behind typed reacti const catalog = CultMesh.documents(document); assert.equal(catalog.canReplace(noteAliasDocument), true); - assert.equal(catalog.canSet(noteAliasDocument), true); - await catalog.set(noteAliasDocument, { - noteId: "note:1", - body: "catalog-updated", - }, { + await catalog.authoritativeWriter(noteAliasDocument, { parse: value => noteAliasDocument.schema.parse(value), context: "browser-client", + }).write({ + noteId: "note:1", + body: "catalog-updated", }); assert.equal( (await catalog.latest(noteAliasDocument, "browser-client", { @@ -145,7 +143,7 @@ test("CultMesh TS document handles hide local cache plumbing behind typed reacti })).body, "catalog-updated", ); - const incremented = await bound.update(value => ({ + const incremented = await bound.authoritativeWriter().update(value => ({ ...value, body: `${value.body}:updated-through-bound-set`, })); @@ -183,17 +181,15 @@ test("CultMesh TS document handles submit predictions through configured authori assert.equal(document.canReplace, false); assert.equal(document.canSubmitPrediction, true); - assert.equal(document.canSet, true); assert.equal(bound.canSubmitPrediction, true); - assert.equal(bound.canSet, true); assert.equal(catalog.canSubmitPrediction(noteAliasDocument), true); - assert.equal(catalog.canSet(noteAliasDocument), true); - await catalog.set(noteAliasDocument, "pilot-a", { + await catalog.predictionWriter(noteAliasDocument, { + parse: value => noteAliasDocument.schema.parse(value), + context: "pilot-a", + }).write({ noteId: "note:prediction", body: "predicted", - }, { - parse: value => noteAliasDocument.schema.parse(value), }); assert.deepEqual(contexts, ["pilot-a:network"]); @@ -204,14 +200,14 @@ test("CultMesh TS document handles submit predictions through configured authori parse: value => noteAliasDocument.schema.parse(value), }); assert.equal(alias.canSubmitPrediction, true); - await alias.set({ + await alias.predictionWriter().write({ noteId: "note:prediction", body: "alias-predicted", }); assert.deepEqual(predictions, ["predicted", "alias-predicted"]); assert.equal((await document.latest("pilot-a")).body, "alias-predicted"); - const updated = await document.update("pilot-a", value => ({ + const updated = await document.predictionWriter("pilot-a").update(value => ({ ...value, body: "updated-as-prediction", })); @@ -227,6 +223,34 @@ test("CultMesh TS document handles submit predictions through configured authori ); }); +test("CultMesh TS documents with both capabilities require an immutable writer choice", async () => { + let current = { noteId: "note:dual", body: "initial" }; + const writes: string[] = []; + const document = CultMesh.document( + "cultmesh.note:dual", + noteDocument, + async () => current, + { + replaceDocument: async (_context, value) => { + writes.push(`authoritative:${value.body}`); + current = value; + }, + submitPrediction: async (_context, value) => { + writes.push(`prediction:${value.body}`); + current = value; + }, + }, + ); + + assert.equal((document as unknown as { set?: unknown }).set, undefined); + await document.authoritativeWriter().write({ noteId: "note:dual", body: "canonical" }); + const predictionWriter = document.predictionWriter(); + await predictionWriter.write({ noteId: "note:dual", body: "predicted" }); + + assert.equal(predictionWriter.kind, "prediction"); + assert.deepEqual(writes, ["authoritative:canonical", "prediction:predicted"]); +}); + test("CultMesh TS store document handles choose compatible foreign schema payloads", async () => { const store = { async pullAll() { @@ -280,7 +304,7 @@ test("CultMesh TS store document handles choose compatible foreign schema payloa }); }); -test("CultMesh TS reactive documents submit predictions from member writes", async () => { +test("CultMesh TS reactive documents submit predictions from explicit updates", async () => { let current = { noteId: "note:reactive", body: "initial", @@ -299,23 +323,71 @@ test("CultMesh TS reactive documents submit predictions from member writes", asy }, ); - const reactive = document.reactive({ + const reactive = document.predictionWriter().reactive({ watch: false, }); await reactive.ready; - reactive.current.body = "member-write-prediction"; - reactive.current.body = "member-write-prediction-final"; + reactive.update(value => { + value.body = "explicit-update-prediction"; + }); + reactive.update(value => { + value.body = "explicit-update-prediction-final"; + }); await waitFor( - () => predictions.includes("member-write-prediction-final"), - "reactive member write prediction", + () => predictions.includes("explicit-update-prediction-final"), + "reactive explicit update prediction", ); - assert.equal(current.body, "member-write-prediction-final"); - assert.deepEqual(predictions, ["member-write-prediction-final"]); + assert.equal(current.body, "explicit-update-prediction-final"); + assert.deepEqual(predictions, ["explicit-update-prediction-final"]); reactive.dispose(); - reactive.current.body = "after-dispose"; + assert.throws(() => reactive.update(value => { + value.body = "after-dispose"; + }), /disposed/); + assert.deepEqual(predictions, ["explicit-update-prediction-final"]); +}); + +test("CultMesh TS reactive updates publish nested edits while snapshot mutation stays local", async () => { + let current = { + shipId: "ship:nested", + loadout: { + weapon: { + key: "weapon:laser", + tuning: { damage: 10 }, + }, + }, + }; + const predictions: typeof current[] = []; + const document = CultMesh.document( + "cultmesh.ship:nested", + { schemaId: "cultmesh.ship.nested.v0" }, + async () => current, + { + submitPrediction: async (_context, value) => { + current = structuredClone(value); + predictions.push(structuredClone(value)); + }, + }, + ); + const reactive = document.predictionWriter().reactive({ watch: false }); + await reactive.ready; + + reactive.update(draft => { + draft.loadout.weapon.tuning.damage = 12; + }); + await reactive.flush(); + + assert.equal(current.loadout.weapon.tuning.damage, 12); + assert.equal(predictions.length, 1); + + const borrowed = reactive.snapshot; + borrowed.loadout.weapon.tuning.damage = 99; await new Promise(resolve => setTimeout(resolve, 20)); - assert.deepEqual(predictions, ["member-write-prediction-final"]); + + assert.equal(reactive.snapshot.loadout.weapon.tuning.damage, 12); + assert.equal(current.loadout.weapon.tuning.damage, 12); + assert.equal(predictions.length, 1); + reactive.dispose(); }); test("CultMesh TS node opens same-schema aliases with one call", async () => { @@ -333,7 +405,7 @@ test("CultMesh TS node opens same-schema aliases with one call", async () => { }); assert.equal(aliasHandle.documentId, "cultmesh.note.ui:note:node-reactive"); assert.equal((await aliasHandle.latest()).body, "initial"); - await aliasHandle.set({ + await aliasHandle.authoritativeWriter().write({ noteId: "note:node-reactive", body: "edited-through-alias-handle", }); @@ -355,26 +427,29 @@ test("CultMesh TS node opens same-schema aliases with one call", async () => { ["edited-through-alias-put"], ); - const reactive = node.reactiveDocument(noteAliasDocument, "note:node-reactive", { - context: CultMesh.queryContext("browser-client"), + const reactive = aliasHandle.authoritativeWriter( + CultMesh.queryContext("browser-client"), + ).reactive({ watch: false, }); await reactive.ready; - reactive.current.body = "edited-through-node-helper"; + reactive.update(value => { + value.body = "edited-through-node-helper"; + }); await waitFor( () => node.getRequired(noteDocument, "note:node-reactive").body === "edited-through-node-helper", "node-level reactive alias write", ); - assert.equal(reactive.current.noteId, "note:node-reactive"); + assert.equal(reactive.snapshot.noteId, "note:node-reactive"); assert.equal( node.document(noteDocument, "note:node-reactive").asSchemaAlias(noteAliasDocument, { parse: value => noteAliasDocument.schema.parse(value), - }).canSet, + }).canReplace, true, ); assert.equal( - (await CultMesh.reactiveDocument(node, noteAliasDocument, "note:node-reactive", { + (await node.document(noteAliasDocument, "note:node-reactive").observe({ watch: false, }).ready).body, "edited-through-node-helper", @@ -409,10 +484,12 @@ test("CultMesh TS reactive documents store reconciliation deltas after mispredic }, ); - const reactive = document.reactive(); + const reactive = document.predictionWriter().reactive(); await reactive.ready; - reactive.current.body = "predicted"; - reactive.current.revision = 7; + reactive.update(value => { + value.body = "predicted"; + value.revision = 7; + }); await waitFor(() => predictions.length === 1, "reactive reconciliation prediction"); watcher?.({ @@ -421,8 +498,8 @@ test("CultMesh TS reactive documents store reconciliation deltas after mispredic revision: 5, }); - assert.equal(reactive.current.body, "authoritative"); - assert.equal(reactive.current.revision, 5); + assert.equal(reactive.snapshot.body, "authoritative"); + assert.equal(reactive.snapshot.revision, 5); assert.equal(reactive.reconciliation?.version, 1); assert.deepEqual(reactive.reconciliation?.canonical, { noteId: "note:reconcile", @@ -444,7 +521,7 @@ test("CultMesh TS reactive documents store reconciliation deltas after mispredic reactive.dispose(); }); -test("CultMesh TS reactive documents coalesce same-frame member writes", async () => { +test("CultMesh TS reactive documents coalesce same-frame explicit updates", async () => { let current = { noteId: "note:reactive-frame", body: "initial", @@ -468,14 +545,14 @@ test("CultMesh TS reactive documents coalesce same-frame member writes", async ( }, ); - const reactive = document.reactive({ + const reactive = document.predictionWriter().reactive({ watch: false, }); try { await reactive.ready; - reactive.current.body = "frame-1"; - reactive.current.body = "frame-2"; - reactive.current.body = "frame-3"; + reactive.update(value => { value.body = "frame-1"; }); + reactive.update(value => { value.body = "frame-2"; }); + reactive.update(value => { value.body = "frame-3"; }); assert.equal(frameCallbacks.length, 1); frameCallbacks[0](performance.now()); @@ -495,6 +572,118 @@ test("CultMesh TS reactive documents coalesce same-frame member writes", async ( } }); +test("CultMesh TS headless reactive documents coalesce on one microtask", async () => { + const previousRequestAnimationFrame = globalThis.requestAnimationFrame; + delete (globalThis as unknown as { + requestAnimationFrame?: (callback: FrameRequestCallback) => number; + }).requestAnimationFrame; + let current = { noteId: "note:reactive-headless", body: "initial" }; + const predictions: string[] = []; + const document = CultMesh.document( + "cultmesh.note:reactive-headless", + noteDocument, + async () => current, + { + submitPrediction: async (_context, value) => { + predictions.push(value.body); + current = value; + }, + }, + ); + const reactive = document.predictionWriter().reactive({ watch: false }); + try { + await reactive.ready; + reactive.update(value => { value.body = "microtask-1"; }); + reactive.update(value => { value.body = "microtask-2"; }); + reactive.update(value => { value.body = "microtask-3"; }); + assert.equal(predictions.length, 0, "the current turn remains coalesced"); + + await waitFor(() => predictions.length === 1, "headless microtask prediction"); + assert.deepEqual(predictions, ["microtask-3"]); + } finally { + reactive.dispose(); + if (previousRequestAnimationFrame) { + globalThis.requestAnimationFrame = previousRequestAnimationFrame; + } + } +}); + +test("CultMesh TS reactive scheduling scales with changed documents only", async () => { + const previousRequestAnimationFrame = globalThis.requestAnimationFrame; + const previousCancelAnimationFrame = globalThis.cancelAnimationFrame; + const frameCallbacks = new Map(); + let nextFrame = 1; + globalThis.requestAnimationFrame = callback => { + const frame = nextFrame++; + frameCallbacks.set(frame, callback); + return frame; + }; + globalThis.cancelAnimationFrame = frame => { + frameCallbacks.delete(frame); + }; + + try { + for (const documentCount of [1, 100, 1000]) { + const reactiveDocuments = []; + for (let index = 0; index < documentCount; index++) { + let current = { + noteId: `note:scale:${index}`, + body: "initial", + }; + const document = CultMesh.document( + `cultmesh.note:scale:${index}`, + noteDocument, + async () => current, + { + submitPrediction: async (_context, value) => { + current = value; + }, + }, + ); + reactiveDocuments.push(document.predictionWriter().reactive({ watch: false })); + } + + await Promise.all(reactiveDocuments.map(document => document.ready)); + assert.equal(frameCallbacks.size, 0, `${documentCount} idle documents scheduled no work`); + + const changedDocumentCount = Math.max(1, Math.floor(documentCount / 100)); + for (let index = 0; index < changedDocumentCount; index++) { + reactiveDocuments[index].update(value => { + value.body = `changed-${index}`; + }); + } + + assert.equal( + frameCallbacks.size, + changedDocumentCount, + `${documentCount} documents scheduled only the ${changedDocumentCount} changed documents`, + ); + assert.equal( + reactiveDocuments.filter(document => document.dirty).length, + changedDocumentCount, + ); + + for (const reactive of reactiveDocuments) reactive.dispose(); + assert.equal(frameCallbacks.size, 0, "dispose cancels every scheduled frame flush"); + } + } finally { + if (previousRequestAnimationFrame) { + globalThis.requestAnimationFrame = previousRequestAnimationFrame; + } else { + delete (globalThis as unknown as { + requestAnimationFrame?: (callback: FrameRequestCallback) => number; + }).requestAnimationFrame; + } + if (previousCancelAnimationFrame) { + globalThis.cancelAnimationFrame = previousCancelAnimationFrame; + } else { + delete (globalThis as unknown as { + cancelAnimationFrame?: (handle: number) => void; + }).cancelAnimationFrame; + } + } +}); + test("CultMesh TS reactive documents queue edits made while a prediction is in flight", async () => { let current = { noteId: "note:reactive-in-flight", @@ -524,14 +713,14 @@ test("CultMesh TS reactive documents queue edits made while a prediction is in f }, ); - const reactive = document.reactive({ + const reactive = document.predictionWriter().reactive({ watch: false, debounceMs: 0, }); await reactive.ready; - reactive.current.body = "first"; + reactive.update(value => { value.body = "first"; }); await firstPredictionStarted; - reactive.current.body = "second"; + reactive.update(value => { value.body = "second"; }); releaseFirstPrediction(); await waitFor(() => predictions.length === 2, "in-flight queued prediction"); @@ -540,13 +729,13 @@ test("CultMesh TS reactive documents queue edits made while a prediction is in f reactive.dispose(); }); -test("CultMesh TS reactive read-only documents surface mutation failures and keep watching", async () => { +test("CultMesh TS observed documents never publish borrowed snapshot mutation and keep watching", async () => { let current = { noteId: "note:reactive-readonly", body: "initial", }; let watcher: ((value: typeof current) => void) | undefined; - const errors: string[] = []; + const writes: typeof current[] = []; const document = CultMesh.document( "cultmesh.note:reactive-readonly", noteDocument, @@ -558,15 +747,17 @@ test("CultMesh TS reactive read-only documents surface mutation failures and kee watcher = undefined; }; }, + submitPrediction: async (_context, value) => { + writes.push(value); + }, }, ); - const reactive = document.reactive({ - onError: error => errors.push(String((error as Error).message ?? error)), - }); - await reactive.ready; - reactive.current.body = "local-readonly-edit"; - await waitFor(() => errors.length === 1, "read-only mutation error"); + const observed = document.observe(); + await observed.ready; + observed.current.body = "borrowed-local-edit"; + await new Promise(resolve => setTimeout(resolve, 20)); + assert.deepEqual(writes, []); current = { noteId: "note:reactive-readonly", @@ -574,9 +765,8 @@ test("CultMesh TS reactive read-only documents surface mutation failures and kee }; watcher?.(current); - assert.match(errors[0], /does not support mutation/); - assert.equal(reactive.current.body, "canonical-after-error"); - reactive.dispose(); + assert.equal(observed.current.body, "canonical-after-error"); + observed.dispose(); }); test("CultMesh TS document handles read schema publications from single-file stores", async () => { @@ -796,7 +986,7 @@ test("CultMesh TS syncs configured publications into local node aliases", async assert.deepEqual(facadeSynced, synced); assert.equal(target.getRequired(noteDocument, "note:published").body, "publication source hydrates local alias"); assert.equal(target.getRequired(noteAliasDocument, "note:published").body, "publication source hydrates local alias"); - assert.equal((await target.reactiveDocument(noteAliasDocument, "note:published", { watch: false }).ready).body, "publication source hydrates local alias"); + assert.equal((await target.document(noteAliasDocument, "note:published").observe({ watch: false }).ready).body, "publication source hydrates local alias"); }); test("CultMesh TS syncs configured publication catalogs into local node aliases", async () => { @@ -1524,7 +1714,7 @@ test("CultMesh TS describes typed surface catalogs for tools and generated bindi "aetheria.zone.render", "gamecult.aetheria.render_body.v1", 128, - [CultMesh.nativeSliceColumn("position", "CultMath.float2", 8)], + [CultMesh.nativeSliceColumn("position", "CultVec2", 8)], { route: routeHint }, ); @@ -1957,6 +2147,13 @@ test("CultMesh TS flattens and rehydrates UI state binding records", () => { schemaId: "gamecult.aetheria.daemon_frame.v1", routeKind: "shared-memory", routeDescription: "co-located frame slab", + bindingName: "", + documentId: "", + fieldPath: "", + valueKind: "", + accessMode: "", + authority: "", + writeCommand: "", }); assert.deepEqual(CultMesh.stateBindingFromRecord({ @@ -1978,6 +2175,26 @@ test("CultMesh TS flattens and rehydrates UI state binding records", () => { }); }); +test("CultMesh TS preserves editable binding identity and authority through transport records", () => { + const binding = CultMesh.stateBinding("value", "ghostlight.session-zero.composer", { + sourceId: "session-zero:draft-7", + schemaId: "session_zero.v1", + routeHint: CultMesh.routeHint("network", "Ghostlight provider"), + bindingName: "composer.message", + documentId: "session-zero:draft-7", + fieldPath: "channels.private.messages.pending", + valueKind: "string", + accessMode: "local-draft", + authority: "SessionZeroKernel", + writeCommand: "edit.value", + }); + const record = CultMesh.stateBindingRecord(binding); + assert.equal(record.bindingName, "composer.message"); + assert.equal(record.fieldPath, "channels.private.messages.pending"); + assert.equal(record.accessMode, "local-draft"); + assert.deepEqual(CultMesh.stateBindingFromRecord(record), binding); +}); + test("CultMesh TS describes UI command bindings from typed operations", async () => { const operation = CultMesh.operation< { actor: string }, @@ -2220,7 +2437,7 @@ test("CultMesh TS exposes native slice descriptors", () => { "gamecult.aetheria.zone_render.v1", 12, [ - CultMesh.nativeSliceColumn("position", "CultMath.float3", 12), + CultMesh.nativeSliceColumn("position", "CultVec3", 12), CultMesh.nativeSliceColumn("rotationRadians", "float32", 4), CultMesh.nativeSliceColumn("renderGroupId", "uint32", 4), ], @@ -2233,7 +2450,7 @@ test("CultMesh TS exposes native slice descriptors", () => { assert.equal(CultMesh.denseRowStrideBytes(view), 20); assert.deepEqual(CultMesh.findNativeSliceColumn(view, "position"), { name: "position", - valueType: "CultMath.float3", + valueType: "CultVec3", elementSizeBytes: 12, }); assert.equal(CultMesh.findNativeSliceColumn(view, "missing"), undefined); @@ -2645,6 +2862,19 @@ test("CultMesh TS publishes one registered document to a RUDP catalog", async () } }); +test("CultMesh TS resolves a CultMesh URI through its authority bootstrap", () => { + assert.deepEqual( + CultMesh.resolveRudpEndpoint("cultmesh://odin/rendezvous/provider-catalog", { + CULTMESH_URI_ODIN_RUDP: "127.0.0.1:17871", + }), + { host: "127.0.0.1", port: 17871, uri: "rudp://127.0.0.1:17871" }, + ); + assert.throws( + () => CultMesh.resolveRudpEndpoint("cultmesh://missing/rendezvous/provider-catalog", {}), + /did not resolve/, + ); +}); + test("CultMesh TS returns a correlated provider receipt after a RUDP document mutation", async () => { const connectionId = 0x10203050; const commandBinding = defineCultNetDocumentBinding({ definition: noteDocument }); diff --git a/packages/cultnet-py/README.md b/packages/cultnet-py/README.md new file mode 100644 index 00000000..cca39ba7 --- /dev/null +++ b/packages/cultnet-py/README.md @@ -0,0 +1,156 @@ +# cultnet-py + +`cultnet-py` is the Python CultNet runtime: schema-v0 MessagePack wire helpers, +4-byte big-endian framing, the raw client, database subscriptions, shard +catch-up and replication, simulation observations and consensus, witness +artifact bundles, the RUDP codec/session/socket transport, and the CultMesh +wire contracts that both a node and a bare peer have to agree on. + +It depends on `cultcache-py` and `msgpack`; AES-GCM helpers on `CultNetSecret` +light up with the optional `crypto` extra. `cultmesh-py` builds on it. The +layering mirrors `cultnet-rs` and `cultnet-ts`. + +## Example + +```python +from datetime import UTC, datetime + +from cultnet_py import ( + CultNetSecret, + CultNetServerSecurityOptions, + CultNetRawClient, + CultNetSimulationObservation, + CultNetWitnessArtifactBundle, + apply_raw_snapshot, + apply_shard_log_response, + compute_simulation_claim_hash, + database_subscribe, + decode_frame, + encode_frame, + hello, + parse_message, + shard_catalog_request, + simulation_observation, + witness_artifact_bundle, +) + +payload = hello(runtime_id="python-runtime").to_bytes() +message = parse_message(decode_frame(encode_frame(payload))) + +client = CultNetRawClient("127.0.0.1", 3075) +catalog = client.fetch_schema_catalog(kinds=["wire_message"]) +snapshot = client.fetch_snapshot_response(schema_ids=["cultnet.interop-note"]) +shard_catalog = client.fetch_shard_catalog(schema_ids=["cultnet.interop-note"]) +# With a local CultCache and the matching registered document definitions: +# applied = apply_raw_snapshot(cache, [note_doc], snapshot) + +subscription = database_subscribe(subscription_id="ui", schema_ids=["cultnet.interop-note"]) +shard_request = shard_catalog_request(message_id="shards", schema_ids=["cultnet.interop-note"]) +shard_log = client.fetch_shard_log_response(shard_id="interop", shard_epoch=1, after_sequence=0) +# log_changes = apply_shard_log_response(cache, [note_doc], shard_log) +with client.subscribe_database(subscription_id="ui", schema_ids=["cultnet.interop-note"]) as live: + initial_snapshot = live.read_next_snapshot_response() + +claim_hash = compute_simulation_claim_hash("frame:42", "subject:player-1", "hit") +observation = simulation_observation( + message_id="obs-1", + witness_runtime_id="python-runtime", + shard_id="interop", + shard_epoch=1, + frame=42, + subject_id="player-1", + claim_kind="hit", + claim_hash=claim_hash, +) +typed_observation = CultNetSimulationObservation.from_wire(observation.to_wire()) +witness = witness_artifact_bundle( + bundle_id="bundle-1", + witness_kind="interop-proof", + captured_at="2026-06-13T00:00:00Z", + subject={"documentType": "cultnet.interop-note", "subjectId": "note:python"}, + contracts=[{"role": "payload", "schemaId": "cultnet.interop-note"}], + artifacts=[{"role": "log", "uri": "cultcache://bundle-1/log", "mediaType": "text/plain"}], + provenance={"pipelineId": "interop", "runId": "run-1", "runtimeId": "python-runtime"}, +) +typed_witness = CultNetWitnessArtifactBundle.from_wire(witness) +witness_payload = typed_witness.to_payload() + +security = CultNetServerSecurityOptions.development() +token = CultNetSecret.create_session_token( + "318fb4b6-ff5e-4c4f-b911-d81807de53a8", + datetime(2035, 1, 1, tzinfo=UTC), + security, + session_version=1, +) +session = CultNetSecret.try_validate_session_token(token, security) +``` + +The raw-client request surface can ride RUDP when the peer advertises a +`rudp://` endpoint and shares the connection id for that lane; +`create_rudp_schema_transport(...)` builds that transport, and +`cultmesh_py.CultMesh.create_client(endpoint="rudp://...")` selects it from an +endpoint string. + +## CultMesh wire contracts + +`cultnet_py.cultmesh_contracts` carries the Verse catalog, peer exchange, +authority lease, stream negotiation, and committed simulation fact contracts. +They live here rather than in `cultmesh-py` for the same reason +`CultMeshPeerCard` lives in `cultnet-rs`: the interop peer has to speak them +without owning a node. `cultmesh_py` re-exports every one of them, so callers +that already import from there keep working. + +## RUDP + +`CultNetRudpSession` is the shared reliable/unreliable/sequenced/ordered +session over the RUDP packet codec, with fragmentation above +`max_fragment_bytes`, a bounded reliable window, reliable expiry, and a +bounded fragment reassembly map that evicts the oldest stranded set and counts +it in `fragment_sets_evicted` rather than refusing new payloads. The +socket-backed transport, reconnect policy/controller, and reconnect loop follow +the Rust and TypeScript entrypoint pattern. + +## Interop peer + +The peer can serve, dial, and probe the same raw-state interop lane used by the +TypeScript, Rust, and C# test peers, and answers Verse catalog and peer exchange +requests without a CultMesh node: + +```powershell +python -m cultnet_py.interop_peer serve --runtime-id python-peer --runtime-kind python --display-name "Python Peer" --agent-id python-agent --advertise-host 127.0.0.1 --tcp-port 3075 --discovery-port 4075 --discovery-group 239.77.44.11 --schema-path ..\cultnet-ts\integration\contracts\cultnet.interop-note.schema.json +python -m cultnet_py.interop_peer dial --runtime-id python-client --runtime-kind python --display-name "Python Client" --agent-id python-client --target-host 127.0.0.1 --target-port 3075 --schema-path ..\cultnet-ts\integration\contracts\cultnet.interop-note.schema.json +python -m cultnet_py.interop_peer probe --runtime-id python-prober --discovery-port 4075 --discovery-group 239.77.44.11 +``` + +`packages/cultnet-ts/test/interop/cultnet-interop.test.ts` drives it inside the +live TS/Rust/C#/Python peer ring. + +## Performance baseline + +The package ships a lightweight benchmark for Python-owned hot paths. In a +CultLib source checkout, it can also run the C# reference benchmark beside it: + +```powershell +python -m cultnet_py.benchmark --records 1000 --json +python -m cultnet_py.compare_csharp --records 1000 --json +``` + +It reports slot-indexed `DatabaseEntry` encode/decode throughput, framed +CultNet MessagePack parse throughput, raw snapshot application into a +registered cache, and public `CultCache` upsert/get throughput. The C# compare +command also runs `packages/cultnet-py/tools/GameCult.Caching.Benchmark` and +reports median Python-to-C# ratios for the shared public cache operations. It +uses three samples by default; pass `--samples` when you need a different +evidence shape. Treat the numbers as a local baseline, not a claim that Python +matches the C# reference in every workload. + +## Tests + +```powershell +$env:PYTHONPATH="$PWD\packages\cultcache-py\src;$PWD\packages\cultnet-py\src" +python -m unittest discover -s packages\cultnet-py\tests +``` + +The suite runs without `cultmesh_py` on the path. See +[docs/python-runtime-parity.md](../../docs/python-runtime-parity.md) for the +evidence map across the three Python packages. diff --git a/packages/cultnet-py/pyproject.toml b/packages/cultnet-py/pyproject.toml new file mode 100644 index 00000000..41c4e8f6 --- /dev/null +++ b/packages/cultnet-py/pyproject.toml @@ -0,0 +1,32 @@ +[build-system] +requires = ["setuptools>=68"] +build-backend = "setuptools.build_meta" + +[project] +name = "cultnet-py" +version = "0.1.0" +description = "Python CultNet transport, wire contracts, and replication for GameCult typed state." +readme = "README.md" +requires-python = ">=3.10" +license = "MIT" +authors = [{ name = "GameCult" }] +classifiers = [ + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", +] +dependencies = ["cultcache-py>=0.2.0", "msgpack>=1.0"] +[project.optional-dependencies] +crypto = ["cryptography>=42"] + +[project.scripts] +cultnet-py-interop = "cultnet_py.interop_peer:main" +cultnet-py-benchmark = "cultnet_py.benchmark:main" +cultnet-py-compare-csharp = "cultnet_py.compare_csharp:main" + +[tool.setuptools.packages.find] +where = ["src"] + +[tool.setuptools.package-data] +cultnet_py = ["py.typed"] diff --git a/packages/cultcache-py/src/cultnet_py/__init__.py b/packages/cultnet-py/src/cultnet_py/__init__.py similarity index 79% rename from packages/cultcache-py/src/cultnet_py/__init__.py rename to packages/cultnet-py/src/cultnet_py/__init__.py index ec54059d..b1734f25 100644 --- a/packages/cultcache-py/src/cultnet_py/__init__.py +++ b/packages/cultnet-py/src/cultnet_py/__init__.py @@ -7,6 +7,34 @@ from .client import ( create_rudp_schema_transport, create_tcp_framed_schema_transport, ) +from .cultmesh_contracts import ( + PEER_EXCHANGE_REQUEST, + PEER_EXCHANGE_RESPONSE, + SIMULATION_FACT_DOCUMENT_TYPE, + SIMULATION_FACT_SCHEMA_VERSION, + VERSE_CATALOG_REQUEST, + VERSE_CATALOG_RESPONSE, + AuthorityLeaseVerifier, + CultMeshAuthorityLease, + CultMeshAuthorityLeaseCatalog, + CultMeshHmacAuthorityLeaseVerifier, + CultMeshPeerCard, + CultMeshPeerCatalog, + CultMeshSimulationFact, + CultMeshStreamCatalog, + CultMeshStreamConsumerProfile, + CultMeshStreamDescriptor, + CultMeshStreamFrameHandle, + CultMeshStreamNegotiation, + CultMeshVerseCatalog, + CultMeshVerseCompatibility, + CultMeshVerseDescriptor, + peer_exchange_request, + peer_from_wire, + simulation_fact_document, + verse_catalog_request, + verse_from_wire, +) from .framing import decode_frame, encode_frame, read_frame, write_frame from .messages import ( CultNetMessage, @@ -108,6 +136,32 @@ from .simulation import ( ) __all__ = [ + "PEER_EXCHANGE_REQUEST", + "PEER_EXCHANGE_RESPONSE", + "SIMULATION_FACT_DOCUMENT_TYPE", + "SIMULATION_FACT_SCHEMA_VERSION", + "VERSE_CATALOG_REQUEST", + "VERSE_CATALOG_RESPONSE", + "AuthorityLeaseVerifier", + "CultMeshAuthorityLease", + "CultMeshAuthorityLeaseCatalog", + "CultMeshHmacAuthorityLeaseVerifier", + "CultMeshPeerCard", + "CultMeshPeerCatalog", + "CultMeshSimulationFact", + "CultMeshStreamCatalog", + "CultMeshStreamConsumerProfile", + "CultMeshStreamDescriptor", + "CultMeshStreamFrameHandle", + "CultMeshStreamNegotiation", + "CultMeshVerseCatalog", + "CultMeshVerseCompatibility", + "CultMeshVerseDescriptor", + "peer_exchange_request", + "peer_from_wire", + "simulation_fact_document", + "verse_catalog_request", + "verse_from_wire", "CultNetMessage", "CultNetWitnessArtifactBundle", "CultNetDatabaseSubscription", diff --git a/packages/cultcache-py/src/cultcache_py/benchmark.py b/packages/cultnet-py/src/cultnet_py/benchmark.py similarity index 98% rename from packages/cultcache-py/src/cultcache_py/benchmark.py rename to packages/cultnet-py/src/cultnet_py/benchmark.py index a444acd9..83834635 100644 --- a/packages/cultcache-py/src/cultcache_py/benchmark.py +++ b/packages/cultnet-py/src/cultnet_py/benchmark.py @@ -19,7 +19,7 @@ class BenchItem: def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser(prog="cultcache-py-benchmark") + parser = argparse.ArgumentParser(prog="cultnet-py-benchmark") parser.add_argument("--records", type=int, default=5000) parser.add_argument("--json", action="store_true", dest="as_json") args = parser.parse_args(argv) diff --git a/packages/cultcache-py/src/cultnet_py/client.py b/packages/cultnet-py/src/cultnet_py/client.py similarity index 100% rename from packages/cultcache-py/src/cultnet_py/client.py rename to packages/cultnet-py/src/cultnet_py/client.py diff --git a/packages/cultcache-py/src/cultcache_py/compare_csharp.py b/packages/cultnet-py/src/cultnet_py/compare_csharp.py similarity index 99% rename from packages/cultcache-py/src/cultcache_py/compare_csharp.py rename to packages/cultnet-py/src/cultnet_py/compare_csharp.py index 8a9e4c5b..a2654a33 100644 --- a/packages/cultcache-py/src/cultcache_py/compare_csharp.py +++ b/packages/cultnet-py/src/cultnet_py/compare_csharp.py @@ -16,7 +16,7 @@ DEFAULT_PARITY_THRESHOLD = 0.90 def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser(prog="cultcache-py-compare-csharp") + parser = argparse.ArgumentParser(prog="cultnet-py-compare-csharp") parser.add_argument("--records", type=int, default=5000) parser.add_argument("--samples", type=int, default=DEFAULT_SAMPLE_COUNT) parser.add_argument("--parity-threshold", type=float, default=DEFAULT_PARITY_THRESHOLD) diff --git a/packages/cultcache-py/src/cultmesh_py/wire.py b/packages/cultnet-py/src/cultnet_py/cultmesh_contracts.py similarity index 84% rename from packages/cultcache-py/src/cultmesh_py/wire.py rename to packages/cultnet-py/src/cultnet_py/cultmesh_contracts.py index 9a666a1e..91d0805b 100644 --- a/packages/cultcache-py/src/cultmesh_py/wire.py +++ b/packages/cultnet-py/src/cultnet_py/cultmesh_contracts.py @@ -1,14 +1,18 @@ from __future__ import annotations import base64 +import hashlib import hmac from dataclasses import dataclass, field, replace -from datetime import datetime +from datetime import UTC, datetime from hashlib import sha256 from typing import Any, Callable, Mapping import msgpack +from cultcache_py import define_database_entry_type +from cultcache_py.documents import DocumentDefinition + VERSE_CATALOG_REQUEST = "cultmesh.verse_catalog_request.v0" VERSE_CATALOG_RESPONSE = "cultmesh.verse_catalog_response.v0" @@ -613,3 +617,104 @@ def copy_budget_for(transport: str) -> str: if transport == "inline-bytes": return "opaque-runtime" return "opaque-runtime" + + +SIMULATION_FACT_DOCUMENT_TYPE = "gamecult.mesh.simulation_fact" +SIMULATION_FACT_SCHEMA_VERSION = "gamecult.mesh.simulation_fact.v1" + + +@dataclass(frozen=True) +class CultMeshSimulationFact: + fact_id: str + shard_id: str + shard_epoch: int + frame: int + subject_id: str + claim_kind: str + claim_hash: str + claim_summary: str | None + witness_count: int + support_weight: float + total_weight: float + confidence: float + committed_at: str + + @staticmethod + def compute_fact_id(candidate: dict[str, Any]) -> str: + canonical = "\x1f".join([ + _candidate_string(candidate, "shardId", "shard_id"), + str(_candidate_int(candidate, "shardEpoch", "shard_epoch")), + str(_candidate_int(candidate, "frame", "frame")), + _candidate_string(candidate, "subjectId", "subject_id"), + _candidate_string(candidate, "claimKind", "claim_kind"), + ]) + return hashlib.sha256(canonical.encode("utf-8")).hexdigest() + + @staticmethod + def create_record_key(candidate: dict[str, Any]) -> str: + return f"simulation:{CultMeshSimulationFact.compute_fact_id(candidate)}" + + @staticmethod + def from_candidate(candidate: dict[str, Any], *, committed_at: str | None = None) -> "CultMeshSimulationFact": + return CultMeshSimulationFact( + fact_id=CultMeshSimulationFact.compute_fact_id(candidate), + shard_id=_candidate_string(candidate, "shardId", "shard_id"), + shard_epoch=_candidate_int(candidate, "shardEpoch", "shard_epoch"), + frame=_candidate_int(candidate, "frame", "frame"), + subject_id=_candidate_string(candidate, "subjectId", "subject_id"), + claim_kind=_candidate_string(candidate, "claimKind", "claim_kind"), + claim_hash=_candidate_string(candidate, "claimHash", "claim_hash"), + claim_summary=_candidate_optional_string(candidate, "claimSummary", "claim_summary"), + witness_count=_candidate_int(candidate, "witnessCount", "witness_count"), + support_weight=_candidate_float(candidate, "supportWeight", "support_weight"), + total_weight=_candidate_float(candidate, "totalWeight", "total_weight"), + confidence=_candidate_float(candidate, "confidence", "confidence"), + committed_at=committed_at or datetime.now(UTC).isoformat(), + ) + + +simulation_fact_document: DocumentDefinition[CultMeshSimulationFact] = define_database_entry_type( + SIMULATION_FACT_DOCUMENT_TYPE, + [ + ("fact_id", 0), + ("shard_id", 1), + ("shard_epoch", 2), + ("frame", 3), + ("subject_id", 4), + ("claim_kind", 5), + ("claim_hash", 6), + ("claim_summary", 7, None), + ("witness_count", 8), + ("support_weight", 9), + ("total_weight", 10), + ("confidence", 11), + ("committed_at", 12), + ], + cls=CultMeshSimulationFact, + name="fact_id", + indexes={"shard": "shard_id", "frame": "frame", "subject": "subject_id", "claim_kind": "claim_kind"}, + schema_name=SIMULATION_FACT_DOCUMENT_TYPE, + schema_version=SIMULATION_FACT_SCHEMA_VERSION, +) + + +def _candidate_string(candidate: dict[str, Any], camel: str, snake: str) -> str: + value = candidate.get(camel, candidate.get(snake, "")) + if value is None: + return "" + return str(value) + + +def _candidate_optional_string(candidate: dict[str, Any], camel: str, snake: str) -> str | None: + value = candidate.get(camel, candidate.get(snake)) + if value is None: + return None + return str(value) + + +def _candidate_int(candidate: dict[str, Any], camel: str, snake: str) -> int: + return int(candidate.get(camel, candidate.get(snake, 0))) + + +def _candidate_float(candidate: dict[str, Any], camel: str, snake: str) -> float: + return float(candidate.get(camel, candidate.get(snake, 0.0))) diff --git a/packages/cultcache-py/src/cultnet_py/framing.py b/packages/cultnet-py/src/cultnet_py/framing.py similarity index 100% rename from packages/cultcache-py/src/cultnet_py/framing.py rename to packages/cultnet-py/src/cultnet_py/framing.py diff --git a/packages/cultcache-py/src/cultnet_py/interop_peer.py b/packages/cultnet-py/src/cultnet_py/interop_peer.py similarity index 99% rename from packages/cultcache-py/src/cultnet_py/interop_peer.py rename to packages/cultnet-py/src/cultnet_py/interop_peer.py index d26f1b70..ca3c692d 100644 --- a/packages/cultcache-py/src/cultnet_py/interop_peer.py +++ b/packages/cultnet-py/src/cultnet_py/interop_peer.py @@ -24,7 +24,7 @@ from cultcache_py import ( define_document_type, ) from cultcache_py.documents import DocumentDefinition -from cultmesh_py import ( +from .cultmesh_contracts import ( PEER_EXCHANGE_REQUEST, SIMULATION_FACT_DOCUMENT_TYPE, VERSE_CATALOG_REQUEST, @@ -306,6 +306,8 @@ def rudp_loop( result = peer.session.receive(packet, now_ms()) if result.reply is not None: send_rudp_packet(sock, remote, result.reply) + for ready in result.ready_to_send: + send_rudp_packet(sock, remote, ready) if result.disconnected: peers.pop(remote, None) continue @@ -317,8 +319,8 @@ def rudp_loop( continue for response in handle_server_message(state, message, peer.subscriptions): send_rudp_schema_frame(sock, remote, peer.session, response) - if packet.packet_type == CultNetRudpPacketType.DATA or result.delivered: - send_rudp_packet(sock, remote, peer.session.create_ack()) + if packet.reliable or packet.packet_type == CultNetRudpPacketType.DATA or result.delivered: + send_rudp_packet(sock, remote, peer.session.create_ack_for_received(packet.sequence)) except Exception as error: sys.stderr.write(json.dumps({ "event": "rudpMessageError", diff --git a/packages/cultcache-py/src/cultnet_py/messages.py b/packages/cultnet-py/src/cultnet_py/messages.py similarity index 100% rename from packages/cultcache-py/src/cultnet_py/messages.py rename to packages/cultnet-py/src/cultnet_py/messages.py diff --git a/packages/cultcache-py/src/cultnet_py/prediction.py b/packages/cultnet-py/src/cultnet_py/prediction.py similarity index 100% rename from packages/cultcache-py/src/cultnet_py/prediction.py rename to packages/cultnet-py/src/cultnet_py/prediction.py diff --git a/packages/cultcache-py/src/cultnet_py/py.typed b/packages/cultnet-py/src/cultnet_py/py.typed similarity index 100% rename from packages/cultcache-py/src/cultnet_py/py.typed rename to packages/cultnet-py/src/cultnet_py/py.typed diff --git a/packages/cultcache-py/src/cultnet_py/replication.py b/packages/cultnet-py/src/cultnet_py/replication.py similarity index 100% rename from packages/cultcache-py/src/cultnet_py/replication.py rename to packages/cultnet-py/src/cultnet_py/replication.py diff --git a/packages/cultcache-py/src/cultnet_py/schema_catalog.py b/packages/cultnet-py/src/cultnet_py/schema_catalog.py similarity index 100% rename from packages/cultcache-py/src/cultnet_py/schema_catalog.py rename to packages/cultnet-py/src/cultnet_py/schema_catalog.py diff --git a/packages/cultcache-py/src/cultnet_py/security.py b/packages/cultnet-py/src/cultnet_py/security.py similarity index 100% rename from packages/cultcache-py/src/cultnet_py/security.py rename to packages/cultnet-py/src/cultnet_py/security.py diff --git a/packages/cultcache-py/src/cultnet_py/shard_catalog.py b/packages/cultnet-py/src/cultnet_py/shard_catalog.py similarity index 100% rename from packages/cultcache-py/src/cultnet_py/shard_catalog.py rename to packages/cultnet-py/src/cultnet_py/shard_catalog.py diff --git a/packages/cultcache-py/src/cultnet_py/shard_log.py b/packages/cultnet-py/src/cultnet_py/shard_log.py similarity index 100% rename from packages/cultcache-py/src/cultnet_py/shard_log.py rename to packages/cultnet-py/src/cultnet_py/shard_log.py diff --git a/packages/cultcache-py/src/cultnet_py/shard_replication.py b/packages/cultnet-py/src/cultnet_py/shard_replication.py similarity index 100% rename from packages/cultcache-py/src/cultnet_py/shard_replication.py rename to packages/cultnet-py/src/cultnet_py/shard_replication.py diff --git a/packages/cultcache-py/src/cultnet_py/simulation.py b/packages/cultnet-py/src/cultnet_py/simulation.py similarity index 100% rename from packages/cultcache-py/src/cultnet_py/simulation.py rename to packages/cultnet-py/src/cultnet_py/simulation.py diff --git a/packages/cultcache-py/src/cultnet_py/snapshot.py b/packages/cultnet-py/src/cultnet_py/snapshot.py similarity index 100% rename from packages/cultcache-py/src/cultnet_py/snapshot.py rename to packages/cultnet-py/src/cultnet_py/snapshot.py diff --git a/packages/cultcache-py/src/cultnet_py/subscription.py b/packages/cultnet-py/src/cultnet_py/subscription.py similarity index 100% rename from packages/cultcache-py/src/cultnet_py/subscription.py rename to packages/cultnet-py/src/cultnet_py/subscription.py diff --git a/packages/cultcache-py/src/cultnet_py/transport.py b/packages/cultnet-py/src/cultnet_py/transport.py similarity index 82% rename from packages/cultcache-py/src/cultnet_py/transport.py rename to packages/cultnet-py/src/cultnet_py/transport.py index 25a4c891..73bd1e62 100644 --- a/packages/cultcache-py/src/cultnet_py/transport.py +++ b/packages/cultnet-py/src/cultnet_py/transport.py @@ -280,6 +280,7 @@ class CultNetRudpDeliveredFrame: @dataclass(frozen=True) class CultNetRudpReceiveResult: delivered: tuple[CultNetRudpDeliveredFrame, ...] = field(default_factory=tuple) + ready_to_send: tuple[CultNetRudpPacket, ...] = field(default_factory=tuple) reply: CultNetRudpPacket | None = None pong: bool = False pong_payload: bytes = b"" @@ -395,6 +396,9 @@ def create_rudp_transport_profile( class CultNetRudpSession: + RELIABLE_SEND_WINDOW_PACKETS = 32 + RECEIVED_SEQUENCE_WINDOW = 4_096 + def __init__(self, options: CultNetRudpSessionOptions) -> None: self.connection_id = _uint32(options.connection_id, "connection_id") self.resend_delay_ms = options.resend_delay_ms @@ -402,20 +406,33 @@ class CultNetRudpSession: raise ValueError("RUDP max_pending_reliable_packets must be greater than zero") self.max_pending_reliable_packets = options.max_pending_reliable_packets self._next_sequence = _uint32(options.initial_sequence, "initial_sequence") + self._next_sequenced_by_channel: dict[str, int] = {} self._next_fragment_id = 1 self._connected = False self._last_received_at_ms: int | None = None self._highest_received_sequence: int | None = None self._received_sequences: set[int] = set() + self._latest_sequenced_by_channel: dict[str, int] = {} self._pending_reliable: dict[int, _PendingReliablePacket] = {} + self._queued_reliable: deque[CultNetRudpPacket] = deque() self._ordered_next_sequence_by_channel: dict[str, int] = {} self._ordered_buffers: dict[str, dict[int, tuple[CultNetRudpDeliveredFrame, int]]] = {} self._fragment_buffers: dict[tuple[str, int], dict[str, Any]] = {} + # Matches cultnet-rs max_pending_fragment_sets. A set that loses one + # fragment is never completed; without a bound it is kept for the life + # of the session and the map grows under any loss on a fragmenting channel. + self._max_pending_fragment_sets = 64 + self._fragment_sets_evicted = 0 @property def connected(self) -> bool: return self._connected + @property + def fragment_sets_evicted(self) -> int: + """Incomplete fragment sets dropped to admit newer ones; one lost payload each.""" + return self._fragment_sets_evicted + @property def last_received_at_ms(self) -> int | None: return self._last_received_at_ms @@ -424,6 +441,14 @@ class CultNetRudpSession: def pending_reliable_sequences(self) -> tuple[int, ...]: return tuple(sorted(self._pending_reliable)) + @property + def queued_reliable_packet_count(self) -> int: + return len(self._queued_reliable) + + @property + def outstanding_reliable_packet_count(self) -> int: + return len(self._pending_reliable) + len(self._queued_reliable) + def create_connect(self, now_ms: int = 0, payload: bytes = b"") -> CultNetRudpPacket: self._ensure_reliable_capacity(1) packet = self._create_packet( @@ -465,7 +490,10 @@ class CultNetRudpSession: payload: bytes, options: CultNetRudpSendOptions | None = None, ) -> CultNetRudpPacket: - return self.send_many(channel_id, payload, options)[0] + resolved = options or CultNetRudpSendOptions() + if resolved.reliable and len(self._pending_reliable) >= self.RELIABLE_SEND_WINDOW_PACKETS: + raise ValueError("RUDP reliable send window is full; receive acknowledgements before sending") + return self.send_many(channel_id, payload, resolved)[0] def send_many( self, @@ -479,6 +507,8 @@ class CultNetRudpSession: raise ValueError("Cannot send RUDP data before the session is connected") options = options or CultNetRudpSendOptions() + if options.ordered and not options.reliable: + raise ValueError("RUDP ordered delivery requires reliability") if max_fragment_bytes is not None and max_fragment_bytes <= 0: raise ValueError("RUDP max_fragment_bytes must be greater than zero") @@ -492,9 +522,7 @@ class CultNetRudpSession: ordered=options.ordered, sequenced=options.sequenced, ) - if packet.reliable: - self._track_reliable(packet, options.now_ms) - return (packet,) + return self._admit_reliable_packets((packet,), options.now_ms) if packet.reliable else (packet,) fragment_count = (len(payload) + max_fragment_bytes - 1) // max_fragment_bytes if fragment_count > 0xFFFF: @@ -516,14 +544,13 @@ class CultNetRudpSession: fragment_index=index, fragment_count=fragment_count, ) - if packet.reliable: - self._track_reliable(packet, options.now_ms) packets.append(packet) - return tuple(packets) + return self._admit_reliable_packets(tuple(packets), options.now_ms) if options.reliable else tuple(packets) def receive(self, packet: CultNetRudpPacket, now_ms: int = 0) -> CultNetRudpReceiveResult: self._require_connection(packet) self._apply_acknowledgements(packet) + ready_to_send = self._promote_queued_reliable(now_ms) self._last_received_at_ms = now_ms expected_sequence_if_uninitialized = ( packet.sequence @@ -534,51 +561,93 @@ class CultNetRudpSession: if packet.packet_type == CultNetRudpPacketType.ACCEPT: self._remember_received(packet.sequence) self._connected = True - return CultNetRudpReceiveResult() + return CultNetRudpReceiveResult(ready_to_send=ready_to_send) if packet.packet_type == CultNetRudpPacketType.PING: - self._remember_received(packet.sequence) return CultNetRudpReceiveResult( + ready_to_send=ready_to_send, reply=self._create_packet(CultNetRudpPacketType.PONG, "control", packet.payload) ) if packet.packet_type in (CultNetRudpPacketType.ACK, CultNetRudpPacketType.PONG): - self._remember_received(packet.sequence) return CultNetRudpReceiveResult( + ready_to_send=ready_to_send, pong=packet.packet_type == CultNetRudpPacketType.PONG, pong_payload=bytes(packet.payload) if packet.packet_type == CultNetRudpPacketType.PONG else b"", ) if packet.packet_type == CultNetRudpPacketType.DISCONNECT: - self._remember_received(packet.sequence) self._connected = False return CultNetRudpReceiveResult( + ready_to_send=ready_to_send, disconnected=True, disconnect_reason=bytes(packet.payload), ) if packet.packet_type != CultNetRudpPacketType.DATA: - return CultNetRudpReceiveResult() + return CultNetRudpReceiveResult(ready_to_send=ready_to_send) - duplicate = packet.sequence in self._received_sequences - self._remember_received(packet.sequence) + duplicate = packet.reliable and ( + packet.sequence in self._received_sequences or ( + self._highest_received_sequence is not None + and packet.sequence < self._highest_received_sequence + and self._highest_received_sequence - packet.sequence >= self.RECEIVED_SEQUENCE_WINDOW + ) + ) + if packet.reliable: + self._remember_received(packet.sequence) if duplicate: - return CultNetRudpReceiveResult() + return CultNetRudpReceiveResult(ready_to_send=ready_to_send) reassembled = self._reassemble(packet) if reassembled is None: - return CultNetRudpReceiveResult() + return CultNetRudpReceiveResult(ready_to_send=ready_to_send) frame, ordered, next_sequence = reassembled + if not ordered and packet.sequenced: + newest_sequence = next_sequence - 1 + latest_sequence = self._latest_sequenced_by_channel.get(frame.channel_id) + if latest_sequence is not None and newest_sequence <= latest_sequence: + return CultNetRudpReceiveResult(ready_to_send=ready_to_send) + self._latest_sequenced_by_channel[frame.channel_id] = newest_sequence + return CultNetRudpReceiveResult(delivered=(frame,), ready_to_send=ready_to_send) if not ordered: - return CultNetRudpReceiveResult(delivered=(frame,)) + return CultNetRudpReceiveResult(delivered=(frame,), ready_to_send=ready_to_send) return CultNetRudpReceiveResult( delivered=tuple( self._deliver_ordered(frame, next_sequence, expected_sequence_if_uninitialized) - ) + ), + ready_to_send=ready_to_send, ) def create_ack(self) -> CultNetRudpPacket: - return self._create_packet(CultNetRudpPacketType.ACK, "control", b"") + ack, ack_mask = self._ack_state() + return CultNetRudpPacket( + packet_type=CultNetRudpPacketType.ACK, + connection_id=self.connection_id, + sequence=0, + ack=ack, + ack_mask=ack_mask, + channel_id="control", + payload=b"", + ) + + def create_ack_for(self, sequence: int) -> CultNetRudpPacket: + return CultNetRudpPacket( + packet_type=CultNetRudpPacketType.ACK, + connection_id=self.connection_id, + sequence=0, + ack=_uint32(sequence, "ack sequence"), + ack_mask=0, + channel_id="control", + payload=b"", + ) + + def create_ack_for_received(self, sequence: int) -> CultNetRudpPacket: + received_sequence = _uint32(sequence, "received sequence") + ack, _ = self._ack_state() + if ack >= received_sequence and ack - received_sequence <= 32: + return self.create_ack() + return self.create_ack_for(received_sequence) def create_ping(self, payload: bytes = b"") -> CultNetRudpPacket: return self._create_packet(CultNetRudpPacketType.PING, "control", payload) @@ -616,8 +685,17 @@ class CultNetRudpSession: fragment_index: int = 0, fragment_count: int = 0, ) -> CultNetRudpPacket: - sequence = self._next_sequence - self._next_sequence = _uint32(self._next_sequence + 1, "sequence") + if reliable: + sequence = self._next_sequence + self._next_sequence = _uint32(self._next_sequence + 1, "reliable sequence") + elif sequenced: + sequence = self._next_sequenced_by_channel.get(channel_id, 1) + self._next_sequenced_by_channel[channel_id] = _uint32( + sequence + 1, + "sequenced channel sequence", + ) + else: + sequence = 0 ack, ack_mask = self._ack_state() return CultNetRudpPacket( packet_type=packet_type, @@ -655,10 +733,31 @@ class CultNetRudpSession: last_sent_at_ms=now_ms, ) + def _admit_reliable_packets( + self, + packets: tuple[CultNetRudpPacket, ...], + now_ms: int, + ) -> tuple[CultNetRudpPacket, ...]: + available = max(0, self.RELIABLE_SEND_WINDOW_PACKETS - len(self._pending_reliable)) + ready = packets[:available] + for packet in ready: + self._track_reliable(packet, now_ms) + self._queued_reliable.extend(packets[available:]) + return ready + + def _promote_queued_reliable(self, now_ms: int) -> tuple[CultNetRudpPacket, ...]: + available = max(0, self.RELIABLE_SEND_WINDOW_PACKETS - len(self._pending_reliable)) + ready: list[CultNetRudpPacket] = [] + while len(ready) < available and self._queued_reliable: + packet = self._queued_reliable.popleft() + self._track_reliable(packet, now_ms) + ready.append(packet) + return tuple(ready) + def _ensure_reliable_capacity(self, packet_count: int) -> None: if packet_count == 0 or self.max_pending_reliable_packets is None: return - if len(self._pending_reliable) + packet_count > self.max_pending_reliable_packets: + if self.outstanding_reliable_packet_count + packet_count > self.max_pending_reliable_packets: raise ValueError("RUDP reliable send queue is full") def _apply_acknowledgements(self, packet: CultNetRudpPacket) -> None: @@ -671,6 +770,9 @@ class CultNetRudpSession: self._received_sequences.add(sequence) if self._highest_received_sequence is None or sequence > self._highest_received_sequence: self._highest_received_sequence = sequence + if len(self._received_sequences) > self.RECEIVED_SEQUENCE_WINDOW: + keep_from = max(0, (self._highest_received_sequence or sequence) - self.RECEIVED_SEQUENCE_WINDOW + 1) + self._received_sequences = {received for received in self._received_sequences if received >= keep_from} def _ack_state(self) -> tuple[int, int]: ack = self._highest_received_sequence or 0 @@ -693,6 +795,11 @@ class CultNetRudpSession: raise ValueError("RUDP fragment index must be lower than fragment count") key = (packet.channel_id, packet.fragment_id) + if key not in self._fragment_buffers and len(self._fragment_buffers) >= self._max_pending_fragment_sets: + # Evict the oldest stranded set rather than refusing the payload; dicts + # iterate in insertion order, so the first key is the oldest. + del self._fragment_buffers[next(iter(self._fragment_buffers))] + self._fragment_sets_evicted += 1 buffer = self._fragment_buffers.setdefault( key, { @@ -838,6 +945,10 @@ class CultNetRudpSocketTransportConnection: frames_sent=self._frames_sent, ) + @property + def outstanding_reliable_packet_count(self) -> int: + return self.session.outstanding_reliable_packet_count + def connect(self, payload: bytes = b"") -> None: if self.mode != CultNetRudpSocketMode.CLIENT: raise ValueError("Only a client RUDP socket transport can initiate connect") @@ -879,6 +990,8 @@ class CultNetRudpSocketTransportConnection: result = self.session.receive(packet, _now_ms()) if result.reply is not None: self._send_packet(result.reply) + for ready in result.ready_to_send: + self._send_packet(ready) if result.pong: self.pong_payloads.append(result.pong_payload) if result.disconnected: @@ -892,8 +1005,8 @@ class CultNetRudpSocketTransportConnection: self._frames_received += 1 frame = self._delivered_frames.popleft() if self._delivered_frames else None - if packet.packet_type == CultNetRudpPacketType.ACCEPT or frame is not None: - self._send_packet(self.session.create_ack()) + if packet.reliable or packet.packet_type == CultNetRudpPacketType.ACCEPT or frame is not None: + self._send_packet(self.session.create_ack_for_received(packet.sequence)) return frame def receive(self, timeout_seconds: float | None = None) -> CultNetTransportFrame: @@ -906,6 +1019,29 @@ class CultNetRudpSocketTransportConnection: if deadline is not None and time.monotonic() >= deadline: raise TimeoutError("Timed out waiting for RUDP schema frame") + def flush_reliable(self, timeout_seconds: float = 30.0) -> None: + deadline = time.monotonic() + max(0.0, timeout_seconds) + original_timeout = self.socket.gettimeout() + poll_timeout = min(original_timeout, 0.01) if original_timeout is not None else 0.01 + preserved = deque(self._delivered_frames) + self._delivered_frames.clear() + self.socket.settimeout(poll_timeout) + try: + while self.session.outstanding_reliable_packet_count > 0: + if time.monotonic() >= deadline: + raise TimeoutError( + "RUDP reliable flush timed out with " + f"{self.session.outstanding_reliable_packet_count} packets outstanding" + ) + frame = self.receive_once() + if frame is not None: + preserved.append(frame) + self.poll_resends() + finally: + preserved.extend(self._delivered_frames) + self._delivered_frames = preserved + self.socket.settimeout(original_timeout) + def disconnect(self, reason: bytes = b"") -> None: self._send_packet(self.session.create_disconnect(reason)) diff --git a/packages/cultnet-py/tests/test_cultnet.py b/packages/cultnet-py/tests/test_cultnet.py new file mode 100644 index 00000000..171a34b9 --- /dev/null +++ b/packages/cultnet-py/tests/test_cultnet.py @@ -0,0 +1,2046 @@ +from __future__ import annotations + +import json +import socket +import threading +import time +import unittest +import hashlib +import hmac +import io +from dataclasses import dataclass +from datetime import UTC, datetime +from pathlib import Path +from typing import Callable +from uuid import uuid4 +from cultcache_py import CultCache, define_database_entry_type +from cultnet_py.benchmark import run_benchmark +from cultnet_py.compare_csharp import ( + DEFAULT_PARITY_THRESHOLD, + DEFAULT_SAMPLE_COUNT, + _compare_common_metrics, + _median_result, + _parity_status, +) +from cultnet_py.interop_peer import append_shard_log_put, build_state, raw_snapshot_response +from cultnet_py import ( + compute_simulation_claim_hash, + CultNetDatabaseChange, + CultNetRawClient, + CultNetRawDocumentRecord, + CultNetRawSnapshotResponse, + CultNetSchemaCatalog, + CultNetSchemaDescriptor, + CultNetShardCatalog, + CultNetShardDescriptor, + CultNetShardLogEntry, + CultNetShardLogResponse, + CultNetPeerError, + CultNetClientSecurityOptions, + CultNetSecret, + CultNetServerSecurityOptions, + CultNetSimulationConsensusOptions, + CultNetSimulationConsensusCandidate, + CultNetSimulationObservation, + CultNetRudpPacket, + CultNetRudpPacketType, + CultNetRudpSendOptions, + CultNetRudpSession, + CultNetRudpSessionOptions, + CultNetRudpSocketMode, + CultNetRudpReconnectLoop, + CultNetRudpSocketTransportConnection, + CultNetRudpSocketTransportOptions, + TcpFramedTransportConnection, + CultNetWitnessArtifactBundle, + apply_raw_document_record, + apply_raw_snapshot, + apply_shard_log_response, + create_rudp_transport_profile, + create_tcp_framed_transport_profile, + create_reconnect_policy, + CultNetReconnectController, + database_subscribe, + database_unsubscribe, + decode_frame, + decode_rudp_packet, + decode_witness_artifact_bundle_payload, + document_delete, + document_put_raw, + encode_frame, + encode_rudp_packet, + encode_witness_artifact_bundle_payload, + hello, + login, + login_success, + parse_message, + register, + schema_catalog_request, + schema_document_map, + shard_catalog_request, + shard_log_request, + simulation_observation, + snapshot_request, + verify_session, + wire_message_schema_catalog, + wire_message_schema_descriptors, + witness_artifact_bundle, + compute_reconnect_delay_ms, +) + + +def bind_udp_socket() -> socket.socket: + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + sock.bind(("127.0.0.1", 0)) + sock.settimeout(0.02) + return sock + + +def pump_rudp_handshake( + client: CultNetRudpSocketTransportConnection, + server: CultNetRudpSocketTransportConnection, +) -> None: + for _ in range(20): + server.receive_once() + client.receive_once() + server.receive_once() + if client.connected and server.connected: + return + time.sleep(0.005) + raise AssertionError("RUDP socket handshake did not complete") + + +def receive_rudp_frame(transport: CultNetRudpSocketTransportConnection): + for _ in range(20): + frame = transport.receive_once() + if frame is not None: + return frame + time.sleep(0.005) + raise AssertionError("RUDP socket frame was not delivered") + + +@dataclass +class Item: + name: str + category: str + value: int + + +class CultNetTests(unittest.TestCase): + def test_benchmark_reports_core_hot_path_metrics(self) -> None: + result = run_benchmark(8) + metric_names = {metric["name"] for metric in result["metrics"]} + self.assertEqual(result["records"], 8) + self.assertEqual(result["wireMessageCount"], 8) + self.assertEqual(metric_names, { + "database_entry_encode", + "database_entry_decode", + "cultnet_frame_parse", + "raw_snapshot_apply", + "cache_upsert", + "cache_get", + }) + self.assertTrue(all(metric["opsPerSecond"] > 0 for metric in result["metrics"])) + + def test_compare_csharp_median_result_summarizes_samples(self) -> None: + self.assertEqual(DEFAULT_SAMPLE_COUNT, 3) + self.assertEqual(DEFAULT_PARITY_THRESHOLD, 0.90) + samples = [ + { + "runtime": "python", + "records": 3, + "metrics": [ + {"name": "cache_get", "operations": 3, "elapsedMs": 30.0, "opsPerSecond": 100.0}, + {"name": "cache_upsert", "operations": 3, "elapsedMs": 60.0, "opsPerSecond": 50.0}, + ], + }, + { + "runtime": "python", + "records": 3, + "metrics": [ + {"name": "cache_get", "operations": 3, "elapsedMs": 10.0, "opsPerSecond": 300.0}, + {"name": "cache_upsert", "operations": 3, "elapsedMs": 20.0, "opsPerSecond": 150.0}, + ], + }, + { + "runtime": "python", + "records": 3, + "metrics": [ + {"name": "cache_get", "operations": 3, "elapsedMs": 20.0, "opsPerSecond": 200.0}, + {"name": "cache_upsert", "operations": 3, "elapsedMs": 40.0, "opsPerSecond": 75.0}, + ], + }, + ] + + summarized = _median_result(samples) + + self.assertEqual(summarized["sampleCount"], 3) + self.assertEqual(summarized["metrics"][0]["name"], "cache_get") + self.assertEqual(summarized["metrics"][0]["elapsedMs"], 20.0) + self.assertEqual(summarized["metrics"][0]["opsPerSecond"], 200.0) + self.assertEqual(summarized["metrics"][1]["opsPerSecond"], 75.0) + + def test_compare_csharp_marks_threshold_status_per_metric(self) -> None: + python_result = { + "metrics": [ + {"name": "cache_get", "operations": 3, "elapsedMs": 2.0, "opsPerSecond": 50.0}, + {"name": "cache_upsert", "operations": 3, "elapsedMs": 1.0, "opsPerSecond": 95.0}, + ], + } + csharp_result = { + "metrics": [ + {"name": "cache_get", "operations": 3, "elapsedMs": 1.0, "opsPerSecond": 100.0}, + {"name": "cache_upsert", "operations": 3, "elapsedMs": 1.0, "opsPerSecond": 100.0}, + ], + } + + comparison = _compare_common_metrics(python_result, csharp_result, parity_threshold=0.90) + by_name = {item["name"]: item for item in comparison} + + self.assertFalse(by_name["cache_get"]["meetsParityThreshold"]) + self.assertTrue(by_name["cache_upsert"]["meetsParityThreshold"]) + self.assertEqual(_parity_status(comparison, "ok"), "below-threshold") + self.assertEqual(_parity_status([by_name["cache_upsert"]], "ok"), "meets-threshold") + self.assertEqual(_parity_status(comparison, "failed"), "unknown") + + def test_compare_csharp_marks_meets_threshold_when_all_common_metrics_pass(self) -> None: + comparison = _compare_common_metrics( + { + "metrics": [ + {"name": "cache_get", "operations": 3, "elapsedMs": 1.0, "opsPerSecond": 95.0}, + {"name": "cache_upsert", "operations": 3, "elapsedMs": 1.0, "opsPerSecond": 120.0}, + ], + }, + { + "metrics": [ + {"name": "cache_get", "operations": 3, "elapsedMs": 1.0, "opsPerSecond": 100.0}, + {"name": "cache_upsert", "operations": 3, "elapsedMs": 1.0, "opsPerSecond": 100.0}, + ], + }, + parity_threshold=0.90, + ) + + self.assertEqual(_parity_status(comparison, "ok"), "meets-threshold") + + def test_cultnet_security_options_match_cultlib_defaults_and_environment_rules(self) -> None: + client = CultNetClientSecurityOptions.development() + server = CultNetServerSecurityOptions.development() + + self.assertEqual(client.connection_key, "gamecult-dev-connection-key") + self.assertEqual(server.connection_key, "gamecult-dev-connection-key") + self.assertTrue(server.is_development) + self.assertEqual(server.to_client_options().connection_key, client.connection_key) + self.assertEqual(client.encryption_key(), hashlib.sha256(b"gamecult-dev-connection-key").digest()) + self.assertEqual(server.session_signing_key(), hashlib.sha256(b"gamecult-dev-session-signing-secret").digest()) + self.assertEqual( + CultNetServerSecurityOptions.from_environment({}, allow_development_defaults=True), + server, + ) + with self.assertRaisesRegex(ValueError, "not configured"): + CultNetServerSecurityOptions.from_environment({}) + with self.assertRaisesRegex(ValueError, "GAMECULT_SESSION_SIGNING_SECRET"): + CultNetServerSecurityOptions.from_environment({"GAMECULT_CONNECTION_KEY": "key"}) + configured = CultNetServerSecurityOptions.from_environment({ + "GAMECULT_CONNECTION_KEY": "prod-key", + "GAMECULT_SESSION_SIGNING_SECRET": "prod-secret", + }) + self.assertEqual(configured.connection_key, "prod-key") + self.assertFalse(configured.is_development) + + def test_cultnet_secret_helpers_validate_csharp_versioned_session_tokens(self) -> None: + security = CultNetServerSecurityOptions("connection-key", "session-secret") + user_id = uuid4() + expires = datetime(2035, 1, 2, 3, 4, 5, tzinfo=UTC) + + token = CultNetSecret.create_session_token(user_id, expires, security, session_version=7) + payload_b64, signature_b64 = token.split(".") + self.assertEqual( + CultNetSecret.from_base64url(payload_b64).decode("utf-8"), + f"{user_id.hex}|{int(expires.timestamp())}|7", + ) + self.assertEqual( + CultNetSecret.from_base64url(signature_b64), + hmac.new( + hashlib.sha256(b"session-secret").digest(), + CultNetSecret.from_base64url(payload_b64), + hashlib.sha256, + ).digest(), + ) + + validated = CultNetSecret.try_validate_session_token( + token, + security, + now_utc=datetime(2035, 1, 2, 3, 4, 4, tzinfo=UTC), + ) + self.assertIsNotNone(validated) + assert validated is not None + self.assertEqual(validated.user_id, user_id) + self.assertEqual(validated.expires_at_utc, expires) + self.assertEqual(validated.session_version, 7) + + tampered_payload = CultNetSecret.to_base64url(CultNetSecret.from_base64url(payload_b64) + b"tamper") + tampered = f"{tampered_payload}.{signature_b64}" + self.assertIsNone(CultNetSecret.try_validate_session_token(tampered, security)) + self.assertIsNone(CultNetSecret.try_validate_session_token(token, security, now_utc=expires)) + + def test_cultnet_secret_helpers_validate_legacy_two_field_session_tokens(self) -> None: + security = CultNetServerSecurityOptions.development() + user_id = uuid4() + expires_at_seconds = int(datetime(2036, 1, 1, tzinfo=UTC).timestamp()) + payload = f"{user_id.hex}|{expires_at_seconds}".encode("utf-8") + signature = hmac.new(security.session_signing_key(), payload, hashlib.sha256).digest() + token = f"{CultNetSecret.to_base64url(payload)}.{CultNetSecret.to_base64url(signature)}" + + validated = CultNetSecret.try_validate_session_token( + token, + security, + now_utc=datetime(2035, 12, 31, tzinfo=UTC), + ) + + self.assertIsNotNone(validated) + assert validated is not None + self.assertEqual(validated.user_id, user_id) + self.assertEqual(validated.session_version, 0) + + def test_cultnet_secret_encryption_helpers_are_optional_without_crypto_dependency(self) -> None: + nonce = b"123456789012" + security = CultNetServerSecurityOptions.development() + try: + encrypted = CultNetSecret.encrypt_string("hello", nonce, security.to_client_options()) + except ImportError as exc: + self.assertIn("cryptography", str(exc)) + return + self.assertEqual(CultNetSecret.decrypt_string(encrypted, nonce, security), "hello") + + def test_cultnet_schema_message_frame_round_trip(self) -> None: + message = hello( + runtime_id="python-test", + supported_schema_versions=["cultnet.hello.v0"], + transport_profiles=[ + { + "schemaVersion": "cultnet.transport_profile.v0", + "runtimeId": "python-test", + "transports": [ + { + "transportId": "test", + "protocol": "tcp_framed", + "channels": [{"channelId": "schema", "delivery": "reliable", "ordering": "ordered"}], + } + ], + } + ], + ) + parsed = parse_message(decode_frame(encode_frame(message.to_bytes()))) + self.assertEqual(parsed.schema_version, "cultnet.hello.v0") + self.assertEqual(parsed.body["runtimeId"], "python-test") + self.assertEqual(parsed.body["transportProfiles"][0]["transports"][0]["protocol"], "tcp_framed") + + def test_cultnet_tcp_framed_transport_carries_schema_payloads_with_stats(self) -> None: + payload = b"cultnet-payload" + output = io.BytesIO() + sender = TcpFramedTransportConnection( + output, + profile=create_tcp_framed_transport_profile("sender", transport_id="test-tcp"), + ) + sender.send("schema", payload) + self.assertEqual(sender.stats.frames_sent, 1) + self.assertEqual(sender.stats.bytes_sent, len(payload) + 4) + with self.assertRaisesRegex(ValueError, "only supports the schema channel"): + sender.send("unreliable", b"") + + receiver = TcpFramedTransportConnection( + io.BytesIO(output.getvalue()), + profile=create_tcp_framed_transport_profile("receiver", transport_id="test-tcp"), + ) + frame = receiver.receive() + self.assertEqual(frame.channel_id, "schema") + self.assertEqual(frame.payload, payload) + self.assertEqual(receiver.stats.frames_received, 1) + self.assertEqual(receiver.stats.bytes_received, len(payload) + 4) + self.assertEqual(receiver.profile["transports"][0]["protocol"], "tcp_framed") + + def test_cultnet_raw_client_uses_schema_transport_factory(self) -> None: + import msgpack # type: ignore + + sent_messages: list[dict[str, object]] = [] + test_case = self + + class FakeSchemaTransport: + profile = create_tcp_framed_transport_profile("fake-client") + + def __enter__(self) -> "FakeSchemaTransport": + return self + + def __exit__(self, exc_type: object, exc: object, traceback: object) -> None: + self.close() + + def send(self, channel_id: str, payload: bytes) -> None: + test_case.assertEqual(channel_id, "schema") + sent_messages.append(msgpack.unpackb(payload, raw=False)) + + def receive(self) -> object: + response = { + "schemaVersion": "cultnet.schema_catalog_response.v0", + "messageId": "transport-factory", + "schemas": [], + } + return type("Frame", (), {"channel_id": "schema", "payload": msgpack.packb(response, use_bin_type=True)})() + + def close(self) -> None: + pass + + client = CultNetRawClient( + "unused.example.test", + 1, + create_transport=FakeSchemaTransport, + ) + + response = client.fetch_schema_catalog(message_id="transport-factory") + + self.assertEqual(response["schemaVersion"], "cultnet.schema_catalog_response.v0") + self.assertEqual(sent_messages[0]["schemaVersion"], "cultnet.schema_catalog_request.v0") + self.assertEqual(sent_messages[0]["messageId"], "transport-factory") + + def test_cultnet_rudp_packet_codec_uses_deterministic_reliable_ordered_fixture(self) -> None: + encoded = encode_rudp_packet( + CultNetRudpPacket( + packet_type=CultNetRudpPacketType.DATA, + connection_id=0x01020304, + sequence=0x0000002A, + ack=0x00000029, + ack_mask=0x80000001, + channel_id="schema", + reliable=True, + ordered=True, + fragment_id=7, + fragment_index=1, + fragment_count=3, + payload=b"hello", + ) + ) + + self.assertEqual( + encoded.hex(), + "434e523000030b2a010203040000002a0000002980000001000700010003000000050600736368656d6168656c6c6f", + ) + + decoded = decode_rudp_packet(encoded) + self.assertEqual(decoded.packet_type, CultNetRudpPacketType.DATA) + self.assertEqual(decoded.connection_id, 0x01020304) + self.assertEqual(decoded.sequence, 0x0000002A) + self.assertEqual(decoded.ack, 0x00000029) + self.assertEqual(decoded.ack_mask, 0x80000001) + self.assertEqual(decoded.channel_id, "schema") + self.assertTrue(decoded.reliable) + self.assertTrue(decoded.ordered) + self.assertFalse(decoded.sequenced) + self.assertEqual(decoded.fragment_id, 7) + self.assertEqual(decoded.fragment_index, 1) + self.assertEqual(decoded.fragment_count, 3) + self.assertEqual(decoded.payload, b"hello") + + def test_cultnet_rudp_transport_profile_advertises_state_and_realtime_channels(self) -> None: + profile = create_rudp_transport_profile( + "python-rudp", + transport_id="public-rudp", + host="127.0.0.1", + port=7777, + max_payload_bytes=1200, + max_fragment_bytes=1000, + ) + + self.assertEqual(profile["transports"][0]["protocol"], "rudp") + self.assertEqual(profile["transports"][0]["reconnectPolicy"]["schemaVersion"], "cultnet.reconnect_policy.v0") + self.assertEqual(profile["transports"][0]["reconnectPolicy"]["baseDelayMs"], 1_000) + self.assertEqual( + [ + (channel["channelId"], channel["delivery"], channel["ordering"]) + for channel in profile["transports"][0]["channels"] + ], + [ + ("schema", "reliable", "ordered"), + ("latest", "unreliable", "sequenced"), + ("realtime", "unreliable", "unordered"), + ], + ) + + def test_cultnet_reconnect_policy_exposes_portable_delay_contract(self) -> None: + policy = create_reconnect_policy(policy_id="rudp-default", max_attempts=8) + + self.assertEqual(policy.schema_version, "cultnet.reconnect_policy.v0") + self.assertEqual(policy.policy_id, "rudp-default") + self.assertEqual(policy.max_attempts, 8) + self.assertEqual(policy.to_wire()["policyId"], "rudp-default") + self.assertEqual(policy.to_wire()["maxAttempts"], 8) + self.assertEqual(compute_reconnect_delay_ms(policy, 1), 1_000) + self.assertEqual(compute_reconnect_delay_ms(policy, 3, 17), 4_017) + self.assertEqual(compute_reconnect_delay_ms(policy, 9, 999), 30_250) + self.assertEqual(compute_reconnect_delay_ms(policy, 0, -5), 1_000) + + def test_cultnet_reconnect_controller_schedules_attempts_and_reset(self) -> None: + controller = CultNetReconnectController(create_reconnect_policy(max_attempts=2)) + + first = controller.record_failure(10_000) + self.assertEqual(first.attempt, 1) + self.assertTrue(first.should_retry) + self.assertEqual(first.delay_ms, 1_000) + self.assertEqual(first.next_attempt_at_ms, 11_000) + self.assertFalse(first.exhausted) + self.assertFalse(controller.can_attempt(10_999)) + self.assertTrue(controller.can_attempt(11_000)) + + second = controller.record_failure(11_000, 17) + self.assertEqual(second.attempt, 2) + self.assertEqual(second.delay_ms, 2_017) + self.assertEqual(second.next_attempt_at_ms, 13_017) + self.assertTrue(second.should_retry) + + exhausted = controller.record_failure(13_017) + self.assertEqual(exhausted.attempt, 2) + self.assertFalse(exhausted.should_retry) + self.assertEqual(exhausted.delay_ms, 0) + self.assertIsNone(exhausted.next_attempt_at_ms) + self.assertTrue(exhausted.exhausted) + self.assertFalse(controller.can_attempt(99_000)) + + controller.reset() + self.assertEqual(controller.attempt, 0) + self.assertIsNone(controller.next_attempt_at_ms) + self.assertFalse(controller.exhausted) + self.assertTrue(controller.can_attempt(99_000)) + + def test_cultnet_rudp_reconnect_loop_consumes_shared_controller(self) -> None: + class FakeRudpReconnectTransport: + def __init__(self) -> None: + self.connect_calls = 0 + self.close_calls = 0 + self.last_payload: bytes | None = None + + def connect(self, payload: bytes = b"") -> None: + self.connect_calls += 1 + self.last_payload = bytes(payload) + + def close(self) -> None: + self.close_calls += 1 + + now_ms = 10_000 + captured_callback: list[Callable[[], None]] = [] + transports: list[FakeRudpReconnectTransport] = [] + + def create_transport() -> FakeRudpReconnectTransport: + transport = FakeRudpReconnectTransport() + transports.append(transport) + return transport + + def scheduler(delay_ms: int, callback: Callable[[], None]) -> Callable[[], None]: + self.assertEqual(delay_ms, 1_017) + captured_callback[:] = [callback] + + def cancel() -> None: + captured_callback.clear() + + return cancel + + loop = CultNetRudpReconnectLoop( + create_transport, + reconnect_policy=create_reconnect_policy(max_attempts=2), + connect_payload=b"join", + now_ms=lambda: now_ms, + jitter_ms=lambda: 17, + scheduler=scheduler, + ) + + first = loop.start() + self.assertEqual(first.connect_calls, 1) + self.assertEqual(first.last_payload, b"join") + self.assertIs(loop.transport, first) + + decision = loop.handle_closed() + self.assertIsNotNone(decision) + self.assertEqual(decision.attempt, 1) + self.assertTrue(decision.should_retry) + self.assertEqual(decision.delay_ms, 1_017) + self.assertEqual(decision.next_attempt_at_ms, 11_017) + self.assertEqual(loop.reconnect_controller.attempt, 1) + self.assertEqual(loop.reconnect_controller.next_attempt_at_ms, 11_017) + self.assertEqual(len(captured_callback), 1) + + now_ms = 11_017 + captured_callback[0]() + self.assertEqual(len(transports), 2) + self.assertEqual(transports[1].connect_calls, 1) + self.assertIs(loop.transport, transports[1]) + + loop.mark_connected() + self.assertEqual(loop.reconnect_controller.attempt, 0) + + loop.stop() + self.assertEqual(transports[1].close_calls, 1) + self.assertIsNone(loop.transport) + + def test_cultnet_rudp_session_handshake_acks_reliable_connect_and_accept_packets(self) -> None: + client = CultNetRudpSession( + CultNetRudpSessionOptions(connection_id=0x0A0B0C0D, initial_sequence=1, resend_delay_ms=50) + ) + server = CultNetRudpSession( + CultNetRudpSessionOptions(connection_id=0x0A0B0C0D, initial_sequence=100, resend_delay_ms=50) + ) + + connect = client.create_connect(0, b"join") + self.assertEqual(connect.packet_type, CultNetRudpPacketType.CONNECT) + self.assertEqual(connect.sequence, 1) + self.assertEqual(client.pending_reliable_sequences, (1,)) + + accept = server.accept_connect(connect, 10, b"ok") + self.assertEqual(accept.packet_type, CultNetRudpPacketType.ACCEPT) + self.assertEqual(accept.ack, 1) + self.assertTrue(server.connected) + self.assertEqual(server.pending_reliable_sequences, (100,)) + + client.receive(accept, 20) + self.assertTrue(client.connected) + self.assertEqual(client.pending_reliable_sequences, ()) + + ack = client.create_ack() + self.assertEqual(ack.ack, 100) + server.receive(ack, 30) + self.assertEqual(server.pending_reliable_sequences, ()) + + def test_cultnet_rudp_session_computes_ack_masks_and_clears_pending_reliable_packets(self) -> None: + sender = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=7, initial_sequence=10, resend_delay_ms=100)) + receiver = CultNetRudpSession( + CultNetRudpSessionOptions(connection_id=7, initial_sequence=200, resend_delay_ms=100) + ) + sender.receive( + CultNetRudpPacket(CultNetRudpPacketType.ACCEPT, 7, 1, 0, 0, "control") + ) + receiver.receive( + CultNetRudpPacket(CultNetRudpPacketType.ACCEPT, 7, 2, 0, 0, "control") + ) + + first = sender.send( + "schema", + b"first", + CultNetRudpSendOptions(reliable=True, ordered=True, now_ms=0), + ) + second = sender.send( + "schema", + b"second", + CultNetRudpSendOptions(reliable=True, ordered=True, now_ms=0), + ) + third = sender.send( + "schema", + b"third", + CultNetRudpSendOptions(reliable=True, ordered=True, now_ms=0), + ) + self.assertEqual(sender.pending_reliable_sequences, (10, 11, 12)) + + receiver.receive(first) + receiver.receive(third) + ack_with_gap = receiver.create_ack() + self.assertEqual(ack_with_gap.ack, 12) + self.assertEqual(ack_with_gap.ack_mask, 0b10 | (1 << 9)) + sender.receive(ack_with_gap) + self.assertEqual(sender.pending_reliable_sequences, (11,)) + + receiver.receive(second) + full_ack = receiver.create_ack() + self.assertEqual(full_ack.ack, 12) + self.assertEqual(full_ack.ack_mask, 0b11 | (1 << 9)) + sender.receive(full_ack) + self.assertEqual(sender.pending_reliable_sequences, ()) + + def test_cultnet_rudp_session_schedules_reliable_resends_until_acked(self) -> None: + session = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=99, initial_sequence=1, resend_delay_ms=100)) + session.receive( + CultNetRudpPacket(CultNetRudpPacketType.ACCEPT, 99, 50, 0, 0, "control") + ) + sent = session.send( + "schema", + b"payload", + CultNetRudpSendOptions(reliable=True, ordered=True, now_ms=10), + ) + + self.assertEqual(session.due_resends(90), ()) + self.assertEqual(tuple(packet.sequence for packet in session.due_resends(110)), (sent.sequence,)) + self.assertEqual(session.due_resends(150), ()) + + session.receive( + CultNetRudpPacket(CultNetRudpPacketType.ACK, 99, 51, sent.sequence, 0, "control") + ) + self.assertEqual(session.due_resends(250), ()) + + def test_cultnet_rudp_session_pings_and_detects_receive_timeout(self) -> None: + client = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=101, initial_sequence=1)) + server = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=101, initial_sequence=100)) + connect = client.create_connect(0, b"join") + accept = server.accept_connect(connect, 10) + client.receive(accept, 20) + + ping = client.create_ping(b"pulse") + ping_result = server.receive(ping, 30) + self.assertIsNotNone(ping_result.reply) + self.assertEqual(ping_result.reply.packet_type, CultNetRudpPacketType.PONG) + self.assertEqual(ping_result.reply.payload, b"pulse") + + pong_result = client.receive(ping_result.reply, 40) + self.assertTrue(pong_result.pong) + self.assertEqual(pong_result.pong_payload, b"pulse") + self.assertFalse(client.check_timeout(90, 50)) + self.assertTrue(client.check_timeout(91, 50)) + self.assertFalse(client.connected) + + def test_cultnet_rudp_session_bounds_pending_reliable_packets_before_enqueue(self) -> None: + session = CultNetRudpSession( + CultNetRudpSessionOptions(connection_id=102, initial_sequence=1, max_pending_reliable_packets=2) + ) + session.receive(CultNetRudpPacket(CultNetRudpPacketType.ACCEPT, 102, 50, 0, 0, "control")) + session.send("schema", b"first", CultNetRudpSendOptions(reliable=True, ordered=True)) + session.send("schema", b"second", CultNetRudpSendOptions(reliable=True, ordered=True)) + with self.assertRaisesRegex(ValueError, "reliable send queue is full"): + session.send("schema", b"third", CultNetRudpSendOptions(reliable=True, ordered=True)) + self.assertEqual(session.pending_reliable_sequences, (1, 2)) + + fragmented = CultNetRudpSession( + CultNetRudpSessionOptions(connection_id=103, initial_sequence=1, max_pending_reliable_packets=3) + ) + fragmented.receive(CultNetRudpPacket(CultNetRudpPacketType.ACCEPT, 103, 50, 0, 0, "control")) + with self.assertRaisesRegex(ValueError, "reliable send queue is full"): + fragmented.send_many( + "schema", + b"fragment-me", + CultNetRudpSendOptions(reliable=True, ordered=True), + max_fragment_bytes=3, + ) + self.assertEqual(fragmented.pending_reliable_sequences, ()) + + def test_cultnet_rudp_session_suppresses_duplicates_and_delivers_reliable_ordered_payloads(self) -> None: + sender = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=123, initial_sequence=1)) + receiver = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=123, initial_sequence=100)) + sender.receive( + CultNetRudpPacket(CultNetRudpPacketType.ACCEPT, 123, 90, 0, 0, "control") + ) + receiver.receive( + CultNetRudpPacket(CultNetRudpPacketType.ACCEPT, 123, 91, 0, 0, "control") + ) + + first = sender.send("schema", b"first", CultNetRudpSendOptions(reliable=True, ordered=True)) + second = sender.send("schema", b"second", CultNetRudpSendOptions(reliable=True, ordered=True)) + third = sender.send("schema", b"third", CultNetRudpSendOptions(reliable=True, ordered=True)) + + self.assertEqual([frame.payload.decode("utf-8") for frame in receiver.receive(first).delivered], ["first"]) + self.assertEqual(receiver.receive(third).delivered, ()) + self.assertEqual(receiver.receive(first).delivered, ()) + self.assertEqual( + [frame.payload.decode("utf-8") for frame in receiver.receive(second).delivered], + ["second", "third"], + ) + + def test_cultnet_rudp_lossy_packets_cannot_create_reliable_ordered_gaps(self) -> None: + sender = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=198, initial_sequence=1)) + receiver = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=198, initial_sequence=100)) + sender.receive(CultNetRudpPacket(CultNetRudpPacketType.ACCEPT, 198, 90, 0, 0, "control")) + receiver.receive(CultNetRudpPacket(CultNetRudpPacketType.ACCEPT, 198, 91, 0, 0, "control")) + + realtime = sender.send("realtime", b"discarded realtime", CultNetRudpSendOptions()) + latest = sender.send("latest", b"discarded latest state", CultNetRudpSendOptions(sequenced=True)) + schema = sender.send("schema", b"committed response", CultNetRudpSendOptions(reliable=True, ordered=True)) + + self.assertEqual(realtime.sequence, 0) + self.assertEqual(latest.sequence, 1) + self.assertEqual( + [frame.payload.decode("utf-8") for frame in receiver.receive(schema).delivered], + ["committed response"], + ) + + def test_cultnet_rudp_unreliable_sequenced_delivery_is_scoped_to_its_channel(self) -> None: + sender = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=197, initial_sequence=50)) + receiver = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=197, initial_sequence=100)) + sender.receive(CultNetRudpPacket(CultNetRudpPacketType.ACCEPT, 197, 90, 0, 0, "control")) + receiver.receive(CultNetRudpPacket(CultNetRudpPacketType.ACCEPT, 197, 91, 0, 0, "control")) + + older = sender.send("latest", b"older", CultNetRudpSendOptions(sequenced=True)) + newer = sender.send("latest", b"newer", CultNetRudpSendOptions(sequenced=True)) + + self.assertEqual(older.sequence, 1) + self.assertEqual(newer.sequence, 2) + self.assertEqual( + [frame.payload.decode("utf-8") for frame in receiver.receive(newer).delivered], + ["newer"], + ) + self.assertEqual(receiver.receive(older).delivered, ()) + + def test_cultnet_rudp_rejects_unreliable_ordered_delivery(self) -> None: + session = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=196, initial_sequence=1)) + session.receive(CultNetRudpPacket(CultNetRudpPacketType.ACCEPT, 196, 50, 0, 0, "control")) + + with self.assertRaises(ValueError) as raised: + session.send( + "schema", + b"cannot order what will not retransmit", + CultNetRudpSendOptions(ordered=True), + ) + self.assertIn("ordered delivery requires reliability", str(raised.exception)) + + def test_cultnet_rudp_session_skips_control_packets_while_ordering_schema_payloads(self) -> None: + sender = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=124, initial_sequence=1)) + receiver = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=124, initial_sequence=100)) + sender.receive( + CultNetRudpPacket(CultNetRudpPacketType.ACCEPT, 124, 90, 0, 0, "control") + ) + receiver.receive( + CultNetRudpPacket(CultNetRudpPacketType.ACCEPT, 124, 91, 0, 0, "control") + ) + + first = sender.send("schema", b"first", CultNetRudpSendOptions(reliable=True, ordered=True)) + control = sender.create_ack() + second = sender.send("schema", b"second", CultNetRudpSendOptions(reliable=True, ordered=True)) + + self.assertEqual([frame.payload.decode("utf-8") for frame in receiver.receive(first).delivered], ["first"]) + self.assertEqual(receiver.receive(control).delivered, ()) + self.assertEqual([frame.payload.decode("utf-8") for frame in receiver.receive(second).delivered], ["second"]) + + def test_cultnet_rudp_session_fragments_and_reassembles_reliable_ordered_payloads(self) -> None: + sender = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=456, initial_sequence=1)) + receiver = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=456, initial_sequence=100)) + sender.receive( + CultNetRudpPacket(CultNetRudpPacketType.ACCEPT, 456, 90, 0, 0, "control") + ) + receiver.receive( + CultNetRudpPacket(CultNetRudpPacketType.ACCEPT, 456, 91, 0, 0, "control") + ) + + packets = sender.send_many( + "schema", + b"fragment-me-please", + CultNetRudpSendOptions(reliable=True, ordered=True, now_ms=10), + max_fragment_bytes=5, + ) + self.assertEqual(len(packets), 4) + self.assertEqual(tuple(packet.fragment_count for packet in packets), (4, 4, 4, 4)) + self.assertEqual(tuple(packet.fragment_index for packet in packets), (0, 1, 2, 3)) + self.assertTrue(all(packet.fragment_id == packets[0].fragment_id for packet in packets)) + + self.assertEqual(receiver.receive(packets[0]).delivered, ()) + self.assertEqual(receiver.receive(packets[1]).delivered, ()) + self.assertEqual(receiver.receive(packets[2]).delivered, ()) + delivered = receiver.receive(packets[3]).delivered + self.assertEqual(len(delivered), 1) + self.assertEqual(delivered[0].payload, b"fragment-me-please") + self.assertEqual(delivered[0].sequence, packets[0].sequence) + + def test_cultnet_rudp_session_advances_large_fragment_sets_through_bounded_window(self) -> None: + connection_id = 457 + sender = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=connection_id, initial_sequence=1)) + receiver = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=connection_id, initial_sequence=100)) + sender.receive(CultNetRudpPacket(CultNetRudpPacketType.ACCEPT, connection_id, 0, 0, 0, "control")) + receiver.receive(CultNetRudpPacket(CultNetRudpPacketType.ACCEPT, connection_id, 0, 0, 0, "control")) + + fragment_count = CultNetRudpSession.RELIABLE_SEND_WINDOW_PACKETS + 17 + payload = bytes(index % 251 for index in range(fragment_count * 8)) + wire = list(sender.send_many( + "schema", + payload, + CultNetRudpSendOptions(reliable=True, ordered=True, now_ms=1), + max_fragment_bytes=8, + )) + self.assertEqual(len(wire), CultNetRudpSession.RELIABLE_SEND_WINDOW_PACKETS) + oldest_sequence = wire[0].sequence + self.assertEqual(len(sender.pending_reliable_sequences), len(wire)) + self.assertEqual(sender.queued_reliable_packet_count, 17) + + delivered = [] + while wire: + admitted = len(wire) + for _ in range(admitted): + packet = wire.pop(0) + delivered.extend(receiver.receive(packet, 2).delivered) + acknowledged = sender.receive(receiver.create_ack(), 3) + wire.extend(acknowledged.ready_to_send) + + self.assertEqual(sender.outstanding_reliable_packet_count, 0) + self.assertEqual(len(delivered), 1) + self.assertEqual(delivered[0].payload, payload) + old_ack = receiver.create_ack_for_received(oldest_sequence) + self.assertEqual(old_ack.ack, oldest_sequence) + self.assertEqual(old_ack.ack_mask, 0) + + def test_cultnet_rudp_socket_transport_handshakes_and_carries_reliable_ordered_schema_frames(self) -> None: + server_socket = bind_udp_socket() + client_socket = bind_udp_socket() + connection_id = 0x10203040 + server = CultNetRudpSocketTransportConnection( + CultNetRudpSocketTransportOptions( + runtime_id="python-rudp-server", + socket=server_socket, + mode=CultNetRudpSocketMode.SERVER, + connection_id=connection_id, + initial_sequence=100, + resend_delay_ms=25, + ) + ) + client = CultNetRudpSocketTransportConnection( + CultNetRudpSocketTransportOptions( + runtime_id="python-rudp-client", + socket=client_socket, + mode=CultNetRudpSocketMode.CLIENT, + remote_addr=server_socket.getsockname(), + connection_id=connection_id, + initial_sequence=1, + resend_delay_ms=25, + ) + ) + + try: + client.connect(b"join") + pump_rudp_handshake(client, server) + self.assertTrue(client.connected) + self.assertTrue(server.connected) + + client.send("schema", b"client-state") + server_frame = receive_rudp_frame(server) + self.assertEqual(server_frame.channel_id, "schema") + self.assertEqual(server_frame.payload, b"client-state") + + server.send("schema", b"server-state") + client_frame = receive_rudp_frame(client) + self.assertEqual(client_frame.channel_id, "schema") + self.assertEqual(client_frame.payload, b"server-state") + self.assertEqual(server.profile["transports"][0]["protocol"], "rudp") + self.assertEqual(client.stats.frames_sent, 1) + self.assertEqual(server.stats.frames_received, 1) + finally: + client.close() + server.close() + + def test_cultnet_rudp_socket_transport_carries_fragmented_reliable_ordered_schema_frames(self) -> None: + server_socket = bind_udp_socket() + client_socket = bind_udp_socket() + connection_id = 0x10203041 + server = CultNetRudpSocketTransportConnection( + CultNetRudpSocketTransportOptions( + runtime_id="python-rudp-fragment-server", + socket=server_socket, + mode=CultNetRudpSocketMode.SERVER, + connection_id=connection_id, + initial_sequence=100, + resend_delay_ms=25, + max_fragment_bytes=8, + ) + ) + client = CultNetRudpSocketTransportConnection( + CultNetRudpSocketTransportOptions( + runtime_id="python-rudp-fragment-client", + socket=client_socket, + mode=CultNetRudpSocketMode.CLIENT, + remote_addr=server_socket.getsockname(), + connection_id=connection_id, + initial_sequence=1, + resend_delay_ms=25, + max_fragment_bytes=8, + ) + ) + + try: + payload = b"this-schema-frame-is-larger-than-one-rudp-fragment" + client.connect(b"join") + pump_rudp_handshake(client, server) + client.send("schema", payload) + server_frame = receive_rudp_frame(server) + self.assertEqual(server_frame.channel_id, "schema") + self.assertEqual(server_frame.payload, payload) + self.assertEqual(client.stats.frames_sent, 1) + self.assertEqual(server.stats.frames_received, 1) + finally: + client.close() + server.close() + + def test_cultnet_database_subscription_helpers_match_schema_v0_shape(self) -> None: + subscribe = database_subscribe( + message_id="sub-message", + subscription_id="sub-1", + schema_ids=["schema-a"], + record_keys=["record-a"], + include_snapshot=False, + ).to_wire() + self.assertEqual(subscribe["schemaVersion"], "cultnet.database_subscribe.v0") + self.assertEqual(subscribe["messageId"], "sub-message") + self.assertEqual(subscribe["subscriptionId"], "sub-1") + self.assertEqual(subscribe["schemaIds"], ["schema-a"]) + self.assertEqual(subscribe["recordKeys"], ["record-a"]) + self.assertFalse(subscribe["includeSnapshot"]) + + unsubscribe = database_unsubscribe(message_id="unsub-message", subscription_id="sub-1").to_wire() + self.assertEqual(unsubscribe["schemaVersion"], "cultnet.database_unsubscribe.v0") + self.assertEqual(unsubscribe["messageId"], "unsub-message") + self.assertEqual(unsubscribe["subscriptionId"], "sub-1") + + def test_cultnet_database_change_parses_put_and_delete_shapes(self) -> None: + put_change = CultNetDatabaseChange.from_wire({ + "schemaVersion": "cultnet.database_change_raw.v0", + "messageId": "change-put", + "subscriptionId": "sub-1", + "changeKind": "added", + "document": { + "schemaId": "schema-note", + "recordKey": "note:1", + "payload": b"payload", + }, + }) + self.assertEqual(put_change.schema_id, "schema-note") + self.assertEqual(put_change.record_key, "note:1") + self.assertIsInstance(put_change.raw_document, CultNetRawDocumentRecord) + self.assertEqual(put_change.raw_document.payload, b"payload") + self.assertEqual(put_change.to_wire()["document"]["recordKey"], "note:1") + + delete_change = CultNetDatabaseChange.from_wire({ + "schemaVersion": "cultnet.database_change_raw.v0", + "messageId": "change-delete", + "subscriptionId": "sub-1", + "changeKind": "removed", + "schemaId": "schema-note", + "recordKey": "note:1", + }) + self.assertIsNone(delete_change.document) + self.assertIsNone(delete_change.raw_document) + self.assertEqual(delete_change.schema_id, "schema-note") + self.assertEqual(delete_change.to_wire()["recordKey"], "note:1") + + def test_cultnet_raw_put_helper_carries_message_id(self) -> None: + put = document_put_raw( + message_id="put-1", + key="record-a", + schema_id="schema-a", + stored_at="2026-06-13T00:00:00Z", + payload=b"payload", + source_runtime_id="python-test", + shard_id="interop", + shard_epoch=1, + ).to_wire() + self.assertEqual(put["schemaVersion"], "cultnet.document_put_raw.v0") + self.assertEqual(put["messageId"], "put-1") + self.assertEqual(put["document"]["recordKey"], "record-a") + self.assertEqual(put["document"]["sourceRuntimeId"], "python-test") + self.assertEqual(put["shardId"], "interop") + self.assertEqual(put["shardEpoch"], 1) + + def test_cultnet_catalog_and_snapshot_helpers_accept_filters(self) -> None: + catalog = schema_catalog_request( + message_id="catalog-1", + include_schema_json=True, + schema_ids=["schema-a"], + kinds=["wire_message"], + ).to_wire() + self.assertEqual(catalog["schemaVersion"], "cultnet.schema_catalog_request.v0") + self.assertEqual(catalog["messageId"], "catalog-1") + self.assertTrue(catalog["includeSchemaJson"]) + self.assertEqual(catalog["schemaIds"], ["schema-a"]) + self.assertEqual(catalog["kinds"], ["wire_message"]) + + snapshot = snapshot_request( + message_id="snapshot-1", + schema_ids=["schema-a"], + record_keys=["record-a"], + shard_id="interop", + shard_epoch=1, + ).to_wire() + self.assertEqual(snapshot["schemaVersion"], "cultnet.snapshot_request.v0") + self.assertEqual(snapshot["messageId"], "snapshot-1") + self.assertEqual(snapshot["schemaIds"], ["schema-a"]) + self.assertEqual(snapshot["recordKeys"], ["record-a"]) + self.assertEqual(snapshot["shardId"], "interop") + self.assertEqual(snapshot["shardEpoch"], 1) + + def test_cultnet_raw_snapshot_response_filters_and_roundtrips_records(self) -> None: + response = CultNetRawSnapshotResponse.from_wire({ + "schemaVersion": "cultnet.snapshot_response_raw.v0", + "messageId": "snapshot-1", + "shardId": "notes", + "shardEpoch": 3, + "shardLogSequence": 8, + "documents": [ + CultNetRawDocumentRecord( + schema_id="schema-note", + record_key="note:1", + stored_at="2026-06-14T00:00:00Z", + payload=b"note", + source_runtime_id="python-test", + tags=("snapshot",), + ).to_wire(), + { + "schemaId": "schema-fact", + "recordKey": "fact:1", + "payloadEncoding": "messagepack", + "payload": b"fact", + }, + ], + }) + self.assertEqual(response.shard_id, "notes") + self.assertEqual(response.shard_epoch, 3) + self.assertEqual(response.shard_log_sequence, 8) + self.assertEqual(response.documents[0].source_runtime_id, "python-test") + self.assertEqual(response.filter(schema_ids=["schema-note"])[0].record_key, "note:1") + self.assertEqual(response.filter(record_keys=["fact:1"])[0].schema_id, "schema-fact") + self.assertEqual(response.to_wire()["documents"][0]["tags"], ["snapshot"]) + + def test_cultnet_raw_client_fetches_schema_snapshot_and_shard_reads(self) -> None: + import msgpack # type: ignore + from cultnet_py import read_frame, write_frame + + received_versions: list[str] = [] + ready = threading.Event() + server_error: list[BaseException] = [] + port_holder: list[int] = [] + + def serve_requests() -> None: + try: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as server: + server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + server.bind(("127.0.0.1", 0)) + port_holder.append(server.getsockname()[1]) + server.listen(4) + ready.set() + for _ in range(4): + connection, _ = server.accept() + with connection: + stream = connection.makefile("rwb") + request = msgpack.unpackb(read_frame(stream), raw=False) + received_versions.append(request["schemaVersion"]) + if request["schemaVersion"] == "cultnet.schema_catalog_request.v0": + self.assertEqual(request["kinds"], ["wire_message"]) + response = {"schemaVersion": "cultnet.schema_catalog_response.v0", "messageId": request["messageId"], "schemas": []} + elif request["schemaVersion"] == "cultnet.snapshot_request.v0": + self.assertEqual(request["shardId"], "interop") + response = {"schemaVersion": "cultnet.snapshot_response_raw.v0", "messageId": request["messageId"], "documents": []} + elif request["schemaVersion"] == "cultnet.shard_catalog_request.v0": + response = { + "schemaVersion": "cultnet.shard_catalog_response.v0", + "messageId": request["messageId"], + "shards": [{ + "shardId": "interop", + "ownerRuntimeId": "socket-test", + "epoch": 1, + "schemaIds": request["schemaIds"], + "keyPrefix": "record-", + "primaryEndpoints": ["cultnet://127.0.0.1:1"], + }], + } + elif request["schemaVersion"] == "cultnet.shard_log_request.v0": + self.assertEqual(request["afterSequence"], 7) + response = { + "schemaVersion": "cultnet.shard_log_response.v0", + "messageId": request["messageId"], + "shardId": request["shardId"], + "shardEpoch": request["shardEpoch"], + "entries": [], + "resyncRequired": False, + } + else: + raise AssertionError(f"unexpected request {request['schemaVersion']}") + write_frame(stream, msgpack.packb(response, use_bin_type=True)) + stream.flush() + except BaseException as error: + server_error.append(error) + ready.set() + + thread = threading.Thread(target=serve_requests, daemon=True) + thread.start() + self.assertTrue(ready.wait(2.0)) + self.assertFalse(server_error) + + client = CultNetRawClient("127.0.0.1", port_holder[0], timeout_seconds=2.0) + self.assertEqual(client.fetch_schema_catalog(kinds=["wire_message"])["schemaVersion"], "cultnet.schema_catalog_response.v0") + typed_snapshot = client.fetch_snapshot_response(shard_id="interop", shard_epoch=1) + self.assertEqual(typed_snapshot.shard_id, None) + self.assertEqual(typed_snapshot.documents, ()) + shard_catalog = CultNetShardCatalog() + synced_shards = client.sync_shard_catalog(shard_catalog, schema_ids=["schema-a"]) + self.assertEqual(synced_shards[0].shard_id, "interop") + self.assertEqual(shard_catalog.get("interop"), synced_shards[0]) + shard_log = client.fetch_shard_log_response(shard_id="interop", shard_epoch=1, after_sequence=7) + self.assertEqual(shard_log.shard_id, "interop") + self.assertFalse(shard_log.resync_required) + self.assertEqual(shard_log.last_sequence, 0) + + thread.join(2.0) + self.assertFalse(server_error) + self.assertEqual(received_versions, [ + "cultnet.schema_catalog_request.v0", + "cultnet.snapshot_request.v0", + "cultnet.shard_catalog_request.v0", + "cultnet.shard_log_request.v0", + ]) + + def test_cultnet_raw_client_raises_typed_peer_error_response(self) -> None: + import msgpack # type: ignore + from cultnet_py import read_frame, write_frame + + ready = threading.Event() + port_holder: list[int] = [] + + def serve_error() -> None: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as server: + server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + server.bind(("127.0.0.1", 0)) + port_holder.append(server.getsockname()[1]) + server.listen(1) + ready.set() + connection, _ = server.accept() + with connection: + stream = connection.makefile("rwb") + read_frame(stream) + write_frame(stream, msgpack.packb({ + "schemaVersion": "cultnet.error.v0", + "messageId": "error-1", + "error": "Snapshot document limit exceeded", + }, use_bin_type=True)) + stream.flush() + + thread = threading.Thread(target=serve_error, daemon=True) + thread.start() + self.assertTrue(ready.wait(2.0)) + + client = CultNetRawClient("127.0.0.1", port_holder[0], timeout_seconds=2.0) + with self.assertRaisesRegex(CultNetPeerError, "Snapshot document limit exceeded") as raised: + client.fetch_snapshot() + + self.assertEqual(raised.exception.peer_error, "Snapshot document limit exceeded") + self.assertEqual(raised.exception.response["schemaVersion"], "cultnet.error.v0") + thread.join(2.0) + + def test_cultnet_database_subscription_reads_snapshot_and_change(self) -> None: + import msgpack # type: ignore + from cultnet_py import read_frame, write_frame + + document = define_database_entry_type( + "sub.item", + [ + ("name", 0), + ("category", 1), + ("value", 2, 0), + ], + cls=Item, + ) + schema_id = document.catalog_entry().schema_id + ready = threading.Event() + server_error: list[BaseException] = [] + port_holder: list[int] = [] + received_versions: list[str] = [] + + def serve_subscription() -> None: + try: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as server: + server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + server.bind(("127.0.0.1", 0)) + port_holder.append(server.getsockname()[1]) + server.listen(1) + ready.set() + connection, _ = server.accept() + with connection: + stream = connection.makefile("rwb") + subscribe = msgpack.unpackb(read_frame(stream), raw=False) + received_versions.append(subscribe["schemaVersion"]) + self.assertEqual(subscribe["subscriptionId"], "sub-1") + snapshot = { + "schemaVersion": "cultnet.snapshot_response_raw.v0", + "messageId": subscribe["messageId"], + "documents": [], + } + write_frame(stream, msgpack.packb(snapshot, use_bin_type=True)) + stream.flush() + + put = msgpack.unpackb(read_frame(stream), raw=False) + received_versions.append(put["schemaVersion"]) + change = { + "schemaVersion": "cultnet.database_change_raw.v0", + "messageId": "change-1", + "subscriptionId": "sub-1", + "changeKind": "added", + "document": put["document"], + } + write_frame(stream, msgpack.packb(change, use_bin_type=True)) + stream.flush() + + unsubscribe = msgpack.unpackb(read_frame(stream), raw=False) + received_versions.append(unsubscribe["schemaVersion"]) + except BaseException as error: + server_error.append(error) + ready.set() + + thread = threading.Thread(target=serve_subscription, daemon=True) + thread.start() + self.assertTrue(ready.wait(2.0)) + self.assertFalse(server_error) + + client = CultNetRawClient("127.0.0.1", port_holder[0], timeout_seconds=2.0) + put = document_put_raw( + message_id="put-sub", + key="item:sub", + schema_id=schema_id, + stored_at="2026-06-14T00:00:00Z", + payload=document.encode_payload(Item("orb", "gear", 8)), + ) + with client.subscribe_database(subscription_id="sub-1", schema_ids=[schema_id]) as subscription: + snapshot = subscription.read_next_snapshot_response() + subscription.send(put) + change = subscription.read_next_change() + + thread.join(2.0) + self.assertFalse(server_error) + self.assertIsInstance(snapshot, CultNetRawSnapshotResponse) + self.assertEqual(snapshot.message_id, "cultnet-python-subscribe") + self.assertEqual(snapshot.to_wire()["schemaVersion"], "cultnet.snapshot_response_raw.v0") + self.assertEqual(change.change_kind, "added") + self.assertEqual(change.record_key, "item:sub") + self.assertIsInstance(change.raw_document, CultNetRawDocumentRecord) + self.assertEqual(change.raw_document.record_key, "item:sub") + self.assertEqual(received_versions, [ + "cultnet.database_subscribe.v0", + "cultnet.document_put_raw.v0", + "cultnet.database_unsubscribe.v0", + ]) + + def test_cultnet_database_subscription_raises_typed_peer_error(self) -> None: + import msgpack # type: ignore + from cultnet_py import read_frame, write_frame + + ready = threading.Event() + port_holder: list[int] = [] + + def serve_subscription_error() -> None: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as server: + server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + server.bind(("127.0.0.1", 0)) + port_holder.append(server.getsockname()[1]) + server.listen(1) + ready.set() + connection, _ = server.accept() + with connection: + stream = connection.makefile("rwb") + read_frame(stream) + write_frame(stream, msgpack.packb({ + "schemaVersion": "cultnet.error.v0", + "messageId": "subscription-error", + "error": "subscription rejected", + }, use_bin_type=True)) + stream.flush() + + thread = threading.Thread(target=serve_subscription_error, daemon=True) + thread.start() + self.assertTrue(ready.wait(2.0)) + + client = CultNetRawClient("127.0.0.1", port_holder[0], timeout_seconds=2.0) + with client.subscribe_database(subscription_id="sub-error") as subscription: + with self.assertRaisesRegex(CultNetPeerError, "subscription rejected") as raised: + subscription.read_next() + self.assertEqual(raised.exception.response["messageId"], "subscription-error") + thread.join(2.0) + + def test_cultnet_replication_helpers_apply_raw_snapshot_and_shard_log(self) -> None: + document = define_database_entry_type( + "replica.item", + [ + ("name", 0), + ("category", 1), + ("value", 2, 0), + ], + cls=Item, + ) + cache = CultCache() + cache.register_document_type(document) + schema_id = document.catalog_entry().schema_id + + snapshot = { + "schemaVersion": "cultnet.snapshot_response_raw.v0", + "messageId": "snapshot-1", + "documents": [ + { + "schemaId": schema_id, + "recordKey": "item:1", + "storedAt": "2026-06-13T00:00:00Z", + "payloadEncoding": "messagepack", + "payload": document.encode_payload(Item("sword", "gear", 3)), + } + ], + } + applied_snapshot = apply_raw_snapshot(cache, [document], snapshot) + self.assertEqual(applied_snapshot[0].record_key, "item:1") + self.assertEqual(cache.get_required(document, "item:1").value, 3) + typed_snapshot = CultNetRawSnapshotResponse.from_wire(snapshot) + direct_applied = apply_raw_document_record(cache, schema_document_map([document]), typed_snapshot.documents[0]) + self.assertEqual(direct_applied.schema_id, schema_id) + self.assertEqual(direct_applied.record_key, "item:1") + typed_snapshot_applied = apply_raw_snapshot(cache, [document], typed_snapshot) + self.assertEqual(typed_snapshot_applied[0].record_key, "item:1") + + shard_log = { + "schemaVersion": "cultnet.shard_log_response.v0", + "messageId": "log-1", + "shardId": "interop", + "shardEpoch": 1, + "resyncRequired": False, + "entries": [ + { + "sequence": 0, + "changeKind": "updated", + "put": { + "schemaVersion": "cultnet.document_put_raw.v0", + "messageId": "put-unknown", + "document": { + "schemaId": "replica.unknown.v1", + "recordKey": "unknown:1", + "storedAt": "2026-06-13T00:00:01Z", + "payloadEncoding": "messagepack", + "payload": document.encode_payload(Item("ignored", "unknown", 0)), + }, + "shardId": "interop", + "shardEpoch": 1, + }, + }, + { + "sequence": 1, + "changeKind": "updated", + "put": { + "schemaVersion": "cultnet.document_put_raw.v0", + "messageId": "put-1", + "document": { + "schemaId": schema_id, + "recordKey": "item:1", + "storedAt": "2026-06-13T00:00:01Z", + "payloadEncoding": "messagepack", + "payload": document.encode_payload(Item("sword", "gear", 12)), + }, + "shardId": "interop", + "shardEpoch": 1, + }, + }, + { + "sequence": 2, + "changeKind": "removed", + "delete": { + "schemaVersion": "cultnet.document_delete.v0", + "messageId": "delete-1", + "schemaId": schema_id, + "recordKey": "item:1", + "shardId": "interop", + "shardEpoch": 1, + }, + }, + ], + } + applied_log = apply_shard_log_response(cache, [document], shard_log) + self.assertEqual([change.change_kind for change in applied_log], ["updated", "removed"]) + self.assertIsNone(cache.get(document, "item:1")) + + cache.put(document, "item:1", Item("sword", "gear", 3)) + typed_shard_log = CultNetShardLogResponse.from_wire({ + **shard_log, + "entries": shard_log["entries"][1:], + }) + typed_applied_log = apply_shard_log_response(cache, [document], typed_shard_log) + self.assertEqual([change.change_kind for change in typed_applied_log], ["updated", "removed"]) + self.assertIsNone(cache.get(document, "item:1")) + + def test_cultnet_replication_helpers_recover_schema_stamped_raw_records(self) -> None: + document = define_database_entry_type( + "replica.runtime-policy", + [ + ("schema_version", 0), + ("name", 1), + ("value", 2), + ], + schema_id="replica.runtime-policy.current", + schema_name="replica.runtime_policy", + schema_version="replica.runtime_policy.v1", + ) + cache = CultCache() + cache.register_document_type(document) + stale_schema_id = "sha256:stale-replica-runtime-policy" + payload = document.encode_payload({ + "schema_version": "replica.runtime_policy.v1", + "name": "policy", + "value": "still synced", + }) + record = { + "schemaId": stale_schema_id, + "recordKey": "policy:1", + "storedAt": "2026-06-25T00:00:00Z", + "payloadEncoding": "messagepack", + "payload": payload, + } + snapshot = { + "schemaVersion": "cultnet.snapshot_response_raw.v0", + "messageId": "snapshot-stale", + "documents": [record], + } + + applied_snapshot = apply_raw_snapshot(cache, [document], snapshot) + local_schema_id = document.catalog_entry().schema_id + self.assertEqual(applied_snapshot[0].schema_id, local_schema_id) + self.assertEqual(cache.get_required(document, "policy:1")["value"], "still synced") + self.assertEqual(cache.get_required_envelope(document, "policy:1").schema_id, local_schema_id) + direct_applied = apply_raw_document_record(cache, schema_document_map([document]), record) + self.assertEqual(direct_applied.schema_id, local_schema_id) + self.assertEqual(cache.get_required_envelope(document, "policy:1").schema_id, local_schema_id) + + shard_log = { + "schemaVersion": "cultnet.shard_log_response.v0", + "messageId": "log-stale", + "shardId": "interop", + "shardEpoch": 1, + "resyncRequired": False, + "entries": [ + { + "sequence": 0, + "changeKind": "updated", + "put": { + "schemaVersion": "cultnet.document_put_raw.v0", + "messageId": "put-stale", + "document": { + **record, + "payload": document.encode_payload({ + "schema_version": "replica.runtime_policy.v1", + "name": "policy", + "value": "updated through log", + }), + }, + "shardId": "interop", + "shardEpoch": 1, + }, + } + ], + } + + applied_log = apply_shard_log_response(cache, [document], shard_log) + self.assertEqual(applied_log[0].schema_id, local_schema_id) + self.assertEqual(cache.get_required(document, "policy:1")["value"], "updated through log") + self.assertEqual(cache.get_required_envelope(document, "policy:1").schema_id, local_schema_id) + + delete_log = { + "schemaVersion": "cultnet.shard_log_response.v0", + "messageId": "log-delete-alias", + "shardId": "interop", + "shardEpoch": 1, + "resyncRequired": False, + "entries": [ + { + "sequence": 1, + "changeKind": "removed", + "delete": { + "schemaVersion": "cultnet.document_delete.v0", + "messageId": "delete-version-alias", + "schemaId": "replica.runtime_policy.v1", + "recordKey": "policy:1", + "shardId": "interop", + "shardEpoch": 1, + }, + } + ], + } + applied_delete = apply_shard_log_response(cache, [document], delete_log) + self.assertEqual(applied_delete[0].schema_id, local_schema_id) + self.assertEqual(applied_delete[0].change_kind, "removed") + self.assertIsNone(cache.get(document, "policy:1")) + + def test_cultnet_document_delete_helper_matches_schema_v0_shape(self) -> None: + message = document_delete( + message_id="delete-1", + schema_id="schema-a", + record_key="record-a", + shard_id="interop", + shard_epoch=1, + ).to_wire() + self.assertEqual(message["schemaVersion"], "cultnet.document_delete.v0") + self.assertEqual(message["messageId"], "delete-1") + self.assertEqual(message["schemaId"], "schema-a") + self.assertEqual(message["recordKey"], "record-a") + self.assertEqual(message["shardId"], "interop") + self.assertEqual(message["shardEpoch"], 1) + + def test_cultnet_auth_session_helpers_match_schema_v0_shape(self) -> None: + self.assertEqual( + login(nonce="nonce", auth="encrypted-auth", password="encrypted-password").to_wire(), + { + "schemaVersion": "cultnet.login.v0", + "nonce": "nonce", + "auth": "encrypted-auth", + "password": "encrypted-password", + }, + ) + self.assertEqual( + register( + nonce="nonce", + email="encrypted-email", + password="encrypted-password", + name="encrypted-name", + ).to_wire(), + { + "schemaVersion": "cultnet.register.v0", + "nonce": "nonce", + "email": "encrypted-email", + "password": "encrypted-password", + "name": "encrypted-name", + }, + ) + self.assertEqual( + verify_session(nonce="nonce", session="encrypted-session").to_wire(), + { + "schemaVersion": "cultnet.verify.v0", + "nonce": "nonce", + "session": "encrypted-session", + }, + ) + self.assertEqual( + login_success(nonce="nonce", session="encrypted-session").to_wire(), + { + "schemaVersion": "cultnet.login_success.v0", + "nonce": "nonce", + "session": "encrypted-session", + }, + ) + + def test_cultnet_wire_catalog_describes_python_handled_messages(self) -> None: + descriptors = wire_message_schema_descriptors(include_schema_json=True) + by_version = {descriptor["schemaVersion"]: descriptor for descriptor in descriptors} + for schema_version in [ + "cultnet.error.v0", + "cultnet.login.v0", + "cultnet.register.v0", + "cultnet.verify.v0", + "cultnet.login_success.v0", + "cultnet.transport_profile.v0", + "cultnet.document_delete.v0", + "cultnet.database_change_raw.v0", + "cultnet.shard_log_response.v0", + "cultnet.simulation_consensus_candidate.v0", + "cultmesh.peer_exchange_response.v0", + ]: + self.assertIn(schema_version, by_version) + expected_kind = "shared_contract" if schema_version == "cultnet.transport_profile.v0" else "wire_message" + self.assertEqual(by_version[schema_version]["kind"], expected_kind) + self.assertIn("cultnet.schema.v0", by_version[schema_version]["wireContracts"]) + self.assertIn(schema_version, by_version[schema_version]["schemaJson"]) + self.assertEqual(len(by_version[schema_version]["contentHash"]), 64) + + delete_schema = json.loads(by_version["cultnet.document_delete.v0"]["schemaJson"]) + self.assertEqual( + delete_schema["required"], + ["schemaVersion", "messageId", "schemaId", "recordKey"], + ) + self.assertEqual(delete_schema["properties"]["recordKey"]["type"], "string") + + error_schema = json.loads(by_version["cultnet.error.v0"]["schemaJson"]) + self.assertEqual(error_schema["required"], ["schemaVersion", "error"]) + self.assertEqual(error_schema["properties"]["error"]["type"], "string") + self.assertIn("details", error_schema["properties"]) + + login_schema = json.loads(by_version["cultnet.login.v0"]["schemaJson"]) + self.assertEqual(login_schema["required"], ["schemaVersion", "nonce", "auth", "password"]) + self.assertEqual(login_schema["properties"]["auth"]["type"], "string") + + register_schema = json.loads(by_version["cultnet.register.v0"]["schemaJson"]) + self.assertEqual(register_schema["required"], ["schemaVersion", "nonce", "email", "password", "name"]) + self.assertEqual(register_schema["properties"]["email"]["type"], "string") + + verify_schema = json.loads(by_version["cultnet.verify.v0"]["schemaJson"]) + self.assertEqual(verify_schema["required"], ["schemaVersion", "nonce", "session"]) + self.assertEqual(verify_schema["properties"]["session"]["type"], "string") + + success_schema = json.loads(by_version["cultnet.login_success.v0"]["schemaJson"]) + self.assertEqual(success_schema["required"], ["schemaVersion", "nonce", "session"]) + + transport_profile_schema = json.loads(by_version["cultnet.transport_profile.v0"]["schemaJson"]) + self.assertEqual(transport_profile_schema["required"], ["schemaVersion", "runtimeId", "transports"]) + transport = transport_profile_schema["properties"]["transports"]["items"] + self.assertEqual(transport["properties"]["protocol"]["enum"], ["tcp_framed", "litenetlib", "websocket", "rudp"]) + self.assertEqual( + transport["properties"]["reconnectPolicy"]["properties"]["schemaVersion"]["const"], + "cultnet.reconnect_policy.v0", + ) + channel = transport["properties"]["channels"]["items"] + self.assertEqual(channel["properties"]["ordering"]["enum"], ["ordered", "unordered", "sequenced"]) + + change_schema = json.loads(by_version["cultnet.database_change_raw.v0"]["schemaJson"]) + self.assertEqual(change_schema["properties"]["changeKind"]["enum"], ["added", "updated", "removed"]) + self.assertIn("document", change_schema["properties"]) + self.assertIn("schemaId", change_schema["properties"]) + self.assertIn("recordKey", change_schema["properties"]) + + log_schema = json.loads(by_version["cultnet.shard_log_response.v0"]["schemaJson"]) + self.assertIn("entries", log_schema["required"]) + self.assertIn("resyncRequired", log_schema["required"]) + self.assertIn("compactedThrough", log_schema["properties"]) + + hello_schema = json.loads(by_version["cultnet.hello.v0"]["schemaJson"]) + mutation_contract = hello_schema["properties"]["supportedMutationContracts"]["items"] + self.assertEqual(mutation_contract["type"], "object") + self.assertIn("operations", mutation_contract["properties"]) + self.assertEqual( + hello_schema["properties"]["transportProfiles"]["items"]["$ref"], + "https://github.com/GameCult/cultnet-ts/contracts/cultnet.transport-profile.schema.json", + ) + + consensus_schema = json.loads(by_version["cultnet.simulation_consensus_candidate.v0"]["schemaJson"]) + for required_field in ["witnessCount", "supportWeight", "totalWeight", "hasQuorum", "confidence"]: + self.assertIn(required_field, consensus_schema["required"]) + + def test_cultnet_schema_catalog_applies_filters_and_responses(self) -> None: + catalog = wire_message_schema_catalog(include_schema_json=True) + descriptor = catalog.get("https://github.com/GameCult/cultnet-ts/contracts/cultnet.document-put-raw.schema.json") + self.assertIsNotNone(descriptor) + assert descriptor is not None + self.assertEqual(descriptor.kind, "wire_message") + self.assertEqual(descriptor.schema_version, "cultnet.document_put_raw.v0") + self.assertIsNotNone(descriptor.schema_json) + + filtered = catalog.list(kinds=["wire_message"], schema_ids=[descriptor.schema_id]) + self.assertEqual(filtered, [descriptor]) + + response = catalog.create_response( + message_id="catalog-response", + include_schema_json=False, + schema_ids=[descriptor.schema_id], + kinds=["wire_message"], + ) + self.assertEqual(response["schemaVersion"], "cultnet.schema_catalog_response.v0") + self.assertNotIn("schemaJson", response["schemas"][0]) + + remote = CultNetSchemaCatalog() + applied = remote.apply_response({ + "schemaVersion": "cultnet.schema_catalog_response.v0", + "messageId": "remote", + "schemas": [ + CultNetSchemaDescriptor( + schema_id="schema:custom", + kind="shared_contract", + schema_version="custom.v0", + wire_contracts=("cultnet.schema.v0",), + content_hash="hash", + schema_json="{}", + ).to_wire() + ], + }) + self.assertEqual(applied[0].schema_id, "schema:custom") + self.assertEqual(remote.get("schema:custom"), applied[0]) + + def test_cultnet_shard_helpers_match_schema_v0_shape(self) -> None: + catalog = shard_catalog_request( + message_id="catalog-1", + schema_ids=["schema-a"], + record_keys=["record-a"], + ).to_wire() + self.assertEqual(catalog["schemaVersion"], "cultnet.shard_catalog_request.v0") + self.assertEqual(catalog["messageId"], "catalog-1") + self.assertEqual(catalog["schemaIds"], ["schema-a"]) + self.assertEqual(catalog["recordKeys"], ["record-a"]) + + log = shard_log_request( + message_id="log-1", + shard_id="interop", + shard_epoch=7, + after_sequence=3, + limit=2, + ).to_wire() + self.assertEqual(log["schemaVersion"], "cultnet.shard_log_request.v0") + self.assertEqual(log["messageId"], "log-1") + self.assertEqual(log["shardId"], "interop") + self.assertEqual(log["shardEpoch"], 7) + self.assertEqual(log["afterSequence"], 3) + self.assertEqual(log["limit"], 2) + + def test_cultnet_shard_catalog_applies_filters_and_responses(self) -> None: + catalog = CultNetShardCatalog() + primary = catalog.upsert(CultNetShardDescriptor( + shard_id="notes", + owner_runtime_id="python-runtime", + epoch=2, + is_primary=True, + schema_ids=("schema-note.v1",), + key_prefix="note:", + primary_endpoints=("cultnet://127.0.0.1:3075",), + read_replica_endpoints=("cultnet://127.0.0.1:3075",), + region="local", + )) + catalog.upsert({ + "shardId": "facts", + "ownerRuntimeId": "python-runtime", + "epoch": 1, + "schemaIds": ["schema-fact"], + "keyPrefix": "fact:", + "primaryEndpoints": ["cultnet://127.0.0.1:3076"], + }) + + self.assertTrue(primary.serves(schema_id="schema-note.v1", record_key="note:1")) + self.assertTrue(primary.serves(schema_id="schema-note", record_key="note:1")) + self.assertFalse(primary.serves(schema_id="schema-note", record_key="fact:1")) + self.assertEqual(catalog.list(schema_ids=["schema-note.v1"]), [primary]) + self.assertEqual(catalog.list(schema_ids=["schema-note"]), [primary]) + self.assertEqual(catalog.list(record_keys=["note:1"]), [primary]) + + response = catalog.create_response( + message_id="shards", + schema_ids=["schema-note"], + record_keys=["note:1"], + ) + self.assertEqual(response["schemaVersion"], "cultnet.shard_catalog_response.v0") + self.assertEqual(response["shards"][0]["shardId"], "notes") + self.assertEqual(response["shards"][0]["ownerRuntimeId"], "python-runtime") + + remote = CultNetShardCatalog() + applied = remote.apply_response(response) + self.assertEqual(applied[0].shard_id, "notes") + self.assertEqual(remote.get("notes"), applied[0]) + + def test_cultnet_shard_log_response_tracks_cursor_and_resync_state(self) -> None: + response = CultNetShardLogResponse.from_wire({ + "schemaVersion": "cultnet.shard_log_response.v0", + "messageId": "log", + "shardId": "notes", + "shardEpoch": 2, + "entries": [ + CultNetShardLogEntry( + sequence=2, + change_kind="updated", + put={"schemaVersion": "cultnet.document_put_raw.v0", "messageId": "put", "document": {"schemaId": "schema-note", "recordKey": "note:1", "payload": b"p"}}, + committed_at="2026-06-14T00:00:00Z", + ).to_wire(), + {"sequence": 3, "changeKind": "removed", "delete": {"schemaId": "schema-note", "recordKey": "note:1"}}, + ], + "resyncRequired": False, + }) + self.assertEqual(response.last_sequence, 3) + self.assertFalse(response.resync_required) + self.assertEqual(response.entries[0].put["document"]["recordKey"], "note:1") + self.assertIsInstance(response.entries[0].raw_document, CultNetRawDocumentRecord) + self.assertEqual(response.entries[0].raw_document.record_key, "note:1") + self.assertEqual(response.entries[1].delete_schema_id, "schema-note") + self.assertEqual(response.entries[1].delete_record_key, "note:1") + self.assertEqual(response.to_wire()["entries"][1]["delete"]["recordKey"], "note:1") + self.assertIs(response.require_usable(), response) + + resync = CultNetShardLogResponse.from_wire({ + "schemaVersion": "cultnet.shard_log_response.v0", + "messageId": "log-resync", + "shardId": "notes", + "shardEpoch": 2, + "entries": [], + "resyncRequired": True, + "reason": "compacted", + "compactedThrough": 12, + }) + self.assertEqual(resync.last_sequence, 12) + with self.assertRaisesRegex(ValueError, "compacted"): + resync.require_usable() + + def test_cultnet_simulation_observation_helper_matches_schema_v0_shape(self) -> None: + claim_hash = compute_simulation_claim_hash("frame:42", "subject:player-1", "hit") + self.assertEqual(len(claim_hash), 64) + message = simulation_observation( + message_id="obs-1", + witness_runtime_id="python-test", + shard_id="interop", + shard_epoch=1, + frame=42, + subject_id="player-1", + claim_kind="hit", + claim_hash=claim_hash, + claim_summary="player-1 hit target-a", + observed_at="2026-06-13T00:00:02Z", + ).to_wire() + self.assertEqual(message["schemaVersion"], "cultnet.simulation_observation.v0") + self.assertEqual(message["messageId"], "obs-1") + self.assertEqual(message["observation"]["witnessRuntimeId"], "python-test") + self.assertEqual(message["observation"]["claimHash"], claim_hash) + self.assertEqual(message["observation"]["weight"], 1.0) + observation = CultNetSimulationObservation.from_wire(message) + self.assertEqual(observation.witness_runtime_id, "python-test") + self.assertEqual(observation.claim_summary, "player-1 hit target-a") + self.assertEqual(observation.to_message_wire(message_id="obs-2")["observation"], message["observation"]) + + def test_cultnet_witness_artifact_bundle_uses_csharp_slot_order(self) -> None: + import msgpack # type: ignore + + bundle = witness_artifact_bundle( + bundle_id="bundle-1", + witness_kind="interop-proof", + captured_at="2026-06-13T00:00:03Z", + subject={"documentType": "cultnet.interop-note", "subjectId": "note:python"}, + contracts=[{"role": "payload", "schemaId": "schema-a"}], + artifacts=[{"role": "log", "uri": "cultcache://bundle-1/log", "mediaType": "text/plain"}], + timing_witnesses=[{"stage": "roundtrip", "startedAt": "2026-06-13T00:00:03Z", "completedAt": "2026-06-13T00:00:04Z", "latencyMs": 1.0}], + provenance={"pipelineId": "interop", "runId": "run-1", "runtimeId": "python-test"}, + ) + payload = encode_witness_artifact_bundle_payload(bundle) + decoded = decode_witness_artifact_bundle_payload(payload) + typed = CultNetWitnessArtifactBundle.from_wire(bundle) + typed_payload = typed.to_payload() + typed_decoded = CultNetWitnessArtifactBundle.from_payload(typed_payload) + slots = msgpack.unpackb(payload, raw=False) + + self.assertEqual(payload, typed_payload) + self.assertEqual(slots[0], "bundle-1") + self.assertEqual(slots[1], "interop-proof") + self.assertEqual(slots[2], "2026-06-13T00:00:03Z") + self.assertEqual(slots[3]["subjectId"], "note:python") + self.assertEqual(slots[4][0]["schemaId"], "schema-a") + self.assertEqual(slots[5][0]["uri"], "cultcache://bundle-1/log") + self.assertEqual(slots[6][0]["latencyMs"], 1.0) + self.assertEqual(slots[7]["runtimeId"], "python-test") + self.assertEqual(decoded["bundleId"], "bundle-1") + self.assertEqual(decoded["witnessKind"], "interop-proof") + self.assertEqual(decoded["subject"]["subjectId"], "note:python") + self.assertEqual(decoded["contracts"][0]["schemaId"], "schema-a") + self.assertEqual(decoded["artifacts"][0]["mediaType"], "text/plain") + self.assertEqual(decoded["provenance"]["runtimeId"], "python-test") + self.assertEqual(typed.bundle_id, "bundle-1") + self.assertEqual(typed_decoded.to_wire(), bundle) + + def test_python_interop_peer_filters_shard_snapshot_by_logged_membership(self) -> None: + runtime_id = f"python-interop-test-{uuid4().hex}" + state = build_state( + runtime_id=runtime_id, + runtime_kind="python", + display_name="Python Interop Test", + agent_id="python-interop-test-agent", + schema_path=str(Path("packages/cultnet-ts/integration/contracts/cultnet.interop-note.schema.json")), + ) + binding = state.bindings["note"] + logged_value = { + "schemaVersion": "cultnet.interop_note.v0", + "documentId": "note:logged", + "authorRuntimeId": runtime_id, + "title": "Logged", + "body": "This record belongs to the shard log.", + "tags": ["interop", "logged"], + } + unlogged_value = { + **logged_value, + "documentId": "note:unlogged", + "title": "Unlogged", + "body": "This record is in cache but not in the shard log.", + } + state.cache.put(binding.document, "note:logged", logged_value) + state.cache.put(binding.document, "note:unlogged", unlogged_value) + logged_record = { + "schemaId": state.note_schema_id, + "recordKey": "note:logged", + "storedAt": "2026-06-14T00:00:00Z", + "payloadEncoding": "messagepack", + "payload": binding.document.encode_payload(logged_value), + } + append_shard_log_put( + state, + {"messageId": "logged-put"}, + logged_record, + ) + + response = raw_snapshot_response( + state, + { + "schemaVersion": "cultnet.snapshot_request.v0", + "messageId": "interop-shard-snapshot", + "schemaIds": [state.note_schema_id], + "shardId": state.shard_id, + }, + ) + + self.assertEqual(response["shardId"], state.shard_id) + self.assertEqual(response["shardLogSequence"], 1) + self.assertEqual([record["recordKey"] for record in response["documents"]], ["note:logged"]) + + def test_cultnet_simulation_consensus_dedupes_witnesses_and_requires_quorum(self) -> None: + from cultnet_py import CultNetSimulationConsensus + + consensus = CultNetSimulationConsensus( + CultNetSimulationConsensusOptions(minimum_witnesses=2, quorum_ratio=1.0) + ) + claim_hash = compute_simulation_claim_hash("hit", "alice", "bob", "frame:100") + observations = [ + { + "witnessRuntimeId": "watcher-1", + "shardId": "arena", + "shardEpoch": 4, + "frame": 100, + "subjectId": "bob", + "claimKind": "hit", + "claimHash": claim_hash, + "claimSummary": "alice shot bob first", + "weight": 1.0, + }, + { + "witnessRuntimeId": "watcher-1", + "shardId": "arena", + "shardEpoch": 4, + "frame": 100, + "subjectId": "bob", + "claimKind": "hit", + "claimHash": "stale-duplicate", + "weight": 0.5, + }, + { + "witnessRuntimeId": "watcher-2", + "shardId": "arena", + "shardEpoch": 4, + "frame": 100, + "subjectId": "bob", + "claimKind": "hit", + "claimHash": claim_hash, + "weight": 1.0, + }, + ] + + candidates = consensus.build_candidates(observations) + typed_candidates = consensus.build_candidate_objects([ + CultNetSimulationObservation.from_wire(observation) + for observation in observations + ]) + + self.assertEqual(len(candidates), 1) + self.assertEqual(candidates[0]["claimHash"], claim_hash) + self.assertEqual(candidates[0]["witnessCount"], 2) + self.assertEqual(candidates[0]["supportWeight"], 2.0) + self.assertTrue(candidates[0]["hasQuorum"]) + self.assertEqual(len(typed_candidates), 1) + self.assertIsInstance(typed_candidates[0], CultNetSimulationConsensusCandidate) + self.assertEqual(typed_candidates[0].claim_hash, claim_hash) + self.assertEqual(typed_candidates[0].to_wire(), candidates[0]) + self.assertEqual(CultNetSimulationConsensusCandidate.from_wire(candidates[0]), typed_candidates[0]) + + +class CultNetRudpFragmentBoundTests(unittest.TestCase): + def _pair(self) -> tuple[CultNetRudpSession, CultNetRudpSession]: + sender = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=311, initial_sequence=1)) + receiver = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=311, initial_sequence=100)) + sender.receive(CultNetRudpPacket(CultNetRudpPacketType.ACCEPT, 311, 90, 0, 0, "control")) + receiver.receive(CultNetRudpPacket(CultNetRudpPacketType.ACCEPT, 311, 91, 0, 0, "control")) + return sender, receiver + + def test_stranded_fragment_sets_are_bounded_and_evicted_oldest_first(self) -> None: + sender, receiver = self._pair() + receiver._max_pending_fragment_sets = 4 + + # Eight payloads each lose their last fragment; the map must not grow past four. + stranded = [] + for fill in range(1, 9): + packets = sender.send_many("media", bytes([fill]) * 2500, max_fragment_bytes=1000) + self.assertEqual(len(packets), 3) + for packet in packets[:2]: + self.assertEqual(receiver.receive(packet).delivered, ()) + stranded.append(packets[2]) + self.assertEqual(len(receiver._fragment_buffers), 4) + self.assertEqual(receiver.fragment_sets_evicted, 4) + + # A complete payload still lands. + delivered = [] + for packet in sender.send_many("media", bytes([9]) * 2500, max_fragment_bytes=1000): + delivered.extend(receiver.receive(packet).delivered) + self.assertEqual([frame.payload for frame in delivered], [bytes([9]) * 2500]) + self.assertEqual(receiver.fragment_sets_evicted, 5) + + # Oldest-first: set 8 still pends and completes late; set 1 was evicted. + self.assertEqual(receiver.receive(stranded[7]).delivered[0].payload, bytes([8]) * 2500) + self.assertEqual(receiver.receive(stranded[0]).delivered, ()) + + +if __name__ == "__main__": + unittest.main() diff --git a/packages/cultcache-py/tools/GameCult.Caching.Benchmark/GameCult.Caching.Benchmark.csproj b/packages/cultnet-py/tools/GameCult.Caching.Benchmark/GameCult.Caching.Benchmark.csproj similarity index 100% rename from packages/cultcache-py/tools/GameCult.Caching.Benchmark/GameCult.Caching.Benchmark.csproj rename to packages/cultnet-py/tools/GameCult.Caching.Benchmark/GameCult.Caching.Benchmark.csproj diff --git a/packages/cultcache-py/tools/GameCult.Caching.Benchmark/Program.cs b/packages/cultnet-py/tools/GameCult.Caching.Benchmark/Program.cs similarity index 96% rename from packages/cultcache-py/tools/GameCult.Caching.Benchmark/Program.cs rename to packages/cultnet-py/tools/GameCult.Caching.Benchmark/Program.cs index 698b07ec..14ec8d5e 100644 --- a/packages/cultcache-py/tools/GameCult.Caching.Benchmark/Program.cs +++ b/packages/cultnet-py/tools/GameCult.Caching.Benchmark/Program.cs @@ -1,6 +1,7 @@ using System.Diagnostics; using System.Text.Json; using GameCult.Caching; +using MessagePack; var records = 5000; var emitJson = false; @@ -93,12 +94,15 @@ internal sealed record BenchmarkResult(string Runtime, int Records, IReadOnlyLis internal sealed record BenchmarkMetric(string Name, int Operations, double ElapsedMs, double OpsPerSecond); [CultDocument("bench.item", "bench.item.v1")] +[MessagePackObject(AllowPrivate = true)] internal sealed class BenchItem { - [CultName] + [Key(0)] [CultName] public string Name { get; set; } = string.Empty; + [Key(1)] public string Category { get; set; } = string.Empty; + [Key(2)] public int Value { get; set; } } diff --git a/packages/cultnet-rs/Cargo.toml b/packages/cultnet-rs/Cargo.toml index 41d14b52..f289d86e 100644 --- a/packages/cultnet-rs/Cargo.toml +++ b/packages/cultnet-rs/Cargo.toml @@ -16,6 +16,7 @@ anyhow = "1" base64 = "0.22" chrono = { version = "0.4", features = ["serde"] } cultcache-rs = { path = "../cultcache-rs" } +ed25519-dalek = "2.2" hmac = "0.12" rand = "0.9" rmp-serde = "1.3" @@ -24,9 +25,13 @@ serde = { version = "1", features = ["derive"] } serde_bytes = "0.11" serde_json = "1" sha2 = "0.10" +zeroize = "1" socket2 = "0.5" uuid = { version = "1", features = ["v4"] } +[target.'cfg(windows)'.dependencies] +windows-sys = { version = "0.61", features = ["Win32_Foundation", "Win32_Security_Cryptography"] } + [dev-dependencies] pretty_assertions = "1.4" tempfile = "3" diff --git a/packages/cultnet-rs/examples/cultnet_interop_peer.rs b/packages/cultnet-rs/examples/cultnet_interop_peer.rs index ab00c6c8..07f6f400 100644 --- a/packages/cultnet-rs/examples/cultnet_interop_peer.rs +++ b/packages/cultnet-rs/examples/cultnet_interop_peer.rs @@ -256,7 +256,7 @@ fn serve(config: PeerConfig) -> Result<()> { register_capability_entry_types(&mut cache)?; cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(runtime_store_path( &config.runtime_id, - ))); + )))?; cache.pull_all_backing_stores()?; let note = build_note(&config.runtime_id, &config.display_name); cache.put(¬e.document_id, ¬e)?; @@ -380,7 +380,7 @@ fn dial(config: DialConfig) -> Result<()> { register_capability_entry_types(&mut cache)?; cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(runtime_store_path( &format!("{}-dial", config.runtime_id), - ))); + )))?; cache.pull_all_backing_stores()?; let mut document_registry = CultNetDocumentRegistry::new(); @@ -986,6 +986,9 @@ fn run_rudp_server( if let Some(reply) = result.reply { send_rudp_packet(&socket, remote_addr, &reply)?; } + for ready in result.ready_to_send { + send_rudp_packet(&socket, remote_addr, &ready)?; + } if packet.packet_type == CultNetRudpPacketType::Data { let ack = peer.session.create_ack(); send_rudp_packet(&socket, remote_addr, &ack)?; @@ -1471,6 +1474,7 @@ impl SchemaMessageSender for RudpSessionSender<'_> { ordered: true, sequenced: false, now_ms: rudp_now_ms(), + reliable_expire_after_ms: None, }, Some(RUDP_INTEROP_MAX_FRAGMENT_BYTES), )?; diff --git a/packages/cultnet-rs/examples/media_delivery_probe.rs b/packages/cultnet-rs/examples/media_delivery_probe.rs new file mode 100644 index 00000000..397eb656 --- /dev/null +++ b/packages/cultnet-rs/examples/media_delivery_probe.rs @@ -0,0 +1,183 @@ +//! Measures what a media channel actually costs on the wire under loss. +//! +//! The claim under test: a reliable media channel retransmits lost packets, so +//! it adds load exactly when the link has least to give, while an unreliable one +//! drops them and holds its rate. Run both halves through `cultnet-impair` with +//! a `loss_basis_points` profile and compare bytes on the wire against bytes of +//! payload offered. +//! +//! ```text +//! media_delivery_probe recv +//! media_delivery_probe send +//! ``` +//! +//! The sender reports amplification: wire bytes divided by offered payload +//! bytes. At 1.0 nothing was retransmitted. Well above 1.0 under loss is the +//! pathology. + +use std::env; +use std::net::{SocketAddr, UdpSocket}; +use std::time::{Duration, Instant}; + +use anyhow::{Result, anyhow}; +use cultnet_rs::{ + CultNetRudpServerHubOptions, CultNetRudpSocketTransportConnection, + CultNetRudpSocketTransportOptions, CultNetTransportDelivery, +}; + +const CHANNEL: &str = "media"; +const DRAIN: Duration = Duration::from_secs(5); + +fn main() -> Result<()> { + let args: Vec = env::args().collect(); + match args.get(1).map(String::as_str) { + Some("recv") => recv( + args.get(2).ok_or_else(|| anyhow!("bind addr"))?.parse()?, + args.get(3).ok_or_else(|| anyhow!("expect count"))?.parse()?, + ), + Some("send") => send( + args.get(2).ok_or_else(|| anyhow!("endpoint"))?.parse()?, + match args.get(3).map(String::as_str) { + Some("unreliable") => CultNetTransportDelivery::Unreliable, + Some("reliable") => CultNetTransportDelivery::Reliable, + other => return Err(anyhow!("delivery must be reliable|unreliable, got {other:?}")), + }, + args.get(4).ok_or_else(|| anyhow!("payload bytes"))?.parse()?, + args.get(5).ok_or_else(|| anyhow!("count"))?.parse()?, + args.get(6).map(|v| v.parse()).transpose()?.unwrap_or(12.0), + args.get(7).map(|v| v.parse()).transpose()?, + ), + _ => Err(anyhow!( + "usage: media_delivery_probe recv | send [mbps] [max_fragment_bytes]" + )), + } +} + +fn recv(bind: SocketAddr, expect: u64) -> Result<()> { + let socket = UdpSocket::bind(bind)?; + socket.set_nonblocking(true)?; + let mut hub = cultnet_rs::CultNetRudpServerHub::new(CultNetRudpServerHubOptions::new( + "probe-recv", + socket, + 0x0BE0_0001, + ))?; + + println!("listening on {bind}, expecting {expect} frames"); + let started = Instant::now(); + let mut frames = 0u64; + let mut payload_bytes = 0u64; + let mut last_progress = Instant::now(); + + loop { + while let Some(event) = hub.receive_event_once()? { + if let cultnet_rs::CultNetRudpServerEvent::Frame { frame, .. } = event + && frame.channel_id == CHANNEL + { + frames += 1; + payload_bytes += frame.payload.len() as u64; + last_progress = Instant::now(); + } + } + if frames >= expect || last_progress.elapsed() > DRAIN { + break; + } + std::thread::sleep(Duration::from_micros(200)); + } + + let stats = hub.stats(); + let elapsed = started.elapsed().as_secs_f64(); + println!("RESULT recv frames={frames} expected={expect} payload_bytes={payload_bytes}"); + println!( + "RESULT recv wire_bytes_in={} wire_bytes_out={} elapsed_s={elapsed:.2} delivered_pct={:.2} fragment_sets_evicted={}", + stats.bytes_received, + stats.bytes_sent, + (frames as f64 / expect as f64) * 100.0, + stats.fragment_sets_evicted + ); + Ok(()) +} + +fn send( + endpoint: SocketAddr, + delivery: CultNetTransportDelivery, + payload_bytes: usize, + count: u64, + target_mbps: f64, + max_fragment_bytes: Option, +) -> Result<()> { + let socket = UdpSocket::bind("0.0.0.0:0")?; + socket.set_nonblocking(true)?; + let mut options = + CultNetRudpSocketTransportOptions::client("probe-send", socket, endpoint, 0x0BE0_0001); + options.media_delivery = Some(delivery); + options.media_reliable_expire_after_ms = None; + // Fragment above this many bytes so a lost datagram costs a whole payload; + // None sends each payload as one datagram, which loopback carries whole. + options.max_fragment_bytes = max_fragment_bytes; + + let mut transport = CultNetRudpSocketTransportConnection::new(options)?; + transport.connect(b"probe".to_vec())?; + let deadline = Instant::now() + Duration::from_secs(10); + while !transport.connected() { + let _ = transport.receive_once()?; + transport.poll_resends()?; + if Instant::now() >= deadline { + return Err(anyhow!("timed out connecting to {endpoint}")); + } + std::thread::sleep(Duration::from_millis(2)); + } + + // Pace to a realistic media rate. An unpaced blast overruns the receiver's + // socket buffer and manufactures loss that has nothing to do with the link. + let per_packet = Duration::from_secs_f64((payload_bytes as f64 * 8.0) / (target_mbps * 1e6)); + let payload = vec![0xA5u8; payload_bytes]; + let offered = payload_bytes as u64 * count; + let started = Instant::now(); + let mut sent = 0u64; + let mut would_block = 0u64; + + while sent < count { + match transport.send(CHANNEL, payload.clone()) { + Ok(()) => sent += 1, + Err(error) if error.to_string().contains("would block") => { + would_block += 1; + transport.poll_resends()?; + let _ = transport.receive_once()?; + std::thread::sleep(Duration::from_micros(500)); + } + Err(error) => return Err(error), + } + if sent % 64 == 0 { + let _ = transport.receive_once()?; + transport.poll_resends()?; + } + let due = started + per_packet.mul_f64(sent as f64); + if let Some(wait) = due.checked_duration_since(Instant::now()) { + std::thread::sleep(wait); + } + } + let send_elapsed = started.elapsed(); + + // Let retransmission run: this is where a reliable channel spends what an + // unreliable one does not. + let drain_start = Instant::now(); + while drain_start.elapsed() < DRAIN { + let _ = transport.receive_once()?; + transport.poll_resends()?; + std::thread::sleep(Duration::from_millis(1)); + } + + let stats = transport.stats(); + let amplification = stats.bytes_sent as f64 / offered as f64; + println!( + "RESULT send delivery={delivery:?} payload_bytes={payload_bytes} count={count} offered={offered} target_mbps={target_mbps}" + ); + println!( + "RESULT send wire_bytes={} amplification={amplification:.3} send_s={:.2} total_s={:.2} would_block={would_block} reliable_expired={}", + stats.bytes_sent, + send_elapsed.as_secs_f64(), + started.elapsed().as_secs_f64(), + stats.reliable_packets_expired + ); + Ok(()) +} diff --git a/packages/cultnet-rs/examples/runtime_presence_vector.rs b/packages/cultnet-rs/examples/runtime_presence_vector.rs new file mode 100644 index 00000000..04571a8b --- /dev/null +++ b/packages/cultnet-rs/examples/runtime_presence_vector.rs @@ -0,0 +1,67 @@ +//! Emit canonical bytes for one fixed runtime-presence record. +//! +//! This is the reference the TypeScript fork is tested against. The two forks +//! agree only if `gamecult.runtime_presence_health.v2` means the same bytes in +//! both, and the only way to know that is to have one produce the vector and +//! the other reproduce it. A fixture written by hand for both sides would agree +//! with itself and prove nothing. + +use cultnet_rs::{GameCultRuntimeCapability, GameCultRuntimePresenceHealthRecord}; + +fn main() -> anyhow::Result<()> { + let record = GameCultRuntimePresenceHealthRecord { + schema_version: "gamecult.runtime_presence_health.v2".into(), + target: "heimdall".into(), + expected_projection_sha256: format!("sha256-{}", "a".repeat(64)), + plan_id: format!("sha256-{}", "b".repeat(64)), + incarnation_id: "service-incarnation-1".into(), + sealed_release_id: format!("sha256-{}", "c".repeat(64)), + activation_witness_sha256: format!("sha256-{}", "d".repeat(64)), + state_schema_generation: Some("v1".into()), + state_contract_sha256: Some(format!("sha256-{}", "e".repeat(64))), + runtime_id: "heimdall-yggdrasil".into(), + runtime_instance_id: format!("sha256-{}", "f".repeat(64)), + bound_endpoint: Some("http://127.0.0.1:14101".into()), + capabilities: vec![ + GameCultRuntimeCapability { + capability: "heimdall.access".into(), + schema: "heimdall.access.v1".into(), + compatibility: "v1".into(), + capacity: 2, + }, + GameCultRuntimeCapability { + capability: "zeta.runtime".into(), + schema: "zeta.runtime.v1".into(), + compatibility: "v1".into(), + capacity: 1, + }, + ], + health_contract: "heimdall.cultnet-rudp-provider-health".into(), + state: "warming".into(), + detail: "Heimdall candidate warming".into(), + write_lease_sha256: None, + signer_identity_id: "1".repeat(64), + publisher_sequence: 7, + observed_at_unix_millis: 1_757_000_000_000, + signature_algorithm: "ed25519".into(), + signature: Vec::new(), + activation_signer_identity_id: "2".repeat(64), + activation_signature: Vec::new(), + }; + + // The proof payload: both signature fields empty. This is what both keys + // sign, so it is the vector that matters most. + let proof = record.canonical_proof_payload()?; + println!("proof {}", hex(&proof)); + + // And the complete record, with both proofs filled, to pin the signed shape. + let mut signed = record; + signed.signature = vec![0x11; 64]; + signed.activation_signature = vec![0x22; 64]; + println!("signed {}", hex(&rmp_serde::to_vec(&signed)?)); + Ok(()) +} + +fn hex(bytes: &[u8]) -> String { + bytes.iter().map(|byte| format!("{byte:02x}")).collect() +} diff --git a/packages/cultnet-rs/src/contracts.rs b/packages/cultnet-rs/src/contracts.rs index e637883c..b5c89ff7 100644 --- a/packages/cultnet-rs/src/contracts.rs +++ b/packages/cultnet-rs/src/contracts.rs @@ -142,6 +142,12 @@ pub struct CultNetTransportChannel { pub max_fragment_bytes: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub max_pending_reliable_packets: Option, + /// Drop a reliable packet on this channel instead of retransmitting it once + /// this many milliseconds have passed since the send. Omitted when absent, + /// so a profile that does not set it is byte-identical to one from a + /// runtime that does not know the field. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub reliable_expire_after_ms: Option, } #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] diff --git a/packages/cultnet-rs/src/cultmesh.rs b/packages/cultnet-rs/src/cultmesh.rs index 20d21232..14998396 100644 --- a/packages/cultnet-rs/src/cultmesh.rs +++ b/packages/cultnet-rs/src/cultmesh.rs @@ -318,6 +318,8 @@ impl CultMesh { ) -> Result { let socket = bind_rudp_socket(&options)?; CultNetRudpSocketTransportConnection::new(CultNetRudpSocketTransportOptions { + media_reliable_expire_after_ms: None, + media_delivery: None, runtime_id: runtime_id.into(), socket, mode: CultNetRudpSocketMode::Server, @@ -341,6 +343,8 @@ impl CultMesh { ) -> Result { let socket = bind_rudp_socket(&options)?; CultNetRudpSocketTransportConnection::new(CultNetRudpSocketTransportOptions { + media_reliable_expire_after_ms: None, + media_delivery: None, runtime_id: runtime_id.into(), socket, mode: CultNetRudpSocketMode::Client, diff --git a/packages/cultnet-rs/src/idunn_deployment_brake.rs b/packages/cultnet-rs/src/idunn_deployment_brake.rs new file mode 100644 index 00000000..e790f2a1 --- /dev/null +++ b/packages/cultnet-rs/src/idunn_deployment_brake.rs @@ -0,0 +1,492 @@ +use anyhow::{Result, bail}; +use cultcache_rs::DatabaseEntry; + +use crate::{ + IdunnDeploymentBrakeOperatorIdentity, IdunnDeploymentBrakeReleasePurpose, + ServiceIdentitySignature, ServiceIdentityTrustAnchor, verify_service_identity_signature, +}; + +pub const IDUNN_DEPLOYMENT_BRAKE_SCHEMA: &str = "idunn.deployment_brake.v1"; +pub const IDUNN_DEPLOYMENT_BRAKE_TYPE: &str = "idunn.deployment_brake"; +pub const IDUNN_DEPLOYMENT_BRAKE_ID: &str = "idunn/deployment-brake"; +pub const IDUNN_DEPLOYMENT_BRAKE_AUTHORITY: &str = "idunn.deployment-brake"; +pub const IDUNN_DEPLOYMENT_BRAKE_SCOPE: &str = "deployment"; +pub const IDUNN_DEPLOYMENT_RELEASE_PURPOSE: &str = "idunn.deployment-release.v1"; +pub const IDUNN_DEPLOYMENT_RELEASE_MAX_LIFETIME_MILLIS: u64 = 15 * 60 * 1_000; + +/// Canonical deployment-facing brake projection consumed by Idunn. A released +/// status is not permission by itself: the authorization fields bind one +/// release and one deployment attempt for a short, explicit interval. +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache(type = "idunn.deployment_brake", schema = "idunn.deployment_brake.v1")] +pub struct IdunnDeploymentBrakeRecord { + #[cultcache(key = 0)] + pub schema_version: String, + #[cultcache(key = 1)] + pub brake_id: String, + #[cultcache(key = 2)] + pub authority: String, + #[cultcache(key = 3)] + pub runtime_id: String, + #[cultcache(key = 4)] + pub status: String, + #[cultcache(key = 5)] + pub scope: String, + #[cultcache(key = 6)] + pub reason: String, + #[cultcache(key = 7)] + pub observed_at_unix_millis: u64, + #[cultcache(key = 8)] + pub expires_at_unix_millis: Option, + #[cultcache(key = 9)] + pub authorization_id: Option, + #[cultcache(key = 10)] + pub authorization_purpose: Option, + #[cultcache(key = 11)] + pub authorized_release_id: Option, + #[cultcache(key = 12)] + pub authorized_deployment_id: Option, + #[cultcache(key = 13)] + pub authorized_by: Option, + #[cultcache(key = 14)] + pub authorization_issued_at_unix_millis: Option, + #[cultcache(key = 15)] + pub authorization_expires_at_unix_millis: Option, + #[cultcache(key = 16)] + pub signature_algorithm: Option, + #[cultcache(key = 17)] + pub signature: Option>, + #[cultcache(key = 18)] + pub private_state_exposed: bool, + /// Human/operator principal responsible for the latest transition. This is + /// distinct from `authorized_by`, which is the cryptographic identity id. + #[cultcache(key = 19)] + pub updated_by: String, +} + +impl IdunnDeploymentBrakeRecord { + pub fn validate(&self) -> Result<()> { + if self.schema_version != IDUNN_DEPLOYMENT_BRAKE_SCHEMA { + bail!("deployment brake schema is unsupported"); + } + identifier(&self.brake_id, "brake id")?; + identifier(&self.authority, "authority")?; + identifier(&self.runtime_id, "runtime id")?; + identifier(&self.reason, "reason")?; + identifier(&self.scope, "scope")?; + identifier(&self.updated_by, "updating operator")?; + if !matches!(self.status.as_str(), "engaged" | "released") + || self.observed_at_unix_millis == 0 + || self + .expires_at_unix_millis + .is_some_and(|v| v <= self.observed_at_unix_millis) + || self.private_state_exposed + { + bail!("deployment brake shape, lifetime, or privacy is invalid"); + } + + let authorization = [ + self.authorization_id.as_ref(), + self.authorization_purpose.as_ref(), + self.authorized_release_id.as_ref(), + self.authorized_deployment_id.as_ref(), + self.authorized_by.as_ref(), + self.signature_algorithm.as_ref(), + ]; + let has_authorization = authorization.iter().any(|v| v.is_some()) + || self.authorization_issued_at_unix_millis.is_some() + || self.authorization_expires_at_unix_millis.is_some() + || self.signature.is_some(); + if self.status == "engaged" { + if has_authorization { + bail!("engaged deployment brake must not carry release authorization"); + } + return Ok(()); + } + if authorization.iter().any(|v| v.is_none()) + || self.authorization_issued_at_unix_millis.is_none() + || self.authorization_expires_at_unix_millis.is_none() + || self.signature.is_none() + { + bail!("released deployment brake authorization is incomplete"); + } + for (value, label) in [ + (&self.authorization_id, "authorization id"), + (&self.authorized_release_id, "authorized release id"), + (&self.authorized_deployment_id, "authorized deployment id"), + (&self.authorized_by, "authorizer"), + ] { + identifier(value.as_deref().unwrap(), label)?; + } + let issued = self.authorization_issued_at_unix_millis.unwrap(); + let expires = self.authorization_expires_at_unix_millis.unwrap(); + if self.authorization_purpose.as_deref() != Some(IDUNN_DEPLOYMENT_RELEASE_PURPOSE) + || self.signature_algorithm.as_deref() != Some("ed25519") + || self.signature.as_ref().is_none_or(|v| v.len() != 64) + || issued < self.observed_at_unix_millis + || expires <= issued + || expires - issued > IDUNN_DEPLOYMENT_RELEASE_MAX_LIFETIME_MILLIS + || self + .expires_at_unix_millis + .is_some_and(|brake_expiry| expires > brake_expiry) + { + bail!("deployment release authorization purpose, signature, or lifetime is invalid"); + } + Ok(()) + } +} + +#[derive(Clone, Copy, Debug)] +pub enum IdunnDeploymentBrakeObservation<'a> { + Missing, + Corrupt, + Present(&'a IdunnDeploymentBrakeRecord), +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum IdunnDeploymentBrakeDenial { + Missing, + Corrupt, + Foreign, + WrongRuntime, + WrongScope, + Engaged, + Expired, + ReleaseMismatch, + DeploymentMismatch, + InvalidSignature, +} + +fn classify_idunn_deployment_brake( + observation: IdunnDeploymentBrakeObservation<'_>, + runtime_id: &str, + release_id: &str, + deployment_id: &str, + now_unix_millis: u64, +) -> std::result::Result<(), IdunnDeploymentBrakeDenial> { + let record = match observation { + IdunnDeploymentBrakeObservation::Missing => { + return Err(IdunnDeploymentBrakeDenial::Missing); + } + IdunnDeploymentBrakeObservation::Corrupt => { + return Err(IdunnDeploymentBrakeDenial::Corrupt); + } + IdunnDeploymentBrakeObservation::Present(record) => record, + }; + if record.validate().is_err() { + return Err(IdunnDeploymentBrakeDenial::Corrupt); + } + if record.brake_id != IDUNN_DEPLOYMENT_BRAKE_ID + || record.authority != IDUNN_DEPLOYMENT_BRAKE_AUTHORITY + { + return Err(IdunnDeploymentBrakeDenial::Foreign); + } + if record.runtime_id != runtime_id { + return Err(IdunnDeploymentBrakeDenial::WrongRuntime); + } + if !matches!(record.scope.as_str(), "all" | IDUNN_DEPLOYMENT_BRAKE_SCOPE) { + return Err(IdunnDeploymentBrakeDenial::WrongScope); + } + if record.status == "engaged" { + return Err(IdunnDeploymentBrakeDenial::Engaged); + } + if record + .expires_at_unix_millis + .is_some_and(|v| now_unix_millis >= v) + || record + .authorization_expires_at_unix_millis + .is_none_or(|v| now_unix_millis >= v) + || record + .authorization_issued_at_unix_millis + .is_none_or(|v| now_unix_millis < v) + { + return Err(IdunnDeploymentBrakeDenial::Expired); + } + if record.authorized_release_id.as_deref() != Some(release_id) { + return Err(IdunnDeploymentBrakeDenial::ReleaseMismatch); + } + if record.authorized_deployment_id.as_deref() != Some(deployment_id) { + return Err(IdunnDeploymentBrakeDenial::DeploymentMismatch); + } + Ok(()) +} + +pub fn evaluate_idunn_deployment_brake( + observation: IdunnDeploymentBrakeObservation<'_>, + operator_anchor: &ServiceIdentityTrustAnchor, + runtime_id: &str, + release_id: &str, + deployment_id: &str, + now_unix_millis: u64, +) -> std::result::Result<(), IdunnDeploymentBrakeDenial> { + classify_idunn_deployment_brake( + observation, + runtime_id, + release_id, + deployment_id, + now_unix_millis, + )?; + let IdunnDeploymentBrakeObservation::Present(record) = observation else { + unreachable!("successful classification always has a present record") + }; + verify_idunn_deployment_brake_authorization(record, operator_anchor) + .map_err(|_| IdunnDeploymentBrakeDenial::InvalidSignature) +} + +/// Verify the release grant using the dedicated operator public anchor. The +/// signed payload is the complete positional record with the signature field +/// empty, so authority, runtime, target IDs, and both time bounds are covered. +pub fn verify_idunn_deployment_brake_authorization( + record: &IdunnDeploymentBrakeRecord, + anchor: &ServiceIdentityTrustAnchor, +) -> Result<()> { + record.validate()?; + if record.status != "released" { + bail!("engaged deployment brake has no release authorization to verify"); + } + let identity_id = record + .authorized_by + .as_ref() + .ok_or_else(|| anyhow::anyhow!("deployment brake authorizer is absent"))?; + let signature = record + .signature + .as_ref() + .ok_or_else(|| anyhow::anyhow!("deployment brake signature is absent"))?; + let mut unsigned = record.clone(); + unsigned.signature = None; + verify_service_identity_signature::< + IdunnDeploymentBrakeOperatorIdentity, + IdunnDeploymentBrakeReleasePurpose, + >( + anchor, + &rmp_serde::to_vec(&unsigned)?, + &ServiceIdentitySignature { + identity_id: identity_id.clone(), + signature: signature.clone(), + }, + ) +} + +fn identifier(value: &str, label: &str) -> Result<()> { + if value.trim().is_empty() || value.len() > 256 || value.chars().any(char::is_control) { + bail!("{label} is empty, oversized, or contains control characters"); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::enroll_service_identity_at; + + fn released() -> IdunnDeploymentBrakeRecord { + IdunnDeploymentBrakeRecord { + schema_version: IDUNN_DEPLOYMENT_BRAKE_SCHEMA.into(), + brake_id: IDUNN_DEPLOYMENT_BRAKE_ID.into(), + authority: IDUNN_DEPLOYMENT_BRAKE_AUTHORITY.into(), + runtime_id: "yggdrasil".into(), + status: "released".into(), + scope: "deployment".into(), + reason: "operator authorized one attempt".into(), + observed_at_unix_millis: 100, + expires_at_unix_millis: Some(1000), + authorization_id: Some("auth/r4".into()), + authorization_purpose: Some(IDUNN_DEPLOYMENT_RELEASE_PURPOSE.into()), + authorized_release_id: Some("release-4".into()), + authorized_deployment_id: Some("deployment-4".into()), + authorized_by: Some("operator/metacrat".into()), + authorization_issued_at_unix_millis: Some(100), + authorization_expires_at_unix_millis: Some(900), + signature_algorithm: Some("ed25519".into()), + signature: Some(vec![7; 64]), + private_state_exposed: false, + updated_by: "operator/metacrat".into(), + } + } + + #[test] + fn only_exact_bounded_release_is_permitted() { + let value = released(); + assert_eq!( + classify_idunn_deployment_brake( + IdunnDeploymentBrakeObservation::Present(&value), + "yggdrasil", + "release-4", + "deployment-4", + 500 + ), + Ok(()) + ); + assert_eq!( + classify_idunn_deployment_brake( + IdunnDeploymentBrakeObservation::Missing, + "yggdrasil", + "release-4", + "deployment-4", + 500 + ), + Err(IdunnDeploymentBrakeDenial::Missing) + ); + assert_eq!( + classify_idunn_deployment_brake( + IdunnDeploymentBrakeObservation::Corrupt, + "yggdrasil", + "release-4", + "deployment-4", + 500 + ), + Err(IdunnDeploymentBrakeDenial::Corrupt) + ); + assert_eq!( + classify_idunn_deployment_brake( + IdunnDeploymentBrakeObservation::Present(&value), + "yggdrasil", + "other", + "deployment-4", + 500 + ), + Err(IdunnDeploymentBrakeDenial::ReleaseMismatch) + ); + assert_eq!( + classify_idunn_deployment_brake( + IdunnDeploymentBrakeObservation::Present(&value), + "yggdrasil", + "release-4", + "other", + 500 + ), + Err(IdunnDeploymentBrakeDenial::DeploymentMismatch) + ); + assert_eq!( + classify_idunn_deployment_brake( + IdunnDeploymentBrakeObservation::Present(&value), + "yggdrasil", + "release-4", + "deployment-4", + 900 + ), + Err(IdunnDeploymentBrakeDenial::Expired) + ); + } + + #[test] + fn hostile_authority_scope_and_shape_substitutions_fail_closed() { + let mut value = released(); + value.authority = "idunn".into(); + assert_eq!( + classify_idunn_deployment_brake( + IdunnDeploymentBrakeObservation::Present(&value), + "yggdrasil", + "release-4", + "deployment-4", + 500 + ), + Err(IdunnDeploymentBrakeDenial::Foreign) + ); + let mut value = released(); + value.runtime_id = "nightwing".into(); + assert_eq!( + classify_idunn_deployment_brake( + IdunnDeploymentBrakeObservation::Present(&value), + "yggdrasil", + "release-4", + "deployment-4", + 500 + ), + Err(IdunnDeploymentBrakeDenial::WrongRuntime) + ); + let mut value = released(); + value.scope = "persona.public_speech".into(); + assert_eq!( + classify_idunn_deployment_brake( + IdunnDeploymentBrakeObservation::Present(&value), + "yggdrasil", + "release-4", + "deployment-4", + 500 + ), + Err(IdunnDeploymentBrakeDenial::WrongScope) + ); + let mut value = released(); + value.signature = Some(vec![0; 63]); + assert_eq!( + classify_idunn_deployment_brake( + IdunnDeploymentBrakeObservation::Present(&value), + "yggdrasil", + "release-4", + "deployment-4", + 500 + ), + Err(IdunnDeploymentBrakeDenial::Corrupt) + ); + let mut value = released(); + value.authorization_expires_at_unix_millis = + Some(100 + IDUNN_DEPLOYMENT_RELEASE_MAX_LIFETIME_MILLIS + 1); + assert!(value.validate().is_err()); + } + + #[test] + fn engaged_record_cannot_smuggle_authorization() { + let mut value = released(); + value.status = "engaged".into(); + assert!(value.validate().is_err()); + value.authorization_id = None; + value.authorization_purpose = None; + value.authorized_release_id = None; + value.authorized_deployment_id = None; + value.authorized_by = None; + value.authorization_issued_at_unix_millis = None; + value.authorization_expires_at_unix_millis = None; + value.signature_algorithm = None; + value.signature = None; + value.validate().unwrap(); + assert_eq!( + classify_idunn_deployment_brake( + IdunnDeploymentBrakeObservation::Present(&value), + "yggdrasil", + "release-4", + "deployment-4", + 500 + ), + Err(IdunnDeploymentBrakeDenial::Engaged) + ); + } + + #[test] + fn evaluation_requires_the_dedicated_operator_signature() -> Result<()> { + let temp = tempfile::tempdir()?; + let signer = enroll_service_identity_at::( + &temp.path().join("operator.cc"), + )?; + let anchor = signer.trust_anchor()?; + let mut value = released(); + value.authorized_by = Some(anchor.identity_id.clone()); + value.signature = None; + let proof = signer.sign::(&rmp_serde::to_vec(&value)?); + value.signature = Some(proof.signature); + assert_eq!( + evaluate_idunn_deployment_brake( + IdunnDeploymentBrakeObservation::Present(&value), + &anchor, + "yggdrasil", + "release-4", + "deployment-4", + 500, + ), + Ok(()) + ); + value.reason = "tampered after authorization".into(); + assert_eq!( + evaluate_idunn_deployment_brake( + IdunnDeploymentBrakeObservation::Present(&value), + &anchor, + "yggdrasil", + "release-4", + "deployment-4", + 500, + ), + Err(IdunnDeploymentBrakeDenial::InvalidSignature) + ); + Ok(()) + } +} diff --git a/packages/cultnet-rs/src/idunn_lifecycle_brake.rs b/packages/cultnet-rs/src/idunn_lifecycle_brake.rs new file mode 100644 index 00000000..86eefa64 --- /dev/null +++ b/packages/cultnet-rs/src/idunn_lifecycle_brake.rs @@ -0,0 +1,325 @@ +use anyhow::{Result, bail}; +use cultcache_rs::DatabaseEntry; + +pub const IDUNN_LIFECYCLE_BRAKE_SCHEMA: &str = "idunn.lifecycle_brake.v1"; +pub const IDUNN_LIFECYCLE_BRAKE_TYPE: &str = "idunn.lifecycle_brake"; +pub const IDUNN_LIFECYCLE_BRAKE_AUTHORITY: &str = "idunn.root"; +pub const IDUNN_LIFECYCLE_BRAKE_SCOPE: &str = "continuity-restart"; + +/// Root-owned, per-target continuity brake. This document has no authority over +/// Idunn startup, deployment, route admission, or any target other than the one +/// named in `target`. +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache(type = "idunn.lifecycle_brake", schema = "idunn.lifecycle_brake.v1")] +pub struct IdunnLifecycleBrakeRecord { + #[cultcache(key = 0)] + pub schema_version: String, + #[cultcache(key = 1)] + pub authority: String, + #[cultcache(key = 2)] + pub runtime_id: String, + #[cultcache(key = 3)] + pub target: String, + #[cultcache(key = 4)] + pub scope: String, + #[cultcache(key = 5)] + pub status: String, + #[cultcache(key = 6)] + pub reason: String, + #[cultcache(key = 7)] + pub updated_at_unix_millis: u64, + #[cultcache(key = 8)] + pub released_until_unix_millis: Option, +} + +impl IdunnLifecycleBrakeRecord { + pub fn validate(&self) -> Result<()> { + if self.schema_version != IDUNN_LIFECYCLE_BRAKE_SCHEMA { + bail!("lifecycle brake schema is unsupported"); + } + identifier(&self.authority, "authority")?; + identifier(&self.runtime_id, "runtime id")?; + identifier(&self.target, "target")?; + identifier(&self.scope, "scope")?; + identifier(&self.reason, "reason")?; + if !matches!(self.status.as_str(), "engaged" | "released") + || self.updated_at_unix_millis == 0 + { + bail!("lifecycle brake status or update time is invalid"); + } + if self.status == "engaged" && self.released_until_unix_millis.is_some() { + bail!("engaged lifecycle brake cannot carry a release expiry"); + } + if self + .released_until_unix_millis + .is_some_and(|expires| expires <= self.updated_at_unix_millis) + { + bail!("lifecycle brake release expiry is not after its update"); + } + Ok(()) + } + + pub fn canonical_bytes(&self) -> Result> { + self.validate()?; + Ok(rmp_serde::to_vec(self)?) + } + + pub fn decode_canonical(payload: &[u8]) -> Result { + if messagepack_array_len(payload) != Some(9) { + bail!("lifecycle brake is not the 9-field positional contract"); + } + let value: Self = rmp_serde::from_slice(payload)?; + value.validate()?; + if rmp_serde::to_vec(&value)? != payload { + bail!("lifecycle brake encoding is not canonical"); + } + Ok(value) + } +} + +#[derive(Clone, Copy, Debug)] +pub enum IdunnLifecycleBrakeObservation<'a> { + Missing, + Corrupt, + Present(&'a IdunnLifecycleBrakeRecord), +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum IdunnLifecycleBrakeDenial { + Corrupt, + Foreign, + WrongRuntime, + WrongTarget, + Engaged, + Expired, +} + +/// Decide only whether Idunn may restart one already-admitted target to keep it +/// alive. Absence is the default allow state; a present record must bind the +/// exact Idunn runtime and target and must be explicitly released. +pub fn evaluate_idunn_continuity_restart( + observation: IdunnLifecycleBrakeObservation<'_>, + runtime_id: &str, + target: &str, + now_unix_millis: u64, +) -> std::result::Result<(), IdunnLifecycleBrakeDenial> { + let record = match observation { + IdunnLifecycleBrakeObservation::Missing => return Ok(()), + IdunnLifecycleBrakeObservation::Corrupt => { + return Err(IdunnLifecycleBrakeDenial::Corrupt); + } + IdunnLifecycleBrakeObservation::Present(record) => record, + }; + if record.validate().is_err() { + return Err(IdunnLifecycleBrakeDenial::Corrupt); + } + if record.authority != IDUNN_LIFECYCLE_BRAKE_AUTHORITY + || record.scope != IDUNN_LIFECYCLE_BRAKE_SCOPE + { + return Err(IdunnLifecycleBrakeDenial::Foreign); + } + if record.runtime_id != runtime_id { + return Err(IdunnLifecycleBrakeDenial::WrongRuntime); + } + if record.target != target { + return Err(IdunnLifecycleBrakeDenial::WrongTarget); + } + if record.status == "engaged" { + return Err(IdunnLifecycleBrakeDenial::Engaged); + } + if record + .released_until_unix_millis + .is_some_and(|expires| now_unix_millis >= expires) + { + return Err(IdunnLifecycleBrakeDenial::Expired); + } + Ok(()) +} + +fn identifier(value: &str, label: &str) -> Result<()> { + if value.trim().is_empty() || value.len() > 256 || value.chars().any(char::is_control) { + bail!("{label} is empty, oversized, or contains control characters"); + } + Ok(()) +} + +fn messagepack_array_len(payload: &[u8]) -> Option { + match *payload.first()? { + marker @ 0x90..=0x9f => Some((marker & 0x0f) as usize), + 0xdc if payload.len() >= 3 => Some(u16::from_be_bytes([payload[1], payload[2]]) as usize), + 0xdd if payload.len() >= 5 => { + Some(u32::from_be_bytes([payload[1], payload[2], payload[3], payload[4]]) as usize) + } + _ => None, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn released() -> IdunnLifecycleBrakeRecord { + IdunnLifecycleBrakeRecord { + schema_version: IDUNN_LIFECYCLE_BRAKE_SCHEMA.into(), + authority: IDUNN_LIFECYCLE_BRAKE_AUTHORITY.into(), + runtime_id: "idunn-yggdrasil".into(), + target: "ghostlight".into(), + scope: IDUNN_LIFECYCLE_BRAKE_SCOPE.into(), + status: "released".into(), + reason: "continuity restart permitted".into(), + updated_at_unix_millis: 100, + released_until_unix_millis: None, + } + } + + fn evaluate( + observation: IdunnLifecycleBrakeObservation<'_>, + target: &str, + now: u64, + ) -> std::result::Result<(), IdunnLifecycleBrakeDenial> { + evaluate_idunn_continuity_restart(observation, "idunn-yggdrasil", target, now) + } + + #[test] + fn lifecycle_brake_round_trips_only_as_the_canonical_positional_contract() -> Result<()> { + let value = released(); + let encoded = value.canonical_bytes()?; + assert_eq!(encoded[0], 0x99); + assert_eq!( + IdunnLifecycleBrakeRecord::decode_canonical(&encoded)?, + value + ); + + let mut noncanonical = vec![0xdc, 0, 9]; + noncanonical.extend_from_slice(&encoded[1..]); + assert!(IdunnLifecycleBrakeRecord::decode_canonical(&noncanonical).is_err()); + + let mut extended = encoded; + extended[0] = 0x9a; + extended.push(0xc0); + assert!(IdunnLifecycleBrakeRecord::decode_canonical(&extended).is_err()); + Ok(()) + } + + #[test] + fn absence_and_explicit_release_allow_only_the_named_continuity_restart() { + assert_eq!( + evaluate(IdunnLifecycleBrakeObservation::Missing, "ghostlight", 500), + Ok(()) + ); + + let mut value = released(); + assert_eq!( + evaluate( + IdunnLifecycleBrakeObservation::Present(&value), + "ghostlight", + 500 + ), + Ok(()) + ); + value.released_until_unix_millis = Some(900); + assert_eq!( + evaluate( + IdunnLifecycleBrakeObservation::Present(&value), + "ghostlight", + 899 + ), + Ok(()) + ); + assert_eq!( + evaluate( + IdunnLifecycleBrakeObservation::Present(&value), + "ghostlight", + 900 + ), + Err(IdunnLifecycleBrakeDenial::Expired) + ); + assert_eq!( + evaluate(IdunnLifecycleBrakeObservation::Missing, "odin", 900), + Ok(()) + ); + assert_eq!( + evaluate(IdunnLifecycleBrakeObservation::Present(&value), "odin", 500), + Err(IdunnLifecycleBrakeDenial::WrongTarget) + ); + } + + #[test] + fn corrupt_foreign_wrong_runtime_and_wrong_target_records_deny() { + assert_eq!( + evaluate(IdunnLifecycleBrakeObservation::Corrupt, "ghostlight", 500), + Err(IdunnLifecycleBrakeDenial::Corrupt) + ); + + let mut value = released(); + value.status = "paused".into(); + assert_eq!( + evaluate( + IdunnLifecycleBrakeObservation::Present(&value), + "ghostlight", + 500 + ), + Err(IdunnLifecycleBrakeDenial::Corrupt) + ); + + let mut value = released(); + value.authority = "service-owned".into(); + assert_eq!( + evaluate( + IdunnLifecycleBrakeObservation::Present(&value), + "ghostlight", + 500 + ), + Err(IdunnLifecycleBrakeDenial::Foreign) + ); + + let mut value = released(); + value.scope = "deployment".into(); + assert_eq!( + evaluate( + IdunnLifecycleBrakeObservation::Present(&value), + "ghostlight", + 500 + ), + Err(IdunnLifecycleBrakeDenial::Foreign) + ); + + let value = released(); + assert_eq!( + evaluate_idunn_continuity_restart( + IdunnLifecycleBrakeObservation::Present(&value), + "idunn-nightwing", + "ghostlight", + 500 + ), + Err(IdunnLifecycleBrakeDenial::WrongRuntime) + ); + assert_eq!( + evaluate(IdunnLifecycleBrakeObservation::Present(&value), "odin", 500), + Err(IdunnLifecycleBrakeDenial::WrongTarget) + ); + } + + #[test] + fn engaged_always_denies_and_cannot_smuggle_a_release_expiry() { + let mut value = released(); + value.status = "engaged".into(); + assert_eq!( + evaluate( + IdunnLifecycleBrakeObservation::Present(&value), + "ghostlight", + 500 + ), + Err(IdunnLifecycleBrakeDenial::Engaged) + ); + value.released_until_unix_millis = Some(900); + assert_eq!( + evaluate( + IdunnLifecycleBrakeObservation::Present(&value), + "ghostlight", + 500 + ), + Err(IdunnLifecycleBrakeDenial::Corrupt) + ); + } +} diff --git a/packages/cultnet-rs/src/lib.rs b/packages/cultnet-rs/src/lib.rs index 6fb1d38d..074c2a29 100644 --- a/packages/cultnet-rs/src/lib.rs +++ b/packages/cultnet-rs/src/lib.rs @@ -1,4 +1,12 @@ mod contracts; +mod idunn_deployment_brake; +mod idunn_lifecycle_brake; +mod media_stream_contracts; +mod media_stream_wire; +mod provider_health_contracts; +mod runtime_authority_contracts; +mod service_identity; +mod snapshot_query; mod cultmesh; mod framing; mod operation_service; @@ -11,6 +19,14 @@ mod shard_catalog; mod transport; pub use contracts::*; +pub use idunn_deployment_brake::*; +pub use idunn_lifecycle_brake::*; +pub use media_stream_contracts::*; +pub use media_stream_wire::*; +pub use provider_health_contracts::*; +pub use runtime_authority_contracts::*; +pub use service_identity::*; +pub use snapshot_query::*; pub use cultmesh::*; pub use framing::*; pub use operation_service::*; diff --git a/packages/cultnet-rs/src/media_stream_contracts.rs b/packages/cultnet-rs/src/media_stream_contracts.rs new file mode 100644 index 00000000..5704912e --- /dev/null +++ b/packages/cultnet-rs/src/media_stream_contracts.rs @@ -0,0 +1,578 @@ +//! CultMesh Media Stream: the typed contract for carrying live audio and video +//! over CultNet. +//! +//! These records describe a media stream and nothing about who produced it. A +//! capture broker, an editor runtime, a recorder, or a synthetic source may all +//! publish them; an OBS bridge, a compositor, a recorder, or a browser lowering +//! may all consume them. Producers advertise their streams through Odin and +//! consumers resolve them there, so neither end needs to name the other. +//! +//! They lived in `odin-core` as `muninn.media_*`, named after the one producer +//! that happened to emit them first. Every field here is producer-agnostic — the +//! name was the only coupling, and a second producer would have had to +//! impersonate Muninn or fork the shape. +//! +//! # Layout is wire contract +//! +//! CultCache encodes these positionally. Field order and `key` indices are the +//! wire format: append new fields with `default` at the next free index, never +//! reorder or renumber. `missing_video_chunk_keys` shows the pattern. +//! +//! Payloads carry `bytes` so they serialize as MessagePack `bin` rather than an +//! array of integers, which is what the C#, TypeScript and Python runtimes +//! expect and what keeps a 848-byte payload at 851 bytes instead of 1234. Note +//! that this is the derive's own attribute — `#[serde(with = "serde_bytes")]` +//! is inert here, because `DatabaseEntry` writes its own `Serialize`. +//! +//! # Timebase +//! +//! `pts_ticks`, `duration_ticks` and `deadline_ticks` are expressed in +//! `timebase_num / timebase_den` units, carried per record so a consumer never +//! has to assume a producer's clock. `deadline_ticks` is the presentation +//! deadline past which a late arrival is worth dropping rather than displaying. +//! +//! # Reliability +//! +//! Nothing here specifies a delivery guarantee. Whether a stream rides a +//! reliable or lossy CultNet channel, and whether it carries parity, is a +//! transport decision made per stream — not a property of the payload shape. +//! Video parity shards exist for producers that choose forward error correction; +//! audio currently has no parity record, and a producer emitting audio on a +//! lossy channel needs one. + +use cultcache_rs::DatabaseEntry; + +pub const GAMECULT_MEDIA_VIDEO_ACCESS_UNIT_SCHEMA: &str = "gamecult.media_video_access_unit.v1"; +pub const GAMECULT_MEDIA_VIDEO_PARITY_SHARD_SCHEMA: &str = "gamecult.media_video_parity_shard.v2"; +pub const GAMECULT_MEDIA_AUDIO_PACKET_SCHEMA: &str = "gamecult.media_audio_packet.v1"; +pub const GAMECULT_MEDIA_RECEIVER_FEEDBACK_SCHEMA: &str = "gamecult.media_receiver_feedback.v1"; +pub const GAMECULT_MEDIA_STREAM_ADVERTISEMENT_SCHEMA: &str = + "gamecult.media_stream_advertisement.v1"; +pub const GAMECULT_MEDIA_STREAM_REQUEST_SCHEMA: &str = "gamecult.media_stream_request.v1"; + +/// A media stream a producer can serve on request, as it appears in Odin. +/// +/// Odin admits these without any registration: it derives the document type +/// from the schema id (`prefix.vN` -> `prefix`) and refuses only a short list +/// of authority-owned types (`gamecult.runtime_presence_health`, +/// `odin.runtime_topology_correlation`, `idunn.expected_incarnation`, +/// `idunn.runtime_activation`, `idunn.process_write_lease`, +/// `gamecult.service_trust_anchor`). A new record whose type collided with one +/// of those would be rejected at the catalog with no mention of the collision. +/// +/// This is the record a consumer's picker is built from: which streams exist, +/// which video and audio sources each can capture, which codecs it can encode, +/// the defaults it would use, and where it serves the stream from. It is state, not media — it travels as a +/// CultMesh document through Odin, never on the media channel — and it names no +/// consumer. `producer_id` is the runtime that answers requests for it. +/// +/// Keyed by `stream_id`. +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache( + type = "gamecult.media_stream_advertisement", + schema = "gamecult.media_stream_advertisement.v1" +)] +pub struct GameCultMediaStreamAdvertisementRecord { + #[cultcache(key = 0)] + pub stream_id: String, + #[cultcache(key = 1)] + pub producer_id: String, + #[cultcache(key = 2)] + pub label: String, + /// `available`, `streaming`, or `unavailable`. + #[cultcache(key = 3)] + pub state: String, + #[cultcache(key = 4)] + pub video_source_ids: Vec, + #[cultcache(key = 5)] + pub video_source_labels: Vec, + #[cultcache(key = 6)] + pub audio_source_ids: Vec, + #[cultcache(key = 7)] + pub audio_source_labels: Vec, + /// Codecs the producer can encode video as, e.g. `h264`. + #[cultcache(key = 8)] + pub video_codecs: Vec, + /// Codecs the producer can emit audio as, e.g. `pcm-f32le-interleaved`. + #[cultcache(key = 9)] + pub audio_codecs: Vec, + #[cultcache(key = 10)] + pub audio_sample_rate: u32, + #[cultcache(key = 11)] + pub audio_channels: u32, + #[cultcache(key = 12)] + pub default_video_bitrate_kbps: u32, + #[cultcache(key = 13)] + pub default_latency_budget_ms: u32, + #[cultcache(key = 14)] + pub media_packet_bytes: u32, + /// `host:port` the producer serves this stream from. A consumer dials it + /// with `media_connection_id` once its request is answered `running`; + /// the producer listens. The consumer's host admits nothing inbound, + /// which is what lets a viewer install a plugin without opening a port. + #[cultcache(key = 15)] + pub media_endpoint: String, + /// The CultNet RUDP connection id a consumer dials `media_endpoint` with. + #[cultcache(key = 16)] + pub media_connection_id: u32, + #[cultcache(key = 17)] + pub updated_at: String, +} + +/// A consumer asking a producer to start or stop serving a stream to it. +/// +/// The producer answers by rewriting `state` and `detail` on the same record +/// key, so a consumer watches one document for the outcome. A newer request for +/// the same `stream_id` and `receiver_id` supersedes an older one. +/// +/// It carries no consumer endpoint. The consumer dials the advertised +/// `media_endpoint` and identifies itself on the wire by `receiver_id`. +/// +/// Keyed by `media_stream_request_key(stream_id, receiver_id)`. +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache( + type = "gamecult.media_stream_request", + schema = "gamecult.media_stream_request.v1" +)] +pub struct GameCultMediaStreamRequestRecord { + #[cultcache(key = 0)] + pub request_id: String, + #[cultcache(key = 1)] + pub stream_id: String, + #[cultcache(key = 2)] + pub producer_id: String, + #[cultcache(key = 3)] + pub receiver_id: String, + /// `start` or `stop`. + #[cultcache(key = 4)] + pub action: String, + /// `pending`, `running`, `stopped`, or `failed`; the producer owns it after + /// `pending`. + #[cultcache(key = 5)] + pub state: String, + /// Empty means no video. + #[cultcache(key = 6)] + pub video_source_id: String, + /// Empty means no audio. + #[cultcache(key = 7)] + pub audio_source_id: String, + #[cultcache(key = 8)] + pub video_codec: String, + #[cultcache(key = 9)] + pub audio_codec: String, + #[cultcache(key = 10)] + pub video_bitrate_kbps: u32, + #[cultcache(key = 11)] + pub latency_budget_ms: u32, + #[cultcache(key = 12)] + pub media_packet_bytes: u32, + #[cultcache(key = 13)] + pub detail: String, + #[cultcache(key = 14)] + pub updated_at: String, +} + +pub const MEDIA_STREAM_REQUEST_ACTIONS: [&str; 2] = ["start", "stop"]; +pub const MEDIA_STREAM_REQUEST_STATES: [&str; 4] = ["pending", "running", "stopped", "failed"]; +pub const MEDIA_STREAM_ADVERTISEMENT_STATES: [&str; 3] = ["available", "streaming", "unavailable"]; + +/// One live request per stream per receiver: a newer one replaces the older. +pub fn media_stream_request_key(stream_id: &str, receiver_id: &str) -> String { + format!("{stream_id}:{receiver_id}") +} + +pub fn validate_media_stream_advertisement( + record: &GameCultMediaStreamAdvertisementRecord, +) -> anyhow::Result<()> { + use anyhow::anyhow; + for (name, value) in [ + ("stream_id", &record.stream_id), + ("producer_id", &record.producer_id), + ("label", &record.label), + ("updated_at", &record.updated_at), + ] { + if value.is_empty() { + return Err(anyhow!("media stream advertisement {name} must be non-empty")); + } + } + if !MEDIA_STREAM_ADVERTISEMENT_STATES.contains(&record.state.as_str()) { + return Err(anyhow!( + "media stream advertisement state {:?} is not one of {:?}", + record.state, + MEDIA_STREAM_ADVERTISEMENT_STATES + )); + } + if record.video_source_ids.len() != record.video_source_labels.len() { + return Err(anyhow!( + "media stream advertisement has {} video source ids and {} labels", + record.video_source_ids.len(), + record.video_source_labels.len() + )); + } + if record.audio_source_ids.len() != record.audio_source_labels.len() { + return Err(anyhow!( + "media stream advertisement has {} audio source ids and {} labels", + record.audio_source_ids.len(), + record.audio_source_labels.len() + )); + } + if !record.audio_source_ids.is_empty() && (record.audio_sample_rate == 0 || record.audio_channels == 0) { + return Err(anyhow!( + "media stream advertisement offers audio without a sample rate and channel count" + )); + } + if record.media_connection_id == 0 { + return Err(anyhow!("media stream advertisement media_connection_id must be non-zero")); + } + if !is_host_port(&record.media_endpoint) { + return Err(anyhow!( + "media stream advertisement media_endpoint {:?} must be host:port", + record.media_endpoint + )); + } + Ok(()) +} + +pub fn validate_media_stream_request( + record: &GameCultMediaStreamRequestRecord, +) -> anyhow::Result<()> { + use anyhow::anyhow; + for (name, value) in [ + ("request_id", &record.request_id), + ("stream_id", &record.stream_id), + ("producer_id", &record.producer_id), + ("receiver_id", &record.receiver_id), + ("updated_at", &record.updated_at), + ] { + if value.is_empty() { + return Err(anyhow!("media stream request {name} must be non-empty")); + } + } + if !MEDIA_STREAM_REQUEST_ACTIONS.contains(&record.action.as_str()) { + return Err(anyhow!( + "media stream request action {:?} is not one of {:?}", + record.action, + MEDIA_STREAM_REQUEST_ACTIONS + )); + } + if !MEDIA_STREAM_REQUEST_STATES.contains(&record.state.as_str()) { + return Err(anyhow!( + "media stream request state {:?} is not one of {:?}", + record.state, + MEDIA_STREAM_REQUEST_STATES + )); + } + if record.action == "start" { + if record.video_source_id.is_empty() && record.audio_source_id.is_empty() { + return Err(anyhow!("media stream request asks for neither video nor audio")); + } + if !record.video_source_id.is_empty() && record.video_codec.is_empty() { + return Err(anyhow!("media stream request names a video source without a codec")); + } + if !record.audio_source_id.is_empty() && record.audio_codec.is_empty() { + return Err(anyhow!("media stream request names an audio source without a codec")); + } + } + Ok(()) +} + +/// A hostname or IP and a non-zero port. Not `SocketAddr`: a producer may +/// advertise a name the consumer resolves. +fn is_host_port(value: &str) -> bool { + let Some((host, port)) = value.rsplit_once(':') else { + return false; + }; + let host = host.trim_start_matches('[').trim_end_matches(']'); + !host.is_empty() && port.parse::().is_ok_and(|port| port != 0) +} + +#[cfg(test)] +mod media_stream_catalog_tests { + use super::*; + + fn advertisement() -> GameCultMediaStreamAdvertisementRecord { + GameCultMediaStreamAdvertisementRecord { + stream_id: "muninn.raven.av.rudp".into(), + producer_id: "raven".into(), + label: "Raven desktop".into(), + state: "available".into(), + video_source_ids: vec!["display:0".into()], + video_source_labels: vec!["Display 1".into()], + audio_source_ids: vec!["wasapi-loopback:Realtek".into()], + audio_source_labels: vec!["Realtek loopback".into()], + video_codecs: vec!["h264".into()], + audio_codecs: vec!["pcm-f32le-interleaved".into()], + audio_sample_rate: 48_000, + audio_channels: 2, + default_video_bitrate_kbps: 12_000, + default_latency_budget_ms: 250, + media_packet_bytes: 848, + media_endpoint: "192.168.178.20:5220".into(), + media_connection_id: 0x6d75_0001, + updated_at: "2026-09-10T00:00:00Z".into(), + } + } + + fn request() -> GameCultMediaStreamRequestRecord { + GameCultMediaStreamRequestRecord { + request_id: "req-1".into(), + stream_id: "muninn.raven.av.rudp".into(), + producer_id: "raven".into(), + receiver_id: "starfire.obs".into(), + action: "start".into(), + state: "pending".into(), + video_source_id: "display:0".into(), + audio_source_id: "wasapi-loopback:Realtek".into(), + video_codec: "h264".into(), + audio_codec: "pcm-f32le-interleaved".into(), + video_bitrate_kbps: 12_000, + latency_budget_ms: 250, + media_packet_bytes: 848, + detail: String::new(), + updated_at: "2026-09-10T00:00:00Z".into(), + } + } + + #[test] + fn a_complete_advertisement_and_request_validate() -> anyhow::Result<()> { + validate_media_stream_advertisement(&advertisement())?; + validate_media_stream_request(&request())?; + Ok(()) + } + + #[test] + fn source_ids_and_labels_must_pair_up() { + let mut record = advertisement(); + record.video_source_labels.clear(); + let error = validate_media_stream_advertisement(&record).unwrap_err(); + assert!(error.to_string().contains("video source ids"), "{error}"); + } + + #[test] + fn audio_on_offer_needs_a_sample_rate_and_channels() { + let mut record = advertisement(); + record.audio_channels = 0; + assert!(validate_media_stream_advertisement(&record).is_err()); + record.audio_source_ids.clear(); + record.audio_source_labels.clear(); + assert!(validate_media_stream_advertisement(&record).is_ok(), "no audio, no requirement"); + } + + /// The producer is the listener, so it must say where. + #[test] + fn an_advertisement_says_where_the_stream_is_served() { + for bad in ["", "raven", "raven:", "raven:0", ":5220"] { + let mut record = advertisement(); + record.media_endpoint = bad.into(); + assert!(validate_media_stream_advertisement(&record).unwrap_err().to_string().contains("host:port"), "{bad:?}"); + } + for good in ["raven:5220", "10.77.0.2:5220", "[::1]:5220"] { + let mut record = advertisement(); + record.media_endpoint = good.into(); + assert!(validate_media_stream_advertisement(&record).is_ok(), "{good:?}"); + } + } + + #[test] + fn a_start_needs_something_to_send() { + let mut record = request(); + record.video_source_id.clear(); + record.audio_source_id.clear(); + assert!(validate_media_stream_request(&record).unwrap_err().to_string().contains("neither")); + let mut record = request(); + record.video_codec.clear(); + assert!(validate_media_stream_request(&record).unwrap_err().to_string().contains("codec")); + } + + #[test] + fn a_stop_needs_no_sources() -> anyhow::Result<()> { + let mut record = request(); + record.action = "stop".into(); + record.video_source_id.clear(); + record.audio_source_id.clear(); + validate_media_stream_request(&record) + } + + #[test] + fn unknown_actions_and_states_are_refused() { + let mut record = request(); + record.action = "pause".into(); + assert!(validate_media_stream_request(&record).is_err()); + let mut record = request(); + record.state = "done".into(); + assert!(validate_media_stream_request(&record).is_err()); + } + + #[test] + fn one_request_per_stream_per_receiver() { + assert_eq!(media_stream_request_key("s", "starfire.obs"), "s:starfire.obs"); + } + + /// The records round-trip through the same positional encoding every + /// other runtime reads, so a Python or TypeScript consumer sees the same + /// slots. + #[test] + fn records_round_trip_positionally() -> anyhow::Result<()> { + let record = advertisement(); + let encoded = rmp_serde::to_vec(&record)?; + let decoded: GameCultMediaStreamAdvertisementRecord = rmp_serde::from_slice(&encoded)?; + assert_eq!(decoded, record); + let record = request(); + let encoded = rmp_serde::to_vec(&record)?; + let decoded: GameCultMediaStreamRequestRecord = rmp_serde::from_slice(&encoded)?; + assert_eq!(decoded, record); + Ok(()) + } +} + +/// One coded video access unit, possibly split across `chunk_count` records +/// sharing a `frame_id`. A consumer reassembles by `(stream_id, session_id, +/// frame_id)` and orders by `chunk_index`. +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache( + type = "gamecult.media_video_access_unit", + schema = "gamecult.media_video_access_unit.v1" +)] +pub struct GameCultMediaVideoAccessUnitRecord { + #[cultcache(key = 0)] + pub stream_id: String, + #[cultcache(key = 1)] + pub session_id: String, + #[cultcache(key = 2)] + pub frame_id: u64, + #[cultcache(key = 3)] + pub codec: String, + #[cultcache(key = 4)] + pub pts_ticks: i64, + #[cultcache(key = 5)] + pub duration_ticks: u32, + #[cultcache(key = 6)] + pub timebase_num: u32, + #[cultcache(key = 7)] + pub timebase_den: u32, + #[cultcache(key = 8)] + pub keyframe: bool, + #[cultcache(key = 9)] + pub dependency_frame_id: Option, + #[cultcache(key = 10)] + pub deadline_ticks: i64, + #[cultcache(key = 11)] + pub chunk_index: u16, + #[cultcache(key = 12)] + pub chunk_count: u16, + #[cultcache(key = 13, bytes)] + pub payload: Vec, +} + +/// Forward-error-correction parity for one video access unit. Carries the +/// framing fields of the access unit it protects so a receiver can recover +/// without having seen the original, plus the shard geometry needed to run +/// recovery: `parity_index` within `parity_count`, and the chunk sizes the +/// parity was computed over. +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache( + type = "gamecult.media_video_parity_shard", + schema = "gamecult.media_video_parity_shard.v2" +)] +pub struct GameCultMediaVideoParityShardRecord { + #[cultcache(key = 0)] + pub stream_id: String, + #[cultcache(key = 1)] + pub session_id: String, + #[cultcache(key = 2)] + pub frame_id: u64, + #[cultcache(key = 3)] + pub codec: String, + #[cultcache(key = 4)] + pub pts_ticks: i64, + #[cultcache(key = 5)] + pub duration_ticks: u32, + #[cultcache(key = 6)] + pub timebase_num: u32, + #[cultcache(key = 7)] + pub timebase_den: u32, + #[cultcache(key = 8)] + pub keyframe: bool, + #[cultcache(key = 9)] + pub dependency_frame_id: Option, + #[cultcache(key = 10)] + pub deadline_ticks: i64, + #[cultcache(key = 11)] + pub chunk_count: u16, + #[cultcache(key = 12)] + pub parity_index: u16, + #[cultcache(key = 13)] + pub parity_count: u16, + #[cultcache(key = 14)] + pub chunk_payload_bytes: u32, + #[cultcache(key = 15)] + pub last_chunk_payload_bytes: u32, + #[cultcache(key = 16, bytes)] + pub payload: Vec, +} + +/// One coded audio packet. Unlike video these are not chunked: a packet is +/// whole or it is absent, and `payload` carries whatever `codec` names. +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache( + type = "gamecult.media_audio_packet", + schema = "gamecult.media_audio_packet.v1" +)] +pub struct GameCultMediaAudioPacketRecord { + #[cultcache(key = 0)] + pub stream_id: String, + #[cultcache(key = 1)] + pub session_id: String, + #[cultcache(key = 2)] + pub packet_id: u64, + #[cultcache(key = 3)] + pub codec: String, + #[cultcache(key = 4)] + pub pts_ticks: i64, + #[cultcache(key = 5)] + pub duration_ticks: u32, + #[cultcache(key = 6)] + pub timebase_num: u32, + #[cultcache(key = 7)] + pub timebase_den: u32, + #[cultcache(key = 8)] + pub deadline_ticks: i64, + #[cultcache(key = 9, bytes)] + pub payload: Vec, +} + +/// What a receiver observed, published back to the producer so it can adapt: +/// what decoded, what is missing, what arrived too late, and how much queue and +/// jitter the receiver is carrying. +/// +/// This is the only record here that flows consumer-to-producer. A producer is +/// free to ignore it; it reports observation, it does not issue commands. +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache( + type = "gamecult.media_receiver_feedback", + schema = "gamecult.media_receiver_feedback.v1" +)] +pub struct GameCultMediaReceiverFeedbackRecord { + #[cultcache(key = 0)] + pub stream_id: String, + #[cultcache(key = 1)] + pub session_id: String, + #[cultcache(key = 2)] + pub receiver_id: String, + #[cultcache(key = 3)] + pub highest_decodable_frame_id: Option, + #[cultcache(key = 4)] + pub missing_frame_ids: Vec, + #[cultcache(key = 5)] + pub late_frame_ids: Vec, + #[cultcache(key = 6)] + pub requested_keyframe: bool, + #[cultcache(key = 7)] + pub jitter_us: i64, + #[cultcache(key = 8)] + pub decode_queue_us: i64, + #[cultcache(key = 9)] + pub observed_at: String, + #[cultcache(key = 10, default)] + pub missing_video_chunk_keys: Vec, +} diff --git a/packages/cultnet-rs/src/media_stream_wire.rs b/packages/cultnet-rs/src/media_stream_wire.rs new file mode 100644 index 00000000..9b7fbc1b --- /dev/null +++ b/packages/cultnet-rs/src/media_stream_wire.rs @@ -0,0 +1,581 @@ +//! How a CultMesh Media Stream record travels: the envelope both ends must +//! agree on. +//! +//! The records in `media_stream_contracts` describe media. This describes how +//! one reaches a peer — wrapped in `cultnet.document_put_raw.v0` with a schema +//! id, a record key, and provenance. It lives here because a producer and a +//! consumer that never link each other still have to agree on it exactly, and +//! the last time they did not, one end was Rust and the other was 6,515 lines +//! of C++ that had drifted. +//! +//! # No producer's name belongs in a shared envelope +//! +//! The Muninn implementation this replaces hardcoded `"muninn-media:"` into +//! every message id and `"muninn.media"` into every tag, so a second producer +//! would have had to label its traffic as Muninn's. The producer namespace is a +//! parameter here: see [`MediaWireProvenance`]. +//! +//! # Record keys are addresses, not decoration +//! +//! A key identifies one piece of media within a session, so a consumer can tell +//! a retransmission from a new frame and a chunk from its siblings. The shapes +//! are fixed by this module rather than by each producer, because a consumer +//! parses them. + +use anyhow::{Result, anyhow}; + +use crate::contracts::{ + CultNetMessage, CultNetRawDocumentRecord, CultNetRawPayloadEncoding, CultNetWireContract, + decode_cultnet_message_from_slice, encode_cultnet_message_to_vec, +}; +use crate::media_stream_contracts::{ + GAMECULT_MEDIA_AUDIO_PACKET_SCHEMA, GAMECULT_MEDIA_RECEIVER_FEEDBACK_SCHEMA, + GAMECULT_MEDIA_VIDEO_ACCESS_UNIT_SCHEMA, GAMECULT_MEDIA_VIDEO_PARITY_SHARD_SCHEMA, + GameCultMediaAudioPacketRecord, GameCultMediaReceiverFeedbackRecord, + GameCultMediaVideoAccessUnitRecord, GameCultMediaVideoParityShardRecord, +}; + +/// The CultNet channel video and receiver feedback ride. +pub const GAMECULT_MEDIA_CHANNEL: &str = "media"; + +/// The CultNet channel audio rides. Audio has its own channel so one session +/// can carry it reliably with a short expiry while video stays lossy under +/// parity; the RUDP transport profile owns each channel's delivery. +pub const GAMECULT_MEDIA_AUDIO_CHANNEL: &str = "audio"; + +/// One media record on its way somewhere. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum GameCultMediaWireRecord { + Video(GameCultMediaVideoAccessUnitRecord), + VideoParity(GameCultMediaVideoParityShardRecord), + Audio(GameCultMediaAudioPacketRecord), + Feedback(GameCultMediaReceiverFeedbackRecord), +} + +/// Who is sending, and when. `producer` namespaces message ids and tags so a +/// stream can be attributed without the envelope knowing any producer by name. +#[derive(Clone, Copy, Debug)] +pub struct MediaWireProvenance<'a> { + pub stored_at: &'a str, + pub runtime_id: &'a str, + pub role: &'a str, + pub producer: &'a str, +} + +impl MediaWireProvenance<'_> { + fn validate(&self) -> Result<()> { + for (name, value) in [ + ("stored_at", self.stored_at), + ("runtime_id", self.runtime_id), + ("role", self.role), + ("producer", self.producer), + ] { + if value.is_empty() { + return Err(anyhow!("media wire provenance {name} must be non-empty")); + } + } + Ok(()) + } +} + +impl GameCultMediaWireRecord { + pub fn schema_id(&self) -> &'static str { + match self { + Self::Video(_) => GAMECULT_MEDIA_VIDEO_ACCESS_UNIT_SCHEMA, + Self::VideoParity(_) => GAMECULT_MEDIA_VIDEO_PARITY_SHARD_SCHEMA, + Self::Audio(_) => GAMECULT_MEDIA_AUDIO_PACKET_SCHEMA, + Self::Feedback(_) => GAMECULT_MEDIA_RECEIVER_FEEDBACK_SCHEMA, + } + } + + /// Addresses this record within its session. + pub fn record_key(&self) -> String { + match self { + Self::Video(record) => format!( + "{}:{}:video:{}:{}", + record.stream_id, record.session_id, record.frame_id, record.chunk_index + ), + Self::VideoParity(record) => format!( + "{}:{}:video-parity:{}:{}", + record.stream_id, record.session_id, record.frame_id, record.parity_index + ), + Self::Audio(record) => format!( + "{}:{}:audio:{}", + record.stream_id, record.session_id, record.packet_id + ), + Self::Feedback(record) => format!( + "{}:{}:feedback:{}", + record.stream_id, record.session_id, record.receiver_id + ), + } + } + + /// The record's own MessagePack form. + /// + /// Serializing the record directly is only correct because the payload + /// fields carry `#[cultcache(key = N, bytes)]`. Muninn previously mirrored + /// each record into a parallel tuple struct — every field restated, in + /// order, by hand — solely so `#[serde(with = "serde_bytes")]` could be + /// applied to the payload, because the `DatabaseEntry` derive does not read + /// serde attributes. Those shadow structs are gone; the derive emits `bin` + /// itself now. Do not reintroduce one. + fn payload_bytes(&self) -> Result> { + match self { + Self::Video(record) => rmp_serde::to_vec(record), + Self::VideoParity(record) => rmp_serde::to_vec(record), + Self::Audio(record) => rmp_serde::to_vec(record), + Self::Feedback(record) => rmp_serde::to_vec(record), + } + .map_err(Into::into) + } +} + +/// Wraps a media record for the wire. +pub fn encode_media_wire_record( + record: &GameCultMediaWireRecord, + provenance: MediaWireProvenance<'_>, +) -> Result> { + provenance.validate()?; + + let schema_id = record.schema_id(); + let record_key = record.record_key(); + let message = CultNetMessage::DocumentPutRaw { + message_id: format!( + "{}-media:{}:{}", + provenance.producer, + schema_id, + record_key.replace(':', "-") + ), + document: CultNetRawDocumentRecord { + schema_id: schema_id.to_string(), + record_key, + stored_at: provenance.stored_at.to_string(), + payload_encoding: CultNetRawPayloadEncoding::Messagepack, + payload: record.payload_bytes()?, + source_runtime_id: Some(provenance.runtime_id.to_string()), + source_agent_id: None, + source_role: Some(provenance.role.to_string()), + tags: Some(vec![format!("{}.media", provenance.producer)]), + }, + }; + + encode_cultnet_message_to_vec(&message, CultNetWireContract::CultNetSchemaV0).map_err(Into::into) +} + +/// Unwraps a media record from the wire. +pub fn decode_media_wire_record(payload: &[u8]) -> Result { + let message = decode_cultnet_message_from_slice(payload, CultNetWireContract::CultNetSchemaV0)?; + let CultNetMessage::DocumentPutRaw { document, .. } = message else { + return Err(anyhow!("expected cultnet.document_put_raw.v0")); + }; + if document.payload_encoding != CultNetRawPayloadEncoding::Messagepack { + return Err(anyhow!("media document payload must be MessagePack")); + } + + let record = match document.schema_id.as_str() { + GAMECULT_MEDIA_VIDEO_ACCESS_UNIT_SCHEMA => { + let record: GameCultMediaVideoAccessUnitRecord = + rmp_serde::from_slice(&document.payload)?; + validate_video_record(&record)?; + GameCultMediaWireRecord::Video(record) + } + GAMECULT_MEDIA_VIDEO_PARITY_SHARD_SCHEMA => { + let record: GameCultMediaVideoParityShardRecord = + rmp_serde::from_slice(&document.payload)?; + validate_video_parity_record(&record)?; + GameCultMediaWireRecord::VideoParity(record) + } + GAMECULT_MEDIA_AUDIO_PACKET_SCHEMA => { + let record: GameCultMediaAudioPacketRecord = + rmp_serde::from_slice(&document.payload)?; + validate_audio_record(&record)?; + GameCultMediaWireRecord::Audio(record) + } + GAMECULT_MEDIA_RECEIVER_FEEDBACK_SCHEMA => { + let record: GameCultMediaReceiverFeedbackRecord = + rmp_serde::from_slice(&document.payload)?; + validate_feedback_record(&record)?; + GameCultMediaWireRecord::Feedback(record) + } + other => return Err(anyhow!("unsupported media schema {other}")), + }; + + // The key is derived from the record's own contents, so a mismatch means the + // envelope and its payload disagree about what this is. Admitting it would + // let a consumer address media by a key its producer never meant. + let expected = record.record_key(); + if document.record_key != expected { + return Err(anyhow!( + "record key mismatch: envelope says {}, record derives {expected}", + document.record_key + )); + } + + Ok(record) +} + +// --------------------------------------------------------------------------- +// Validation +// +// Moved here from Muninn with the envelope. A decoder that does not validate +// leaves each consumer to invent its own idea of a well-formed record, which is +// how the C++ receiver and the Rust sender came to disagree while both looked +// correct in isolation. Muninn's tests for these came too; they now cover the +// contract rather than one producer's copy of it. +// --------------------------------------------------------------------------- + +/// Addresses one chunk of a video frame. The string form appears in receiver +/// feedback and is parsed by consumers, so its shape is contract. +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub struct VideoChunkKey { + pub frame_id: u64, + pub chunk_index: u16, +} + +impl VideoChunkKey { + pub fn new(frame_id: u64, chunk_index: u16) -> Self { + Self { + frame_id, + chunk_index, + } + } + + pub fn parse(value: &str) -> Result { + if value.matches(':').count() != 1 { + return Err(anyhow!( + "video chunk key must contain exactly one ':' separator" + )); + } + let (frame_id, chunk_index) = value + .split_once(':') + .ok_or_else(|| anyhow!("video chunk key must be ':'"))?; + Ok(Self { + frame_id: frame_id + .parse() + .map_err(|_| anyhow!("video chunk key frame_id must be an integer"))?, + chunk_index: chunk_index + .parse() + .map_err(|_| anyhow!("video chunk key chunk_index must be an integer"))?, + }) + } + + pub fn as_feedback_key(&self) -> String { + format!("{}:{}", self.frame_id, self.chunk_index) + } +} + +pub fn video_chunk_feedback_key(frame_id: u64, chunk_index: u16) -> String { + VideoChunkKey::new(frame_id, chunk_index).as_feedback_key() +} + +pub fn normalize_video_chunk_feedback_keys(keys: Vec) -> Result> { + let mut parsed = keys + .iter() + .map(|key| VideoChunkKey::parse(key)) + .collect::>>()?; + parsed.sort_unstable(); + parsed.dedup(); + Ok(parsed + .into_iter() + .map(|key| key.as_feedback_key()) + .collect()) +} + +pub fn validate_video_record(record: &GameCultMediaVideoAccessUnitRecord) -> Result<()> { + if record.stream_id.is_empty() { + return Err(anyhow!("video media record stream_id must be non-empty")); + } + if record.session_id.is_empty() { + return Err(anyhow!("video media record session_id must be non-empty")); + } + if record.codec.is_empty() { + return Err(anyhow!("video media record codec must be non-empty")); + } + if record.timebase_num == 0 || record.timebase_den == 0 { + return Err(anyhow!("video media record timebase must be non-zero")); + } + if record.duration_ticks == 0 { + return Err(anyhow!( + "video media record duration_ticks must be greater than zero" + )); + } + if record.deadline_ticks < record.pts_ticks { + return Err(anyhow!( + "video media record deadline_ticks must not precede pts_ticks" + )); + } + if record.chunk_count == 0 { + return Err(anyhow!("video media record chunk_count must be non-zero")); + } + if record.chunk_index >= record.chunk_count { + return Err(anyhow!( + "video media record chunk_index {} is outside chunk_count {}", + record.chunk_index, + record.chunk_count + )); + } + if record.payload.is_empty() { + return Err(anyhow!("video media record payload must be non-empty")); + } + Ok(()) +} + +pub fn validate_video_parity_record(record: &GameCultMediaVideoParityShardRecord) -> Result<()> { + if record.stream_id.is_empty() { + return Err(anyhow!( + "video parity media record stream_id must be non-empty" + )); + } + if record.session_id.is_empty() { + return Err(anyhow!( + "video parity media record session_id must be non-empty" + )); + } + if record.codec.is_empty() { + return Err(anyhow!("video parity media record codec must be non-empty")); + } + if record.timebase_num == 0 || record.timebase_den == 0 { + return Err(anyhow!( + "video parity media record timebase must be non-zero" + )); + } + if record.duration_ticks == 0 { + return Err(anyhow!( + "video parity media record duration_ticks must be greater than zero" + )); + } + if record.deadline_ticks < record.pts_ticks { + return Err(anyhow!( + "video parity media record deadline_ticks must not precede pts_ticks" + )); + } + if record.chunk_count == 0 { + return Err(anyhow!( + "video parity media record chunk_count must be non-zero" + )); + } + if record.parity_count == 0 || record.parity_index >= record.parity_count { + return Err(anyhow!( + "video parity media record has invalid parity stripe metadata" + )); + } + if record.parity_count > record.chunk_count { + return Err(anyhow!( + "video parity media record parity_count exceeds chunk_count" + )); + } + if record.chunk_payload_bytes == 0 || record.last_chunk_payload_bytes == 0 { + return Err(anyhow!( + "video parity media record chunk lengths must be non-zero" + )); + } + if record.last_chunk_payload_bytes > record.chunk_payload_bytes { + return Err(anyhow!( + "video parity media record last chunk length exceeds regular chunk length" + )); + } + if record.payload.is_empty() { + return Err(anyhow!( + "video parity media record payload must be non-empty" + )); + } + if record.payload.len() > record.chunk_payload_bytes as usize { + return Err(anyhow!( + "video parity media record payload exceeds declared chunk length" + )); + } + Ok(()) +} + +pub fn validate_audio_record(record: &GameCultMediaAudioPacketRecord) -> Result<()> { + if record.stream_id.is_empty() { + return Err(anyhow!("audio media record stream_id must be non-empty")); + } + if record.session_id.is_empty() { + return Err(anyhow!("audio media record session_id must be non-empty")); + } + if record.codec.is_empty() { + return Err(anyhow!("audio media record codec must be non-empty")); + } + if record.timebase_num == 0 || record.timebase_den == 0 { + return Err(anyhow!("audio media record timebase must be non-zero")); + } + if record.duration_ticks == 0 { + return Err(anyhow!( + "audio media record duration_ticks must be greater than zero" + )); + } + if record.deadline_ticks < record.pts_ticks { + return Err(anyhow!( + "audio media record deadline_ticks must not precede pts_ticks" + )); + } + if record.payload.is_empty() { + return Err(anyhow!("audio media record payload must be non-empty")); + } + Ok(()) +} + +pub fn validate_feedback_record(record: &GameCultMediaReceiverFeedbackRecord) -> Result<()> { + if record.stream_id.is_empty() { + return Err(anyhow!( + "receiver feedback media record stream_id must be non-empty" + )); + } + if record.session_id.is_empty() { + return Err(anyhow!( + "receiver feedback media record session_id must be non-empty" + )); + } + if record.receiver_id.is_empty() { + return Err(anyhow!( + "receiver feedback media record receiver_id must be non-empty" + )); + } + if record.observed_at.is_empty() { + return Err(anyhow!( + "receiver feedback media record observed_at must be non-empty" + )); + } + if record.jitter_us < 0 { + return Err(anyhow!( + "receiver feedback media record jitter_us must be non-negative" + )); + } + if record.decode_queue_us < 0 { + return Err(anyhow!( + "receiver feedback media record decode_queue_us must be non-negative" + )); + } + normalize_video_chunk_feedback_keys(record.missing_video_chunk_keys.clone())?; + Ok(()) +} + +/// What a receiver has to say about a stream. The record it builds is the +/// contract's; this normalises it so both ends agree on its shape: damage +/// lists sorted and unique, chunk keys well-formed, and a keyframe requested +/// whenever a whole frame is missing, since a producer's repair cache holds +/// chunks, not references. Missing chunks alone are a repair request, which +/// exists precisely so that a keyframe is not needed. +#[derive(Clone, Debug)] +pub struct ReceiverFeedbackOptions<'a> { + pub stream_id: &'a str, + pub session_id: &'a str, + pub receiver_id: &'a str, + pub highest_decodable_frame_id: Option, + pub missing_frame_ids: Vec, + pub missing_video_chunk_keys: Vec, + pub late_frame_ids: Vec, + pub requested_keyframe: bool, + pub jitter_us: i64, + pub decode_queue_us: i64, + pub observed_at: &'a str, +} + +pub fn build_receiver_feedback( + options: ReceiverFeedbackOptions<'_>, +) -> Result { + let mut missing_frame_ids = options.missing_frame_ids; + missing_frame_ids.sort_unstable(); + missing_frame_ids.dedup(); + let missing_video_chunk_keys = + normalize_video_chunk_feedback_keys(options.missing_video_chunk_keys)?; + let mut late_frame_ids = options.late_frame_ids; + late_frame_ids.sort_unstable(); + late_frame_ids.dedup(); + let requested_keyframe = options.requested_keyframe || !missing_frame_ids.is_empty(); + + let record = GameCultMediaReceiverFeedbackRecord { + stream_id: options.stream_id.to_string(), + session_id: options.session_id.to_string(), + receiver_id: options.receiver_id.to_string(), + highest_decodable_frame_id: options.highest_decodable_frame_id, + missing_frame_ids, + late_frame_ids, + requested_keyframe, + jitter_us: options.jitter_us, + decode_queue_us: options.decode_queue_us, + observed_at: options.observed_at.to_string(), + missing_video_chunk_keys, + }; + validate_feedback_record(&record)?; + Ok(record) +} + +#[cfg(test)] +mod receiver_feedback_tests { + use super::*; + + fn options() -> ReceiverFeedbackOptions<'static> { + ReceiverFeedbackOptions { + stream_id: "muninn.raven.av.rudp", + session_id: "session-1", + receiver_id: "starfire.obs", + highest_decodable_frame_id: Some(40), + missing_frame_ids: Vec::new(), + missing_video_chunk_keys: Vec::new(), + late_frame_ids: Vec::new(), + requested_keyframe: false, + jitter_us: 0, + decode_queue_us: 2_000, + observed_at: "2026-06-18T00:00:00Z", + } + } + + #[test] + fn builds_receiver_feedback_with_sorted_unique_damage_lists() -> Result<()> { + let feedback = build_receiver_feedback(ReceiverFeedbackOptions { + missing_frame_ids: vec![43, 42, 43], + missing_video_chunk_keys: vec![ + video_chunk_feedback_key(43, 2), + video_chunk_feedback_key(42, 1), + video_chunk_feedback_key(42, 1), + ], + late_frame_ids: vec![39, 39, 38], + jitter_us: 700, + ..options() + })?; + + assert_eq!(feedback.missing_frame_ids, vec![42, 43]); + assert_eq!(feedback.missing_video_chunk_keys, vec!["42:1", "43:2"]); + assert_eq!(feedback.late_frame_ids, vec![38, 39]); + assert!(feedback.requested_keyframe, "a whole missing frame is a lost reference"); + Ok(()) + } + + /// A producer answers a keyframe request with an IDR. A repair request is + /// the cheaper alternative and must not carry one by accident. + #[test] + fn a_repair_request_does_not_ask_for_a_keyframe() -> Result<()> { + let feedback = build_receiver_feedback(ReceiverFeedbackOptions { + missing_video_chunk_keys: vec![video_chunk_feedback_key(42, 1)], + ..options() + })?; + assert!(!feedback.requested_keyframe); + Ok(()) + } + + #[test] + fn rejects_negative_receiver_feedback_timing() { + let error = build_receiver_feedback(ReceiverFeedbackOptions { jitter_us: -1, ..options() }) + .unwrap_err(); + assert!(error.to_string().contains("jitter_us")); + } + + #[test] + fn rejects_malformed_receiver_feedback_chunk_keys() { + let error = build_receiver_feedback(ReceiverFeedbackOptions { + missing_video_chunk_keys: vec!["frame:chunk".to_string()], + ..options() + }) + .unwrap_err(); + assert!(error.to_string().contains("frame_id"), "{error}"); + } + + #[test] + fn rejects_an_empty_receiver_id() { + let error = build_receiver_feedback(ReceiverFeedbackOptions { receiver_id: "", ..options() }) + .unwrap_err(); + assert!(error.to_string().contains("receiver_id")); + } +} diff --git a/packages/cultnet-rs/src/provider_health_contracts.rs b/packages/cultnet-rs/src/provider_health_contracts.rs new file mode 100644 index 00000000..632c74f2 --- /dev/null +++ b/packages/cultnet-rs/src/provider_health_contracts.rs @@ -0,0 +1,716 @@ +use anyhow::{Context, Result, anyhow, bail}; +use cultcache_rs::DatabaseEntry; +use serde::{Deserialize, Serialize}; + +use crate::runtime_authority_contracts::{ + GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA, GAMECULT_RUNTIME_PRESENCE_HEALTH_SIGNING_PURPOSE, + ODIN_RUNTIME_TOPOLOGY_CORRELATION_SCHEMA, ODIN_RUNTIME_TOPOLOGY_CORRELATION_SIGNING_PURPOSE, +}; +use crate::{ + GameCultProviderHealthIdentity, IdunnServiceIdentity, OdinTopologyIdentity, + derive_service_identity_id, +}; + +pub const GAMECULT_SERVICE_TRUST_ANCHOR_SCHEMA: &str = "gamecult.service_trust_anchor.v1"; +pub const IDUNN_AUTHENTICATED_PROVIDER_HEALTH_PROJECTION_SCHEMA: &str = + "idunn.authenticated_provider_health_projection.v1"; +pub const IDUNN_SIGNED_DAEMON_HEALTH_SCHEMA: &str = "idunn.signed_daemon_health.v1"; +pub const IDUNN_AUTHENTICATED_PROVIDER_HEALTH_PROJECTION_SIGNING_PURPOSE: &str = + "idunn.authenticated-provider-health-projection.v1"; +pub const IDUNN_PROVIDER_ACTIVE_REASON: &str = "authenticated_provider_active"; +pub const IDUNN_PROVIDER_WARMING_REASON: &str = "authenticated_provider_warming"; +pub const IDUNN_PROVIDER_DEGRADED_REASON: &str = "authenticated_provider_degraded"; +pub const IDUNN_PROVIDER_FAILED_REASON: &str = "authenticated_provider_failed"; +pub const IDUNN_DAEMON_HEALTH_TRUST_BINDING_SCHEMA: &str = "idunn.daemon_health_trust_binding.v1"; + +/// Provider-owned health statement. Ingress verifies `signature` over the +/// complete positional record with this field empty before admitting it. +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache( + type = "idunn.signed_daemon_health", + schema = "idunn.signed_daemon_health.v1" +)] +pub struct IdunnSignedDaemonHealthRecord { + #[cultcache(key = 0)] + pub schema_version: String, + #[cultcache(key = 1)] + pub daemon_id: String, + #[cultcache(key = 2)] + pub health_contract: String, + #[cultcache(key = 3)] + pub source_runtime_id: String, + #[cultcache(key = 4)] + pub state: String, + #[cultcache(key = 5)] + pub detail: String, + #[cultcache(key = 6)] + pub signer_identity_id: String, + #[cultcache(key = 7)] + pub publisher_incarnation_id: String, + #[cultcache(key = 8)] + pub publisher_sequence: u64, + #[cultcache(key = 9)] + pub observed_at_unix_millis: u64, + #[cultcache(key = 10)] + pub release_id: Option, + #[cultcache(key = 11)] + pub release_witness_sha256: Option, + #[cultcache(key = 12)] + pub source_commit: Option, + #[cultcache(key = 13)] + pub deployment_id: Option, + #[cultcache(key = 14)] + pub signature_algorithm: String, + #[cultcache(key = 15, bytes)] + pub signature: Vec, + #[cultcache(key = 16)] + pub private_state_exposed: bool, + #[cultcache(key = 17, default)] + pub activation_witness_sha256: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct LegacyIdunnSignedDaemonHealthWire { + schema_version: String, + daemon_id: String, + health_contract: String, + source_runtime_id: String, + state: String, + detail: String, + signer_identity_id: String, + publisher_incarnation_id: String, + publisher_sequence: u64, + observed_at_unix_millis: u64, + release_id: Option, + release_witness_sha256: Option, + source_commit: Option, + deployment_id: Option, + signature_algorithm: String, + #[serde(with = "serde_bytes")] + signature: Vec, + private_state_exposed: bool, +} + +impl From for IdunnSignedDaemonHealthRecord { + fn from(value: LegacyIdunnSignedDaemonHealthWire) -> Self { + Self { + schema_version: value.schema_version, + daemon_id: value.daemon_id, + health_contract: value.health_contract, + source_runtime_id: value.source_runtime_id, + state: value.state, + detail: value.detail, + signer_identity_id: value.signer_identity_id, + publisher_incarnation_id: value.publisher_incarnation_id, + publisher_sequence: value.publisher_sequence, + observed_at_unix_millis: value.observed_at_unix_millis, + release_id: value.release_id, + release_witness_sha256: value.release_witness_sha256, + source_commit: value.source_commit, + deployment_id: value.deployment_id, + signature_algorithm: value.signature_algorithm, + signature: value.signature, + private_state_exposed: value.private_state_exposed, + activation_witness_sha256: None, + } + } +} + +impl IdunnSignedDaemonHealthRecord { + /// Decode the canonical current or pre-activation positional wire and + /// return the exact wire generation's unsigned signature payload. + pub fn decode_canonical_signed_payload(payload: &[u8]) -> Result<(Self, Vec)> { + match messagepack_array_len(payload) { + Some(18) => { + let statement: Self = rmp_serde::from_slice(payload) + .context("decoding generic signed daemon health")?; + if rmp_serde::to_vec(&statement)? != payload { + bail!("signed daemon health payload is not canonical positional MessagePack"); + } + let mut unsigned = statement.clone(); + unsigned.signature.clear(); + Ok((statement, rmp_serde::to_vec(&unsigned)?)) + } + Some(17) => { + let legacy: LegacyIdunnSignedDaemonHealthWire = rmp_serde::from_slice(payload) + .context("decoding legacy generic signed daemon health")?; + if rmp_serde::to_vec(&legacy)? != payload { + bail!("signed daemon health payload is not canonical positional MessagePack"); + } + let mut unsigned = legacy.clone(); + unsigned.signature.clear(); + let unsigned_payload = rmp_serde::to_vec(&unsigned)?; + Ok((legacy.into(), unsigned_payload)) + } + _ => bail!("signed daemon health payload is not canonical positional MessagePack"), + } + } + + pub fn validate(&self) -> Result<()> { + if self.schema_version != IDUNN_SIGNED_DAEMON_HEALTH_SCHEMA { + bail!("signed daemon health schema is unsupported"); + } + validate_identifier(&self.daemon_id, "daemon id")?; + validate_identifier(&self.health_contract, "health contract")?; + validate_identifier(&self.source_runtime_id, "source runtime id")?; + validate_identifier(&self.signer_identity_id, "signer identity id")?; + validate_identifier(&self.publisher_incarnation_id, "publisher incarnation id")?; + if !matches!( + self.state.as_str(), + "active" | "warming" | "degraded" | "failed" + ) || self.detail.len() > 512 + || self.signature_algorithm != "ed25519" + || self.signature.len() != 64 + || self.publisher_sequence == 0 + || self.observed_at_unix_millis == 0 + { + bail!("signed daemon health shape is invalid"); + } + validate_optional_release_binding( + &self.release_id, + &self.release_witness_sha256, + &self.source_commit, + )?; + validate_optional_identifier(&self.deployment_id, "deployment id")?; + validate_optional_sha256(&self.activation_witness_sha256, "activation witness sha256")?; + if self.private_state_exposed { + bail!("signed daemon health exposes private state"); + } + Ok(()) + } +} + +fn messagepack_array_len(payload: &[u8]) -> Option { + match *payload.first()? { + marker @ 0x90..=0x9f => Some(usize::from(marker & 0x0f)), + 0xdc => Some(usize::from(u16::from_be_bytes([ + *payload.get(1)?, + *payload.get(2)?, + ]))), + 0xdd => usize::try_from(u32::from_be_bytes([ + *payload.get(1)?, + *payload.get(2)?, + *payload.get(3)?, + *payload.get(4)?, + ])) + .ok(), + _ => None, + } +} + +/// Root-owned admission binding. Store ownership is the physical authority; +/// this document names the only signer accepted for one daemon/contract/runtime. +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache( + type = "idunn.daemon_health_trust_binding", + schema = "idunn.daemon_health_trust_binding.v1" +)] +pub struct IdunnDaemonHealthTrustBindingRecord { + #[cultcache(key = 0)] + pub schema_version: String, + #[cultcache(key = 1)] + pub binding_id: String, + #[cultcache(key = 2)] + pub daemon_id: String, + #[cultcache(key = 3)] + pub health_contract: String, + #[cultcache(key = 4)] + pub source_runtime_id: String, + #[cultcache(key = 5)] + pub signer_identity_id: String, + #[cultcache(key = 6)] + pub signer_public_key: Vec, + #[cultcache(key = 7)] + pub binding_authority: String, + #[cultcache(key = 8)] + pub bound_at_unix_millis: u64, + #[cultcache(key = 9)] + pub release_binding_required: bool, + #[cultcache(key = 10)] + pub private_state_exposed: bool, +} + +impl IdunnDaemonHealthTrustBindingRecord { + pub fn validate(&self) -> Result<()> { + if self.schema_version != IDUNN_DAEMON_HEALTH_TRUST_BINDING_SCHEMA { + bail!("daemon health trust binding schema is unsupported"); + } + validate_identifier(&self.binding_id, "binding id")?; + validate_identifier(&self.daemon_id, "daemon id")?; + validate_identifier(&self.health_contract, "health contract")?; + validate_identifier(&self.source_runtime_id, "source runtime id")?; + validate_identifier(&self.signer_identity_id, "signer identity id")?; + if self.signer_public_key.len() != 32 + || self.binding_authority != "root" + || self.bound_at_unix_millis == 0 + { + bail!("daemon health trust binding authority or key is invalid"); + } + if self.private_state_exposed { + bail!("daemon health trust binding exposes private state"); + } + if self.signer_identity_id + != derive_service_identity_id::( + &self.signer_public_key, + )? + { + bail!("daemon health trust binding identity does not match its public key"); + } + Ok(()) + } +} + +/// Root-distributed public key binding for one service-owned signed contract. +/// Consumers pin this document; a self-declared key inside a signed projection +/// is never authority. +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache( + type = "gamecult.service_trust_anchor", + schema = "gamecult.service_trust_anchor.v1" +)] +pub struct GameCultServiceTrustAnchorRecord { + #[cultcache(key = 0)] + pub schema_version: String, + #[cultcache(key = 1)] + pub trust_anchor_id: String, + #[cultcache(key = 2)] + pub service_id: String, + #[cultcache(key = 3)] + pub runtime_id: String, + #[cultcache(key = 4)] + pub signer_identity_id: String, + #[cultcache(key = 5)] + pub signer_public_key: Vec, + #[cultcache(key = 6)] + pub signature_algorithm: String, + #[cultcache(key = 7)] + pub signing_purpose: String, + #[cultcache(key = 8)] + pub signed_schema: String, + #[cultcache(key = 9)] + pub binding_authority: String, + #[cultcache(key = 10)] + pub bound_at_unix_millis: u64, + #[cultcache(key = 11)] + pub expires_at_unix_millis: Option, + #[cultcache(key = 12)] + pub private_state_exposed: bool, +} + +impl GameCultServiceTrustAnchorRecord { + pub fn validate(&self) -> Result<()> { + if self.schema_version != GAMECULT_SERVICE_TRUST_ANCHOR_SCHEMA { + bail!("service trust anchor schema is unsupported"); + } + validate_identifier(&self.trust_anchor_id, "trust anchor id")?; + validate_identifier(&self.service_id, "service id")?; + validate_identifier(&self.runtime_id, "runtime id")?; + validate_identifier(&self.signer_identity_id, "signer identity id")?; + validate_identifier(&self.signing_purpose, "signing purpose")?; + validate_identifier(&self.signed_schema, "signed schema")?; + if self.signer_public_key.len() != 32 + || self.signature_algorithm != "ed25519" + || self.binding_authority != "root" + || self.bound_at_unix_millis == 0 + || self + .expires_at_unix_millis + .is_some_and(|expires_at| expires_at <= self.bound_at_unix_millis) + || self.private_state_exposed + { + bail!("service trust anchor authority, key, lifetime, or privacy is invalid"); + } + if self.signed_schema == IDUNN_AUTHENTICATED_PROVIDER_HEALTH_PROJECTION_SCHEMA + && (self.service_id != "idunn" + || self.signing_purpose + != IDUNN_AUTHENTICATED_PROVIDER_HEALTH_PROJECTION_SIGNING_PURPOSE + || self.signer_identity_id + != derive_service_identity_id::(&self.signer_public_key)?) + { + bail!("Idunn provider-health trust anchor profile is invalid"); + } + let is_runtime_presence_profile = self + .signed_schema + .starts_with("gamecult.runtime_presence_health.") + || self + .signing_purpose + .starts_with("gamecult.runtime_presence_health."); + if is_runtime_presence_profile + && (self.signed_schema != GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA + || self.signing_purpose != GAMECULT_RUNTIME_PRESENCE_HEALTH_SIGNING_PURPOSE + || self.signer_identity_id + != derive_service_identity_id::( + &self.signer_public_key, + )?) + { + bail!("runtime presence trust anchor profile is invalid"); + } + if (self.signed_schema == ODIN_RUNTIME_TOPOLOGY_CORRELATION_SCHEMA + || self.signing_purpose == ODIN_RUNTIME_TOPOLOGY_CORRELATION_SIGNING_PURPOSE) + && (self.signed_schema != ODIN_RUNTIME_TOPOLOGY_CORRELATION_SCHEMA + || self.signing_purpose != ODIN_RUNTIME_TOPOLOGY_CORRELATION_SIGNING_PURPOSE + || self.service_id != "odin" + || self.signer_identity_id + != derive_service_identity_id::(&self.signer_public_key)?) + { + bail!("Odin topology correlation trust anchor profile is invalid"); + } + Ok(()) + } +} + +/// Idunn-owned public projection of one current authenticated generic provider +/// admission. Absence and dependency failure produce no record; they are not +/// provider states Idunn may manufacture. +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache( + type = "idunn.authenticated_provider_health_projection", + schema = "idunn.authenticated_provider_health_projection.v1" +)] +pub struct IdunnAuthenticatedProviderHealthProjectionRecord { + #[cultcache(key = 0)] + pub schema_version: String, + #[cultcache(key = 1)] + pub projection_id: String, + #[cultcache(key = 2)] + pub daemon_id: String, + #[cultcache(key = 3)] + pub health_contract: String, + #[cultcache(key = 4)] + pub provider_state: String, + #[cultcache(key = 5)] + pub reason_code: String, + #[cultcache(key = 6)] + pub provider_observed_at_unix_millis: u64, + #[cultcache(key = 7)] + pub admitted_at_unix_millis: u64, + #[cultcache(key = 8)] + pub evaluated_at_unix_millis: u64, + #[cultcache(key = 9)] + pub trust_binding_id: String, + #[cultcache(key = 10)] + pub trust_binding_sha256: String, + #[cultcache(key = 11)] + pub signed_health_sha256: String, + #[cultcache(key = 12)] + pub authenticated_admission_sha256: String, + #[cultcache(key = 13)] + pub provider_signer_identity_id: String, + #[cultcache(key = 14)] + pub provider_incarnation_id: String, + #[cultcache(key = 15)] + pub provider_sequence: u64, + #[cultcache(key = 16)] + pub release_id: Option, + #[cultcache(key = 17)] + pub release_witness_sha256: Option, + #[cultcache(key = 18)] + pub source_commit: Option, + #[cultcache(key = 19)] + pub deployment_id: Option, + #[cultcache(key = 20)] + pub idunn_runtime_id: String, + #[cultcache(key = 21)] + pub idunn_signer_identity_id: String, + #[cultcache(key = 22)] + pub projection_incarnation_id: String, + #[cultcache(key = 23)] + pub projection_sequence: u64, + #[cultcache(key = 24)] + pub signature_algorithm: String, + #[cultcache(key = 25)] + pub signature: Vec, + #[cultcache(key = 26)] + pub private_state_exposed: bool, + #[cultcache(key = 27)] + pub expires_at_unix_millis: u64, +} + +/// Closed derivation for an authenticated provider state. `None` means no +/// provider-health projection may be published. +pub fn authenticated_provider_health_reason_code(state: &str) -> Option<&'static str> { + match state { + "active" => Some(IDUNN_PROVIDER_ACTIVE_REASON), + "warming" => Some(IDUNN_PROVIDER_WARMING_REASON), + "degraded" => Some(IDUNN_PROVIDER_DEGRADED_REASON), + "failed" => Some(IDUNN_PROVIDER_FAILED_REASON), + _ => None, + } +} + +impl IdunnAuthenticatedProviderHealthProjectionRecord { + pub fn validate(&self) -> Result<()> { + if self.schema_version != IDUNN_AUTHENTICATED_PROVIDER_HEALTH_PROJECTION_SCHEMA { + bail!("authenticated provider health projection schema is unsupported"); + } + validate_identifier(&self.projection_id, "projection id")?; + validate_identifier(&self.daemon_id, "daemon id")?; + validate_identifier(&self.health_contract, "health contract")?; + validate_identifier(&self.trust_binding_id, "trust binding id")?; + validate_identifier( + &self.provider_signer_identity_id, + "provider signer identity id", + )?; + validate_identifier(&self.provider_incarnation_id, "provider incarnation id")?; + validate_identifier(&self.idunn_runtime_id, "Idunn runtime id")?; + validate_identifier(&self.idunn_signer_identity_id, "Idunn signer identity id")?; + validate_identifier(&self.projection_incarnation_id, "projection incarnation id")?; + let reason_is_valid = authenticated_provider_health_reason_code(&self.provider_state) + .is_some_and(|reason| reason == self.reason_code); + if !reason_is_valid + || self.provider_observed_at_unix_millis == 0 + || self.admitted_at_unix_millis < self.provider_observed_at_unix_millis + || self.evaluated_at_unix_millis < self.admitted_at_unix_millis + || self.expires_at_unix_millis <= self.evaluated_at_unix_millis + || self.provider_sequence == 0 + || self.projection_sequence == 0 + || !is_sha256(&self.trust_binding_sha256) + || !is_sha256(&self.signed_health_sha256) + || !is_sha256(&self.authenticated_admission_sha256) + || self.signature_algorithm != "ed25519" + || self.signature.len() != 64 + || self.private_state_exposed + { + bail!( + "authenticated provider health projection shape, lineage, reason, signature, or privacy is invalid" + ); + } + validate_optional_release_binding( + &self.release_id, + &self.release_witness_sha256, + &self.source_commit, + )?; + validate_optional_identifier(&self.deployment_id, "deployment id")?; + if self.deployment_id.is_some() != self.release_id.is_some() { + bail!("authenticated provider health projection release lineage is partial"); + } + Ok(()) + } +} + +fn validate_identifier(value: &str, label: &str) -> Result<()> { + if value.trim().is_empty() || value.len() > 256 || value.chars().any(char::is_control) { + bail!("{label} is empty, oversized, or contains control characters"); + } + Ok(()) +} + +fn validate_optional_identifier(value: &Option, label: &str) -> Result<()> { + if let Some(value) = value { + validate_identifier(value, label)?; + } + Ok(()) +} + +fn validate_optional_sha256(value: &Option, label: &str) -> Result<()> { + if value.as_deref().is_some_and(|value| !is_sha256(value)) { + bail!("{label} is malformed"); + } + Ok(()) +} + +fn validate_optional_release_binding( + release_id: &Option, + witness: &Option, + source_commit: &Option, +) -> Result<()> { + if release_id.is_some() || witness.is_some() || source_commit.is_some() { + validate_optional_identifier(release_id, "release id")?; + let witness = witness + .as_deref() + .ok_or_else(|| anyhow!("release witness is absent"))?; + let commit = source_commit + .as_deref() + .ok_or_else(|| anyhow!("source commit is absent"))?; + if release_id.is_none() || !is_sha256(witness) || !is_lower_hex(commit, 40) { + bail!("release binding is incomplete or malformed"); + } + } + Ok(()) +} + +fn is_sha256(value: &str) -> bool { + value + .strip_prefix("sha256-") + .is_some_and(|digest| is_lower_hex(digest, 64)) +} + +fn is_lower_hex(value: &str, length: usize) -> bool { + value.len() == length + && value + .bytes() + .all(|byte| byte.is_ascii_hexdigit() && !byte.is_ascii_uppercase()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn signed_health() -> IdunnSignedDaemonHealthRecord { + IdunnSignedDaemonHealthRecord { + schema_version: IDUNN_SIGNED_DAEMON_HEALTH_SCHEMA.into(), + daemon_id: "epiphany".into(), + health_contract: "epiphany.cultnet-rudp-runtime-health".into(), + source_runtime_id: "epiphany-yggdrasil".into(), + state: "active".into(), + detail: "managed-services-current".into(), + signer_identity_id: "provider-signing-key".into(), + publisher_incarnation_id: "9c2cba2e-c2de-4750-b222-b732f97d0435".into(), + publisher_sequence: 1, + observed_at_unix_millis: 100, + release_id: Some("release-1".into()), + release_witness_sha256: Some(format!("sha256-{}", "d".repeat(64))), + source_commit: Some("e".repeat(40)), + deployment_id: Some("deployment-1".into()), + signature_algorithm: "ed25519".into(), + signature: vec![6; 64], + private_state_exposed: false, + activation_witness_sha256: Some(format!("sha256-{}", "f".repeat(64))), + } + } + + fn anchor() -> GameCultServiceTrustAnchorRecord { + let public_key = vec![5; 32]; + GameCultServiceTrustAnchorRecord { + schema_version: GAMECULT_SERVICE_TRUST_ANCHOR_SCHEMA.into(), + trust_anchor_id: "root/idunn/provider-health".into(), + service_id: "idunn".into(), + runtime_id: "idunn-yggdrasil".into(), + signer_identity_id: derive_service_identity_id::(&public_key) + .unwrap(), + signer_public_key: public_key, + signature_algorithm: "ed25519".into(), + signing_purpose: IDUNN_AUTHENTICATED_PROVIDER_HEALTH_PROJECTION_SIGNING_PURPOSE.into(), + signed_schema: IDUNN_AUTHENTICATED_PROVIDER_HEALTH_PROJECTION_SCHEMA.into(), + binding_authority: "root".into(), + bound_at_unix_millis: 100, + expires_at_unix_millis: Some(200), + private_state_exposed: false, + } + } + + fn projection() -> IdunnAuthenticatedProviderHealthProjectionRecord { + IdunnAuthenticatedProviderHealthProjectionRecord { + schema_version: IDUNN_AUTHENTICATED_PROVIDER_HEALTH_PROJECTION_SCHEMA.into(), + projection_id: "authenticated-provider-health:epiphany".into(), + daemon_id: "epiphany".into(), + health_contract: "health.v1".into(), + provider_state: "active".into(), + reason_code: IDUNN_PROVIDER_ACTIVE_REASON.into(), + provider_observed_at_unix_millis: 100, + admitted_at_unix_millis: 101, + evaluated_at_unix_millis: 102, + trust_binding_id: "root/epiphany".into(), + trust_binding_sha256: format!("sha256-{}", "a".repeat(64)), + signed_health_sha256: format!("sha256-{}", "b".repeat(64)), + authenticated_admission_sha256: format!("sha256-{}", "c".repeat(64)), + provider_signer_identity_id: "provider-signing-key".into(), + provider_incarnation_id: "provider/boot/1".into(), + provider_sequence: 1, + release_id: Some("release-1".into()), + release_witness_sha256: Some(format!("sha256-{}", "d".repeat(64))), + source_commit: Some("e".repeat(40)), + deployment_id: Some("deployment-1".into()), + idunn_runtime_id: "idunn-yggdrasil".into(), + idunn_signer_identity_id: "idunn-signing-key".into(), + projection_incarnation_id: "idunn/boot/1".into(), + projection_sequence: 1, + signature_algorithm: "ed25519".into(), + signature: vec![6; 64], + private_state_exposed: false, + expires_at_unix_millis: 200, + } + } + + #[test] + fn contracts_accept_exact_root_and_lineage_shapes() -> Result<()> { + let anchor = anchor(); + let projection = projection(); + let health = signed_health(); + anchor.validate()?; + projection.validate()?; + health.validate()?; + let encoded = rmp_serde::to_vec(&health)?; + assert_eq!(encoded.first().copied(), Some(0xdc)); + assert_eq!(&encoded[1..3], &[0, 18]); + assert!( + encoded + .windows(4) + .any(|window| window == [0xc4, 0x40, 6, 6]), + "provider signature must use the portable MessagePack binary shape" + ); + assert_eq!( + rmp_serde::from_slice::(&encoded)?, + health + ); + assert_eq!( + rmp_serde::from_slice::(&rmp_serde::to_vec( + &anchor + )?)?, + anchor + ); + assert_eq!( + rmp_serde::from_slice::( + &rmp_serde::to_vec(&projection)? + )?, + projection + ); + assert_eq!(authenticated_provider_health_reason_code("missing"), None); + Ok(()) + } + + #[test] + fn activation_binding_is_additive_for_legacy_generic_health_records() -> Result<()> { + let mut health = signed_health(); + health.activation_witness_sha256 = None; + let mut legacy = rmp_serde::to_vec(&health)?; + assert_eq!(&legacy[..3], &[0xdc, 0, 18]); + assert_eq!(legacy.pop(), Some(0xc0)); + legacy[2] = 17; + let decoded: IdunnSignedDaemonHealthRecord = rmp_serde::from_slice(&legacy)?; + assert_eq!(decoded.activation_witness_sha256, None); + decoded.validate()?; + let (decoded, unsigned) = + IdunnSignedDaemonHealthRecord::decode_canonical_signed_payload(&legacy)?; + assert_eq!(decoded.activation_witness_sha256, None); + assert_eq!(&unsigned[..3], &[0xdc, 0, 17]); + Ok(()) + } + + #[test] + fn contracts_refuse_substitution_and_manufactured_states() { + let mut value = anchor(); + value.binding_authority = "idunn".into(); + assert!(value.validate().is_err()); + let mut value = anchor(); + value.signer_identity_id = "caller-selected".into(); + assert!(value.validate().is_err()); + let mut value = projection(); + value.reason_code = "release_drift".into(); + assert!(value.validate().is_err()); + let mut value = projection(); + value.provider_state = "missing".into(); + assert!(value.validate().is_err()); + let mut value = projection(); + value.deployment_id = None; + assert!(value.validate().is_err()); + let mut value = projection(); + value.signature.clear(); + assert!(value.validate().is_err()); + let mut value = projection(); + value.signed_health_sha256 = format!("sha256-{}", "A".repeat(64)); + assert!(value.validate().is_err()); + let mut value = signed_health(); + value.publisher_sequence = 0; + assert!(value.validate().is_err()); + let mut value = signed_health(); + value.activation_witness_sha256 = Some(format!("sha256-{}", "F".repeat(64))); + assert!(value.validate().is_err()); + let mut value = signed_health(); + value.release_witness_sha256 = None; + assert!(value.validate().is_err()); + let mut value = signed_health(); + value.private_state_exposed = true; + assert!(value.validate().is_err()); + } +} diff --git a/packages/cultnet-rs/src/replication.rs b/packages/cultnet-rs/src/replication.rs index d02a141f..7073f1a9 100644 --- a/packages/cultnet-rs/src/replication.rs +++ b/packages/cultnet-rs/src/replication.rs @@ -540,7 +540,7 @@ impl CultNetDocumentRegistry { }) } - fn raw_document_record_from_envelope( + pub fn raw_document_record_from_envelope( &self, envelope: &CultCacheEnvelope, ) -> Result { diff --git a/packages/cultnet-rs/src/rudp.rs b/packages/cultnet-rs/src/rudp.rs index ae1ef00b..174739be 100644 --- a/packages/cultnet-rs/src/rudp.rs +++ b/packages/cultnet-rs/src/rudp.rs @@ -6,8 +6,9 @@ use std::collections::VecDeque; use std::io::ErrorKind; use std::net::SocketAddr; use std::net::UdpSocket; -use std::time::SystemTime; -use std::time::UNIX_EPOCH; +use std::thread; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; +use uuid::Uuid; use crate::CultNetMessage; use crate::CultNetReconnectController; @@ -30,7 +31,21 @@ const RUDP_MAGIC: [u8; 4] = [0x43, 0x4e, 0x52, 0x30]; const RUDP_VERSION: u8 = 0; const RUDP_FIXED_HEADER_BYTES: usize = 36; pub const CULTNET_RUDP_DEFAULT_MAX_PAYLOAD_BYTES: usize = 16 * 1024 * 1024; +pub const CULTNET_RUDP_RELIABLE_SEND_WINDOW_PACKETS: usize = 32; const RUDP_RECEIVED_SEQUENCE_WINDOW: usize = 4_096; +/// Live media tolerates loss but not lateness: a frame retransmitted after this +/// long describes a moment that has already passed. State channels never +/// expire, so this default applies only to the media channel. +const DEFAULT_MEDIA_RELIABLE_EXPIRE_AFTER_MS: u64 = 75; +/// A peer chooses its own channel ids and its own send order, so the per-channel +/// maps and the out-of-order hold buffer are both attacker-sized unless capped. +const RUDP_MAX_TRACKED_CHANNELS: usize = 64; +/// A peer cannot be trusted to pick the next sequence honestly. One packet far +/// ahead of the current highest would otherwise advance the receive state past +/// every legitimate sequence still in flight and starve the session. +const RUDP_RECEIVE_AHEAD_WINDOW: u32 = 1_024; +const RUDP_MAX_ORDERED_BUFFERED_FRAMES: usize = 1_024; +const RUDP_MAX_ORDERED_BUFFERED_BYTES: usize = 4 * 1024 * 1024; #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum CultNetRudpPacketType { @@ -70,6 +85,7 @@ pub struct CultNetRudpDeliveredFrame { #[derive(Clone, Debug, PartialEq, Eq)] pub struct CultNetRudpReceiveResult { pub delivered: Vec, + pub ready_to_send: Vec, pub reply: Option, pub pong: bool, pub pong_payload: Vec, @@ -102,12 +118,40 @@ pub struct CultNetRudpSendOptions { pub ordered: bool, pub sequenced: bool, pub now_ms: u64, + /// Drop a reliable packet instead of retransmitting it once this many + /// milliseconds have passed since the send call. `None` retransmits until + /// acknowledged, which is correct for state and wrong for live media, where + /// a late frame is worse than a missing one. + pub reliable_expire_after_ms: Option, +} + +/// Identifies every transport packet belonging to one non-expiring reliable +/// send. Receipts are local to the session that issued them and cannot survive +/// a peer reset. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CultNetRudpReliableSendReceipt { + session_scope: Uuid, + sequences: Vec, +} + +impl CultNetRudpReliableSendReceipt { + pub fn packet_sequences(&self) -> &[u32] { + &self.sequences + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum CultNetRudpReliableSendStatus { + Pending, + Acknowledged, + Invalidated, } #[derive(Clone, Debug, PartialEq, Eq)] struct PendingReliablePacket { packet: CultNetRudpPacket, last_sent_at_ms: u64, + expires_at_ms: Option, } #[derive(Clone, Debug, PartialEq, Eq)] @@ -123,9 +167,14 @@ struct FragmentBuffer { fragment_count: u16, payloads: BTreeMap>, sequences: BTreeMap, + /// Session-local arrival ordinal of the newest fragment in this set. A + /// set that stops being touched is one that lost a fragment; it is the + /// first to go when the pending-set bound is reached. + last_touched: u64, } pub struct CultNetRudpSession { + session_scope: Uuid, connection_id: u32, resend_delay_ms: u64, max_pending_reliable_packets: Option, @@ -133,20 +182,27 @@ pub struct CultNetRudpSession { max_pending_fragment_sets: usize, initial_sequence: u32, next_sequence: u32, + next_sequenced_by_channel: BTreeMap, next_fragment_id: u16, connected: bool, last_received_at_ms: Option, highest_received_sequence: Option, received_sequences: BTreeSet, + latest_sequenced_by_channel: BTreeMap, pending_reliable: BTreeMap, + queued_reliable: VecDeque<(CultNetRudpPacket, Option)>, + reliable_packets_expired: u64, ordered_next_sequence_by_channel: BTreeMap, ordered_buffers: BTreeMap>, fragment_buffers: BTreeMap<(String, u16), FragmentBuffer>, + fragment_touches: u64, + fragment_sets_evicted: u64, } impl CultNetRudpSession { pub fn new(options: CultNetRudpSessionOptions) -> Self { Self { + session_scope: Uuid::new_v4(), connection_id: options.connection_id, resend_delay_ms: options.resend_delay_ms, max_pending_reliable_packets: options.max_pending_reliable_packets, @@ -154,15 +210,21 @@ impl CultNetRudpSession { max_pending_fragment_sets: 64, initial_sequence: options.initial_sequence, next_sequence: options.initial_sequence, + next_sequenced_by_channel: BTreeMap::new(), next_fragment_id: 1, connected: false, last_received_at_ms: None, highest_received_sequence: None, received_sequences: BTreeSet::new(), + latest_sequenced_by_channel: BTreeMap::new(), pending_reliable: BTreeMap::new(), + queued_reliable: VecDeque::new(), + reliable_packets_expired: 0, ordered_next_sequence_by_channel: BTreeMap::new(), ordered_buffers: BTreeMap::new(), fragment_buffers: BTreeMap::new(), + fragment_touches: 0, + fragment_sets_evicted: 0, } } @@ -178,10 +240,37 @@ impl CultNetRudpSession { self.connected } + pub fn assume_connected(&mut self, now_ms: u64) { + self.connected = true; + self.last_received_at_ms = Some(now_ms); + } + pub fn pending_reliable_sequences(&self) -> Vec { self.pending_reliable.keys().copied().collect() } + /// Reliable packets dropped by their expiry deadline rather than + /// acknowledged. A rising count means the peer is not keeping up. + pub fn reliable_packets_expired(&self) -> u64 { + self.reliable_packets_expired + } + + /// Incomplete fragment sets discarded to admit a newer one. Each is a + /// payload that lost at least one fragment on the wire; a rising count is + /// the link's loss rate multiplied by the fragment count, and the reason + /// media payloads should fit in one packet. + pub fn fragment_sets_evicted(&self) -> u64 { + self.fragment_sets_evicted + } + + pub fn queued_reliable_packet_count(&self) -> usize { + self.queued_reliable.len() + } + + pub fn outstanding_reliable_packet_count(&self) -> usize { + self.pending_reliable.len() + self.queued_reliable.len() + } + fn pending_accept_for_resend(&mut self, now_ms: u64) -> Option { let pending = self .pending_reliable @@ -191,6 +280,39 @@ impl CultNetRudpSession { Some(pending.packet.clone()) } + fn reliable_send_status( + &self, + receipt: &CultNetRudpReliableSendReceipt, + ) -> CultNetRudpReliableSendStatus { + if receipt.session_scope != self.session_scope { + return CultNetRudpReliableSendStatus::Invalidated; + } + if receipt + .sequences + .iter() + .any(|sequence| self.pending_reliable.contains_key(sequence)) + { + CultNetRudpReliableSendStatus::Pending + } else { + CultNetRudpReliableSendStatus::Acknowledged + } + } + + fn non_expiring_reliable_receipt( + &self, + packets: &[CultNetRudpPacket], + ) -> Result { + if packets.is_empty() || packets.iter().any(|packet| !packet.reliable) { + return Err(anyhow!( + "RUDP acknowledgement receipts require a non-empty reliable packet set" + )); + } + Ok(CultNetRudpReliableSendReceipt { + session_scope: self.session_scope, + sequences: packets.iter().map(|packet| packet.sequence).collect(), + }) + } + pub fn last_received_at_ms(&self) -> Option { self.last_received_at_ms } @@ -213,13 +335,17 @@ impl CultNetRudpSession { } pub fn reset_peer_state(&mut self) { + self.session_scope = Uuid::new_v4(); self.next_sequence = self.initial_sequence; + self.next_sequenced_by_channel.clear(); self.next_fragment_id = 1; self.connected = false; self.last_received_at_ms = None; self.highest_received_sequence = None; self.received_sequences.clear(); + self.latest_sequenced_by_channel.clear(); self.pending_reliable.clear(); + self.queued_reliable.clear(); self.ordered_next_sequence_by_channel.clear(); self.ordered_buffers.clear(); self.fragment_buffers.clear(); @@ -235,7 +361,7 @@ impl CultNetRudpSession { true, false, ); - self.track_reliable(packet.clone(), now_ms); + self.track_reliable(packet.clone(), now_ms, None); Ok(packet) } @@ -265,7 +391,7 @@ impl CultNetRudpSession { true, false, ); - self.track_reliable(response.clone(), now_ms); + self.track_reliable(response.clone(), now_ms, None); Ok(response) } @@ -275,6 +401,13 @@ impl CultNetRudpSession { payload: Vec, options: CultNetRudpSendOptions, ) -> Result { + if options.reliable + && self.pending_reliable.len() >= CULTNET_RUDP_RELIABLE_SEND_WINDOW_PACKETS + { + return Err(anyhow!( + "RUDP reliable send window is full; receive acknowledgements before sending" + )); + } self.send_many(channel_id, payload, options, None)? .into_iter() .next() @@ -293,7 +426,16 @@ impl CultNetRudpSession { "Cannot send RUDP data before the session is connected" )); } + if options.ordered && !options.reliable { + return Err(anyhow!("RUDP ordered delivery requires reliability")); + } self.require_payload_size(payload.len())?; + // Reclaim deadline-passed sends before measuring window capacity, so an + // expiring stream cannot be refused for a backlog it no longer owns. + self.purge_expired_reliable(options.now_ms); + let expires_at_ms = options + .reliable_expire_after_ms + .map(|ttl| options.now_ms.saturating_add(ttl)); if let Some(max_fragment_bytes) = max_fragment_bytes { if max_fragment_bytes == 0 { @@ -321,12 +463,13 @@ impl CultNetRudpSession { index as u16, fragment_count as u16, ); - if packet.reliable { - self.track_reliable(packet.clone(), options.now_ms); - } packets.push(packet); } - return Ok(packets); + return Ok(if options.reliable { + self.admit_reliable_packets(packets, options.now_ms, expires_at_ms) + } else { + packets + }); } } @@ -340,7 +483,7 @@ impl CultNetRudpSession { options.sequenced, ); if packet.reliable { - self.track_reliable(packet.clone(), options.now_ms); + return Ok(self.admit_reliable_packets(vec![packet], options.now_ms, expires_at_ms)); } Ok(vec![packet]) } @@ -352,6 +495,8 @@ impl CultNetRudpSession { ) -> Result { self.require_connection(packet)?; self.apply_acknowledgements(packet); + self.purge_expired_reliable(now_ms); + let ready_to_send = self.promote_queued_reliable(now_ms); self.last_received_at_ms = Some(now_ms); let expected_sequence_if_uninitialized = self .highest_received_sequence @@ -363,6 +508,7 @@ impl CultNetRudpSession { self.connected = true; return Ok(CultNetRudpReceiveResult { delivered: Vec::new(), + ready_to_send, reply: None, pong: false, pong_payload: Vec::new(), @@ -372,9 +518,9 @@ impl CultNetRudpSession { } if packet.packet_type == CultNetRudpPacketType::Ping { - self.remember_received(packet.sequence); return Ok(CultNetRudpReceiveResult { delivered: Vec::new(), + ready_to_send, reply: Some(self.create_packet( CultNetRudpPacketType::Pong, "control", @@ -393,11 +539,9 @@ impl CultNetRudpSession { if packet.packet_type == CultNetRudpPacketType::Ack || packet.packet_type == CultNetRudpPacketType::Pong { - if packet.packet_type == CultNetRudpPacketType::Pong { - self.remember_received(packet.sequence); - } return Ok(CultNetRudpReceiveResult { delivered: Vec::new(), + ready_to_send, reply: None, pong: packet.packet_type == CultNetRudpPacketType::Pong, pong_payload: if packet.packet_type == CultNetRudpPacketType::Pong { @@ -411,10 +555,10 @@ impl CultNetRudpSession { } if packet.packet_type == CultNetRudpPacketType::Disconnect { - self.remember_received(packet.sequence); self.connected = false; return Ok(CultNetRudpReceiveResult { delivered: Vec::new(), + ready_to_send, reply: None, pong: false, pong_payload: Vec::new(), @@ -426,6 +570,7 @@ impl CultNetRudpSession { if packet.packet_type != CultNetRudpPacketType::Data { return Ok(CultNetRudpReceiveResult { delivered: Vec::new(), + ready_to_send, reply: None, pong: false, pong_payload: Vec::new(), @@ -434,15 +579,38 @@ impl CultNetRudpSession { }); } - let duplicate = self.received_sequences.contains(&packet.sequence) - || self.highest_received_sequence.is_some_and(|highest| { - packet.sequence < highest - && highest - packet.sequence >= RUDP_RECEIVED_SEQUENCE_WINDOW as u32 + let far_ahead = packet.reliable + && self.highest_received_sequence.is_some_and(|highest| { + packet.sequence > highest + && packet.sequence - highest > RUDP_RECEIVE_AHEAD_WINDOW }); - self.remember_received(packet.sequence); + if far_ahead { + // Refuse without remembering. Recording it would move the highest + // received sequence out of reach of everything still in flight. + return Ok(CultNetRudpReceiveResult { + delivered: Vec::new(), + ready_to_send, + reply: None, + pong: false, + pong_payload: Vec::new(), + disconnected: false, + disconnect_reason: Vec::new(), + }); + } + + let duplicate = packet.reliable + && (self.received_sequences.contains(&packet.sequence) + || self.highest_received_sequence.is_some_and(|highest| { + packet.sequence < highest + && highest - packet.sequence >= RUDP_RECEIVED_SEQUENCE_WINDOW as u32 + })); + if packet.reliable { + self.remember_received(packet.sequence); + } if duplicate { return Ok(CultNetRudpReceiveResult { delivered: Vec::new(), + ready_to_send, reply: None, pong: false, pong_payload: Vec::new(), @@ -454,6 +622,7 @@ impl CultNetRudpSession { let Some((frame, ordered, next_sequence)) = self.reassemble(packet)? else { return Ok(CultNetRudpReceiveResult { delivered: Vec::new(), + ready_to_send, reply: None, pong: false, pong_payload: Vec::new(), @@ -462,12 +631,33 @@ impl CultNetRudpSession { }); }; let delivered = if ordered { - self.deliver_ordered(frame, next_sequence, expected_sequence_if_uninitialized) + self.deliver_ordered(frame, next_sequence, expected_sequence_if_uninitialized)? + } else if packet.sequenced { + if !self + .latest_sequenced_by_channel + .contains_key(&frame.channel_id) + && self.latest_sequenced_by_channel.len() >= RUDP_MAX_TRACKED_CHANNELS + { + return Err(anyhow!("RUDP session has too many sequenced channels")); + } + let latest = self + .latest_sequenced_by_channel + .get(&frame.channel_id) + .copied(); + let newest_sequence = next_sequence.saturating_sub(1); + if latest.is_some_and(|latest| newest_sequence <= latest) { + Vec::new() + } else { + self.latest_sequenced_by_channel + .insert(frame.channel_id.clone(), newest_sequence); + vec![frame] + } } else { vec![frame] }; Ok(CultNetRudpReceiveResult { delivered, + ready_to_send, reply: None, pong: false, pong_payload: Vec::new(), @@ -513,6 +703,15 @@ impl CultNetRudpSession { } } + pub fn create_ack_for_received(&mut self, sequence: u32) -> CultNetRudpPacket { + let (ack, _) = self.ack_state(); + if ack >= sequence && ack - sequence <= 32 { + self.create_ack() + } else { + self.create_ack_for(sequence) + } + } + pub fn create_ping(&mut self, payload: Vec) -> CultNetRudpPacket { self.create_packet( CultNetRudpPacketType::Ping, @@ -597,11 +796,23 @@ impl CultNetRudpSession { fragment_index: u16, fragment_count: u16, ) -> CultNetRudpPacket { - let sequence = self.next_sequence; - self.next_sequence = self - .next_sequence - .checked_add(1) - .expect("sequence overflow"); + let sequence = if reliable { + let sequence = self.next_sequence; + self.next_sequence = self + .next_sequence + .saturating_add(1); + sequence + } else if sequenced { + let next = self + .next_sequenced_by_channel + .entry(channel_id.to_string()) + .or_insert(1); + let sequence = *next; + *next = next.saturating_add(1); + sequence + } else { + 0 + }; let (ack, ack_mask) = self.ack_state(); CultNetRudpPacket { packet_type, @@ -620,25 +831,88 @@ impl CultNetRudpSession { } } - fn track_reliable(&mut self, packet: CultNetRudpPacket, now_ms: u64) { + fn track_reliable( + &mut self, + packet: CultNetRudpPacket, + now_ms: u64, + expires_at_ms: Option, + ) { self.pending_reliable.insert( packet.sequence, PendingReliablePacket { packet, last_sent_at_ms: now_ms, + expires_at_ms, }, ); } + /// Drop pending and queued reliable packets past their deadline. The + /// deadline is fixed when the send is admitted, so time spent waiting in + /// the backlog counts against it exactly as time spent awaiting an ACK does. + fn purge_expired_reliable(&mut self, now_ms: u64) { + let before = self.pending_reliable.len() + self.queued_reliable.len(); + self.pending_reliable + .retain(|_, pending| pending.expires_at_ms.is_none_or(|at| now_ms <= at)); + self.queued_reliable + .retain(|(_, expires_at_ms)| expires_at_ms.is_none_or(|at| now_ms <= at)); + let after = self.pending_reliable.len() + self.queued_reliable.len(); + self.reliable_packets_expired = self + .reliable_packets_expired + .saturating_add((before - after) as u64); + } + + fn admit_reliable_packets( + &mut self, + packets: Vec, + now_ms: u64, + expires_at_ms: Option, + ) -> Vec { + let available = + CULTNET_RUDP_RELIABLE_SEND_WINDOW_PACKETS.saturating_sub(self.pending_reliable.len()); + let mut ready = Vec::with_capacity(available.min(packets.len())); + for packet in packets { + if ready.len() < available { + self.track_reliable(packet.clone(), now_ms, expires_at_ms); + ready.push(packet); + } else { + self.queued_reliable.push_back((packet, expires_at_ms)); + } + } + ready + } + + fn promote_queued_reliable(&mut self, now_ms: u64) -> Vec { + let available = + CULTNET_RUDP_RELIABLE_SEND_WINDOW_PACKETS.saturating_sub(self.pending_reliable.len()); + let mut ready = Vec::with_capacity(available.min(self.queued_reliable.len())); + for _ in 0..available { + let Some((packet, expires_at_ms)) = self.queued_reliable.pop_front() else { + break; + }; + self.track_reliable(packet.clone(), now_ms, expires_at_ms); + ready.push(packet); + } + ready + } + fn ensure_reliable_capacity(&self, packet_count: usize) -> Result<()> { if packet_count == 0 { return Ok(()); } if let Some(limit) = self.max_pending_reliable_packets { - if self.pending_reliable.len() + packet_count > limit { + if self.outstanding_reliable_packet_count() + packet_count > limit { return Err(anyhow!("RUDP reliable send queue is full")); } } + // Refuse before allocation rather than at it. A session that runs out of + // reliable sequence space must report it, not abort the process. + let required = u32::try_from(packet_count) + .ok() + .filter(|required| self.next_sequence.checked_add(*required).is_some()); + if required.is_none() { + return Err(anyhow!("RUDP reliable sequence space is exhausted")); + } Ok(()) } @@ -710,7 +984,19 @@ impl CultNetRudpSession { if !self.fragment_buffers.contains_key(&key) && self.fragment_buffers.len() >= self.max_pending_fragment_sets { - return Err(anyhow!("RUDP pending fragment-set limit reached")); + // A set that is still incomplete when the bound is reached has + // almost certainly lost a fragment; its siblings were delivered + // long ago. Drop the one untouched longest so the session keeps + // receiving. Erroring here made any lossy fragmenting channel a + // countdown to a dead receiver. + let stalest = self + .fragment_buffers + .iter() + .min_by_key(|(_, buffer)| buffer.last_touched) + .map(|(key, _)| key.clone()) + .expect("bound reached implies a pending set exists"); + self.fragment_buffers.remove(&stalest); + self.fragment_sets_evicted += 1; } let buffered_bytes = self .fragment_buffers @@ -737,12 +1023,15 @@ impl CultNetRudpSession { fragment_count: packet.fragment_count, payloads: BTreeMap::new(), sequences: BTreeMap::new(), + last_touched: 0, }); if buffer.fragment_count != packet.fragment_count || buffer.ordered != packet.ordered { return Err(anyhow!( "RUDP fragment metadata changed within a fragment set" )); } + self.fragment_touches += 1; + buffer.last_touched = self.fragment_touches; buffer .payloads .insert(packet.fragment_index, packet.payload.clone()); @@ -789,13 +1078,30 @@ impl CultNetRudpSession { Ok(()) } + fn ordered_buffered_frames(&self) -> usize { + self.ordered_buffers.values().map(BTreeMap::len).sum() + } + + fn ordered_buffered_bytes(&self) -> usize { + self.ordered_buffers + .values() + .flat_map(BTreeMap::values) + .map(|pending| pending.frame.payload.len()) + .sum() + } + fn deliver_ordered( &mut self, frame: CultNetRudpDeliveredFrame, next_sequence_after_frame: u32, expected_sequence_if_uninitialized: u32, - ) -> Vec { + ) -> Result> { let channel_id = frame.channel_id.clone(); + if !self.ordered_next_sequence_by_channel.contains_key(&channel_id) + && self.ordered_next_sequence_by_channel.len() >= RUDP_MAX_TRACKED_CHANNELS + { + return Err(anyhow!("RUDP session has too many ordered channels")); + } let mut next = if let Some(next) = self .ordered_next_sequence_by_channel .get(&channel_id) @@ -823,10 +1129,16 @@ impl CultNetRudpSession { } if frame.sequence < next { - return Vec::new(); + return Ok(Vec::new()); } if frame.sequence > next { + if self.ordered_buffered_frames() >= RUDP_MAX_ORDERED_BUFFERED_FRAMES + || self.ordered_buffered_bytes() + frame.payload.len() + > RUDP_MAX_ORDERED_BUFFERED_BYTES + { + return Err(anyhow!("RUDP ordered hold buffer is full")); + } self.ordered_buffers.entry(channel_id).or_default().insert( frame.sequence, PendingOrderedFrame { @@ -834,14 +1146,14 @@ impl CultNetRudpSession { next_sequence: next_sequence_after_frame, }, ); - return Vec::new(); + return Ok(Vec::new()); } self.ordered_next_sequence_by_channel .insert(channel_id.clone(), next_sequence_after_frame); let mut delivered = vec![frame]; delivered.extend(self.drain_ordered(&channel_id)); - delivered + Ok(delivered) } fn drain_ordered(&mut self, channel_id: &str) -> Vec { @@ -890,7 +1202,9 @@ impl CultNetRudpSession { fn allocate_fragment_id(&mut self) -> u16 { let fragment_id = self.next_fragment_id; - self.next_fragment_id = self.next_fragment_id.saturating_add(1); + // Wrap, never saturate: `saturating_add` parked every set after the + // 65,535th on one shared id, and the reassembler stitched them together. + self.next_fragment_id = self.next_fragment_id.wrapping_add(1); if self.next_fragment_id == 0 { self.next_fragment_id = 1; } @@ -928,6 +1242,9 @@ pub struct CultNetRudpSocketTransportOptions { pub max_fragment_bytes: Option, pub max_pending_reliable_packets: Option, pub reconnect_policy: Option, + pub media_reliable_expire_after_ms: Option, + /// `None` means the historical default, `Reliable`. + pub media_delivery: Option, } impl CultNetRudpSocketTransportOptions { @@ -938,6 +1255,8 @@ impl CultNetRudpSocketTransportOptions { connection_id: u32, ) -> Self { Self { + media_reliable_expire_after_ms: Some(DEFAULT_MEDIA_RELIABLE_EXPIRE_AFTER_MS), + media_delivery: None, runtime_id: runtime_id.into(), socket, mode: CultNetRudpSocketMode::Client, @@ -955,6 +1274,8 @@ impl CultNetRudpSocketTransportOptions { pub fn server(runtime_id: impl Into, socket: UdpSocket, connection_id: u32) -> Self { Self { + media_reliable_expire_after_ms: Some(DEFAULT_MEDIA_RELIABLE_EXPIRE_AFTER_MS), + media_delivery: None, runtime_id: runtime_id.into(), socket, mode: CultNetRudpSocketMode::Server, @@ -995,11 +1316,16 @@ pub struct CultNetRudpServerHubOptions { pub max_fragment_bytes: Option, pub max_pending_reliable_packets: Option, pub max_peers: usize, + pub media_reliable_expire_after_ms: Option, + /// `None` means the historical default, `Reliable`. + pub media_delivery: Option, } impl CultNetRudpServerHubOptions { pub fn new(runtime_id: impl Into, socket: UdpSocket, connection_id: u32) -> Self { Self { + media_reliable_expire_after_ms: Some(DEFAULT_MEDIA_RELIABLE_EXPIRE_AFTER_MS), + media_delivery: None, runtime_id: runtime_id.into(), socket, connection_id, @@ -1085,6 +1411,8 @@ impl CultNetRudpServerHub { let profile = create_rudp_transport_profile( options.runtime_id, RudpTransportProfileOptions { + media_reliable_expire_after_ms: options.media_reliable_expire_after_ms, + media_delivery: options.media_delivery, transport_id: options.transport_id, host: Some(local_addr.ip().to_string()), port: Some(local_addr.port()), @@ -1119,7 +1447,18 @@ impl CultNetRudpServerHub { } pub fn stats(&self) -> CultNetTransportStats { - self.stats.clone() + let mut stats = self.stats.clone(); + stats.reliable_packets_expired = self + .peers + .values() + .map(|peer| peer.session.reliable_packets_expired()) + .fold(0_u64, |total, expired| total.saturating_add(expired)); + stats.fragment_sets_evicted = self + .peers + .values() + .map(|peer| peer.session.fragment_sets_evicted()) + .fold(0_u64, |total, evicted| total.saturating_add(evicted)); + stats } pub fn sessions(&self) -> Vec { @@ -1154,7 +1493,12 @@ impl CultNetRudpServerHub { peer.session.send_many( channel_id, payload, - channel_send_options(channel_id, now_ms()), + channel_send_options( + channel_id, + now_ms(), + channel_reliable_expire_after_ms(&self.profile, channel_id), + channel_delivery(&self.profile, channel_id), + ), self.max_fragment_bytes, )? }; @@ -1194,18 +1538,35 @@ impl CultNetRudpServerHub { Ok(true) } + /// The next event, or `None` only when nothing is waiting on the socket. + /// + /// Packets that carry no event for the caller — acknowledgements, pings, + /// a repeated Connect, a fragment that completes nothing yet — are + /// consumed here and reading continues. Returning `None` after one of + /// those made a caller that polls once per send fall behind its own + /// peers' acknowledgements, and a reliable channel then resent everything. pub fn receive_event_once(&mut self) -> Result> { - if let Some(event) = self.pending_events.pop_front() { - return Ok(Some(event)); + loop { + if let Some(event) = self.pending_events.pop_front() { + return Ok(Some(event)); + } + if !self.receive_packet_once()? { + return Ok(None); + } } + } + /// Reads one datagram and applies it. `false` when the socket is empty. + fn receive_packet_once(&mut self) -> Result { let mut wire = vec![0_u8; 65_535]; let (received, remote_addr) = match self.socket.recv_from(&mut wire) { Ok(value) => value, Err(error) - if error.kind() == ErrorKind::WouldBlock || error.kind() == ErrorKind::TimedOut => + if error.kind() == ErrorKind::WouldBlock + || error.kind() == ErrorKind::TimedOut + || error.kind() == ErrorKind::ConnectionReset => { - return Ok(None); + return Ok(false); } Err(error) => return Err(error.into()), }; @@ -1213,7 +1574,7 @@ impl CultNetRudpServerHub { self.stats.bytes_received += received as u64; let packet = decode_rudp_packet(&wire)?; if packet.connection_id != self.connection_id { - return Ok(None); + return Ok(true); } if packet.packet_type == CultNetRudpPacketType::Connect { @@ -1224,9 +1585,9 @@ impl CultNetRudpServerHub { let reply = peer .session .pending_accept_for_resend(now_ms()) - .unwrap_or_else(|| peer.session.create_ack_for(packet.sequence)); + .unwrap_or_else(|| peer.session.create_ack()); self.send_packet(remote_addr, &reply)?; - return Ok(None); + return Ok(true); } if !self.peers.contains_key(&remote_addr) && self.peers.len() >= self.max_peers { return Err(anyhow!("RUDP server hub peer limit reached")); @@ -1267,22 +1628,25 @@ impl CultNetRudpServerHub { } self.pending_events .push_back(CultNetRudpServerEvent::Connected { session: context }); - return Ok(self.pending_events.pop_front()); + return Ok(true); } let Some(peer) = self.peers.get_mut(&remote_addr) else { - return Ok(None); + return Ok(true); }; let result = peer.session.receive(&packet, now_ms())?; let context = peer.context.clone(); let ack = if packet.reliable { - Some(peer.session.create_ack_for(packet.sequence)) + Some(peer.session.create_ack_for_received(packet.sequence)) } else { None }; if let Some(reply) = result.reply { self.send_packet(remote_addr, &reply)?; } + for ready in result.ready_to_send { + self.send_packet(remote_addr, &ready)?; + } if let Some(ack) = ack { self.send_packet(remote_addr, &ack)?; } @@ -1311,7 +1675,7 @@ impl CultNetRudpServerHub { }); self.peers.remove(&remote_addr); } - Ok(self.pending_events.pop_front()) + Ok(true) } pub fn poll_resends(&mut self) -> Result<()> { @@ -1368,6 +1732,8 @@ impl CultNetRudpSocketTransportConnection { let profile = create_rudp_transport_profile( options.runtime_id, RudpTransportProfileOptions { + media_reliable_expire_after_ms: options.media_reliable_expire_after_ms, + media_delivery: options.media_delivery, transport_id: options.transport_id, host: Some(local_addr.ip().to_string()), port: Some(local_addr.port()), @@ -1406,7 +1772,10 @@ impl CultNetRudpSocketTransportConnection { } pub fn stats(&self) -> CultNetTransportStats { - self.stats.clone() + let mut stats = self.stats.clone(); + stats.reliable_packets_expired = self.session.reliable_packets_expired(); + stats.fragment_sets_evicted = self.session.fragment_sets_evicted(); + stats } pub fn disconnect_reason(&self) -> Option<&[u8]> { @@ -1428,17 +1797,69 @@ impl CultNetRudpSocketTransportConnection { } pub fn send(&mut self, channel_id: &str, payload: Vec) -> Result<()> { - let options = channel_send_options(channel_id, now_ms()); + let options = + channel_send_options( + channel_id, + now_ms(), + channel_reliable_expire_after_ms(&self.profile, channel_id), + channel_delivery(&self.profile, channel_id), + ); let packets = self.session .send_many(channel_id, payload, options, self.max_fragment_bytes)?; - for packet in packets { - self.send_packet(&packet)?; - } + self.send_data_packets(&packets)?; self.stats.frames_sent += 1; Ok(()) } + /// Sends one frame on a reliable channel whose packets cannot expire and + /// returns the transport receipt used to observe its exact acknowledgement. + pub fn send_reliable( + &mut self, + channel_id: &str, + payload: Vec, + ) -> Result { + let options = + channel_send_options( + channel_id, + now_ms(), + channel_reliable_expire_after_ms(&self.profile, channel_id), + channel_delivery(&self.profile, channel_id), + ); + // This lineage has no reliable-expiry concept, so every reliable channel is + // already non-expiring. Restore the expiry half of this guard if one is added. + if !options.reliable { + return Err(anyhow!( + "RUDP acknowledgement receipts require a non-expiring reliable channel" + )); + } + let packets = + self.session + .send_many(channel_id, payload, options, self.max_fragment_bytes)?; + self.send_data_packets(&packets)?; + let receipt = self.session.non_expiring_reliable_receipt(&packets)?; + self.stats.frames_sent += 1; + Ok(receipt) + } + + pub fn reliable_send_status( + &self, + receipt: &CultNetRudpReliableSendReceipt, + ) -> CultNetRudpReliableSendStatus { + self.session.reliable_send_status(receipt) + } + + fn send_data_packets(&mut self, packets: &[CultNetRudpPacket]) -> Result<()> { + let fragmented = packets.len() > 1; + for (index, packet) in packets.iter().enumerate() { + self.send_packet(packet)?; + if fragmented && (index + 1) % 16 == 0 { + thread::sleep(Duration::from_millis(1)); + } + } + Ok(()) + } + pub fn send_schema_message(&mut self, message: &CultNetMessage) -> Result<()> { let payload = encode_cultnet_message_to_vec(message, CultNetWireContract::CultNetSchemaV0)?; self.send("schema", payload) @@ -1474,18 +1895,75 @@ impl CultNetRudpSocketTransportConnection { self.session.check_timeout(now_ms(), timeout_ms) } - pub fn receive_once(&mut self) -> Result> { - if let Some(frame) = self.delivered_frames.pop_front() { - return Ok(Some(frame)); - } + pub fn pending_reliable_packet_count(&self) -> usize { + self.session.pending_reliable_sequences().len() + } + pub fn outstanding_reliable_packet_count(&self) -> usize { + self.session.outstanding_reliable_packet_count() + } + + pub fn flush_reliable(&mut self, timeout: Duration) -> Result<()> { + let original_timeout = self.socket.read_timeout()?; + let poll_timeout = original_timeout + .map(|configured| configured.min(Duration::from_millis(10))) + .unwrap_or(Duration::from_millis(10)); + self.socket.set_read_timeout(Some(poll_timeout))?; + + let deadline = Instant::now() + timeout; + let mut preserved_frames = VecDeque::new(); + let result = (|| { + while self.outstanding_reliable_packet_count() > 0 { + if Instant::now() >= deadline { + return Err(anyhow!( + "RUDP reliable flush timed out with {} packets outstanding", + self.outstanding_reliable_packet_count() + )); + } + if let Some(frame) = self.receive_once()? { + preserved_frames.push_back(frame); + } + self.poll_resends()?; + } + Ok(()) + })(); + + preserved_frames.append(&mut self.delivered_frames); + self.delivered_frames = preserved_frames; + let restore = self.socket.set_read_timeout(original_timeout); + match (result, restore) { + (Err(error), _) => Err(error), + (Ok(()), Err(error)) => Err(error.into()), + (Ok(()), Ok(())) => Ok(()), + } + } + + /// The next delivered frame, or `None` only when nothing is waiting on + /// the socket. Acknowledgements, pongs and incomplete fragments are + /// consumed and reading continues; see the hub's `receive_event_once`. + pub fn receive_once(&mut self) -> Result> { + loop { + if let Some(frame) = self.delivered_frames.pop_front() { + return Ok(Some(frame)); + } + if !self.receive_packet_once()? { + return Ok(None); + } + } + } + + /// Reads one datagram and applies it. `false` when the socket is empty + /// or the session has been disconnected. + fn receive_packet_once(&mut self) -> Result { let mut wire = vec![0_u8; 65_535]; let (received, remote_addr) = match self.socket.recv_from(&mut wire) { Ok(value) => value, Err(error) - if error.kind() == ErrorKind::WouldBlock || error.kind() == ErrorKind::TimedOut => + if error.kind() == ErrorKind::WouldBlock + || error.kind() == ErrorKind::TimedOut + || error.kind() == ErrorKind::ConnectionReset => { - return Ok(None); + return Ok(false); } Err(error) => return Err(error.into()), }; @@ -1500,14 +1978,14 @@ impl CultNetRudpSocketTransportConnection { { self.remote_addr = Some(remote_addr); } else { - return Ok(None); + return Ok(true); } } } else { if self.mode == CultNetRudpSocketMode::Server && packet.packet_type != CultNetRudpPacketType::Connect { - return Ok(None); + return Ok(true); } self.remote_addr = Some(remote_addr); } @@ -1517,19 +1995,22 @@ impl CultNetRudpSocketTransportConnection { self.session.reset_peer_state(); let accept = self.session.accept_connect(&packet, now_ms(), Vec::new())?; self.send_packet(&accept)?; - return Ok(None); + return Ok(true); } let result = self.session.receive(&packet, now_ms())?; if let Some(reply) = result.reply { self.send_packet(&reply)?; } + for ready in result.ready_to_send { + self.send_packet(&ready)?; + } if result.pong { self.pong_payloads.push_back(result.pong_payload); } if result.disconnected { self.disconnect_reason = Some(result.disconnect_reason); - return Ok(None); + return Ok(false); } for frame in result.delivered { @@ -1539,13 +2020,14 @@ impl CultNetRudpSocketTransportConnection { }); self.stats.frames_received += 1; } - let frame = self.delivered_frames.pop_front(); - if packet.reliable || packet.packet_type == CultNetRudpPacketType::Accept || frame.is_some() - { - let ack = self.session.create_ack_for(packet.sequence); + // Only what the sender is waiting on is acknowledged. An unreliable + // frame carries no sequence, so an ACK for it tells the sender + // nothing and costs it one datagram to read for every one it sent. + if packet.reliable || packet.packet_type == CultNetRudpPacketType::Accept { + let ack = self.session.create_ack_for_received(packet.sequence); self.send_packet(&ack)?; } - Ok(frame) + Ok(true) } pub fn poll_resends(&mut self) -> Result<()> { @@ -1669,6 +2151,9 @@ pub struct RudpTransportProfileOptions { pub max_fragment_bytes: Option, pub max_pending_reliable_packets: Option, pub reconnect_policy: Option, + pub media_reliable_expire_after_ms: Option, + /// `None` means the historical default, `Reliable`. + pub media_delivery: Option, } pub fn create_rudp_transport_profile( @@ -1699,6 +2184,7 @@ pub fn create_rudp_transport_profile( max_payload_bytes: options.max_payload_bytes, max_fragment_bytes: options.max_fragment_bytes, max_pending_reliable_packets: options.max_pending_reliable_packets, + reliable_expire_after_ms: None, }, CultNetTransportChannel { channel_id: "latest".to_string(), @@ -1707,6 +2193,7 @@ pub fn create_rudp_transport_profile( max_payload_bytes: options.max_payload_bytes, max_fragment_bytes: options.max_fragment_bytes, max_pending_reliable_packets: options.max_pending_reliable_packets, + reliable_expire_after_ms: None, }, CultNetTransportChannel { channel_id: "realtime".to_string(), @@ -1715,14 +2202,29 @@ pub fn create_rudp_transport_profile( max_payload_bytes: options.max_payload_bytes, max_fragment_bytes: options.max_fragment_bytes, max_pending_reliable_packets: options.max_pending_reliable_packets, + reliable_expire_after_ms: None, }, CultNetTransportChannel { channel_id: "media".to_string(), + delivery: options + .media_delivery + .unwrap_or(CultNetTransportDelivery::Reliable), + ordering: CultNetTransportOrdering::Unordered, + max_payload_bytes: options.max_payload_bytes, + max_fragment_bytes: options.max_fragment_bytes, + max_pending_reliable_packets: options.max_pending_reliable_packets, + reliable_expire_after_ms: options.media_reliable_expire_after_ms, + }, + // Audio beside lossy video on one session: reliable, but a packet + // older than the media expiry is not worth a retransmit. + CultNetTransportChannel { + channel_id: "audio".to_string(), delivery: CultNetTransportDelivery::Reliable, ordering: CultNetTransportOrdering::Unordered, max_payload_bytes: options.max_payload_bytes, max_fragment_bytes: options.max_fragment_bytes, max_pending_reliable_packets: options.max_pending_reliable_packets, + reliable_expire_after_ms: options.media_reliable_expire_after_ms, }, ], }], @@ -1831,31 +2333,77 @@ fn packet_type_to_code(packet_type: CultNetRudpPacketType) -> u8 { } } -fn channel_send_options(channel_id: &str, now_ms: u64) -> CultNetRudpSendOptions { +/// The advertised profile owns each channel's expiry. Resolving it here keeps +/// one answer for what a channel does, rather than a runtime field that can +/// disagree with the profile the peer was shown. +fn channel_reliable_expire_after_ms( + profile: &CultNetTransportProfile, + channel_id: &str, +) -> Option { + profile + .transports + .iter() + .flat_map(|transport| transport.channels.iter()) + .find(|channel| channel.channel_id == channel_id) + .and_then(|channel| channel.reliable_expire_after_ms) +} + +/// The advertised profile owns each channel's delivery, for the same reason it +/// owns expiry: a send path that hardcodes reliability can disagree with the +/// profile the peer was shown, and then no one can say what the channel does. +/// +/// Falls back to `Reliable` for a channel the profile does not describe, which +/// is what every channel got before delivery was configurable. +fn channel_delivery( + profile: &CultNetTransportProfile, + channel_id: &str, +) -> CultNetTransportDelivery { + profile + .transports + .iter() + .flat_map(|transport| transport.channels.iter()) + .find(|channel| channel.channel_id == channel_id) + .map(|channel| channel.delivery) + .unwrap_or(CultNetTransportDelivery::Reliable) +} + +fn channel_send_options( + channel_id: &str, + now_ms: u64, + reliable_expire_after_ms: Option, + delivery: CultNetTransportDelivery, +) -> CultNetRudpSendOptions { match channel_id { "schema" => CultNetRudpSendOptions { reliable: true, ordered: true, sequenced: false, now_ms, + reliable_expire_after_ms: None, }, "latest" => CultNetRudpSendOptions { reliable: false, ordered: false, sequenced: true, now_ms, + reliable_expire_after_ms: None, }, - "media" => CultNetRudpSendOptions { - reliable: true, + // The one channel whose delivery a caller chooses. Media that has gone + // stale is worth dropping, not retransmitting: a reliable media channel + // under loss adds load exactly when the link has least to give. + "media" | "audio" => CultNetRudpSendOptions { + reliable: matches!(delivery, CultNetTransportDelivery::Reliable), ordered: false, sequenced: false, now_ms, + reliable_expire_after_ms, }, _ => CultNetRudpSendOptions { reliable: false, ordered: false, sequenced: false, now_ms, + reliable_expire_after_ms: None, }, } } diff --git a/packages/cultnet-rs/src/runtime_authority_contracts.rs b/packages/cultnet-rs/src/runtime_authority_contracts.rs new file mode 100644 index 00000000..426be1fb --- /dev/null +++ b/packages/cultnet-rs/src/runtime_authority_contracts.rs @@ -0,0 +1,3309 @@ +use anyhow::{Context, Result, bail, ensure}; +use cultcache_rs::DatabaseEntry; +use ed25519_dalek::{Signature, Signer, SigningKey, Verifier, VerifyingKey}; +use rand::RngCore; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use std::io::{Read, Write}; +use zeroize::Zeroizing; + +pub const GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA: &str = "gamecult.runtime_presence_health.v2"; +pub const GAMECULT_RUNTIME_PRESENCE_HEALTH_SIGNING_PURPOSE: &str = + "gamecult.runtime_presence_health.v2"; +pub const GAMECULT_RUNTIME_ACTIVATION_PROOF_SIGNING_PURPOSE: &str = + "gamecult.runtime_presence.activation-proof.v1"; +pub const IDUNN_RUNTIME_ACTIVATION_SCHEMA: &str = "idunn.runtime_activation.v2"; +pub const IDUNN_RUNTIME_ACTIVATION_SIGNING_PURPOSE: &str = "idunn.runtime_activation.v2"; +pub const IDUNN_RUNTIME_ACTIVATION_CREDENTIAL_NAME: &str = "gamecult-idunn-runtime-activation-key"; +pub const IDUNN_PROCESS_WRITE_LEASE_SCHEMA: &str = "idunn.process_write_lease.v1"; +pub const IDUNN_EXPECTED_INCARNATION_SCHEMA: &str = "idunn.expected_incarnation.v2"; +pub const ODIN_RUNTIME_TOPOLOGY_CORRELATION_SCHEMA: &str = "odin.runtime_topology_correlation.v2"; +pub const ODIN_RUNTIME_TOPOLOGY_CORRELATION_SIGNING_PURPOSE: &str = + "odin.runtime_topology_correlation.v2"; + +const MAX_RUNTIME_CAPABILITIES: usize = 256; +const MAX_RUNTIME_DEPENDENCIES: usize = 256; +const MAX_TOPOLOGY_DISAGREEMENTS: usize = 256; +const IDUNN_RUNTIME_ACTIVATION_ID_DOMAIN: &[u8] = b"idunn.runtime-activation.id.v1\0"; +const IDUNN_RUNTIME_ACTIVATION_SIGNATURE_DOMAIN: &[u8] = b"idunn.runtime-activation.signature.v1\0"; + +/// One provider-owned capability claim inside a runtime-presence statement. +/// The enclosing vector is canonical only when these records are strictly +/// increasing by `(capability, schema, compatibility)`. Capacity describes +/// that one claim and does not create a second identity for the same claim. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct GameCultRuntimeCapability { + pub capability: String, + pub schema: String, + pub compatibility: String, + pub capacity: u32, +} + +impl GameCultRuntimeCapability { + fn identity(&self) -> (&str, &str, &str) { + (&self.capability, &self.schema, &self.compatibility) + } +} + +/// Idunn's admitted requirement for one provider-owned runtime capability. +/// The provider reports actual capacity in `GameCultRuntimeCapability`; this +/// record states only the minimum needed for this incarnation to be Ready. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct IdunnExpectedCapability { + pub capability: String, + pub schema: String, + pub compatibility: String, + pub minimum_capacity: u32, +} + +impl IdunnExpectedCapability { + fn identity(&self) -> (&str, &str, &str) { + (&self.capability, &self.schema, &self.compatibility) + } +} + +/// Dual-proved claim from one exact runtime launch. The stable provider key and +/// Idunn's activation-scoped ephemeral key both sign `canonical_proof_payload`: +/// the canonical positional encoding of this complete record with both proof +/// byte fields empty. Present authority requires a matching Idunn-published +/// current activation; neither proof can establish admission by itself. +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache( + type = "gamecult.runtime_presence_health", + schema = "gamecult.runtime_presence_health.v2" +)] +pub struct GameCultRuntimePresenceHealthRecord { + #[cultcache(key = 0)] + pub schema_version: String, + #[cultcache(key = 1)] + pub target: String, + #[cultcache(key = 2)] + pub expected_projection_sha256: String, + #[cultcache(key = 3)] + pub plan_id: String, + #[cultcache(key = 4)] + pub incarnation_id: String, + #[cultcache(key = 5)] + pub sealed_release_id: String, + #[cultcache(key = 6)] + pub activation_witness_sha256: String, + #[cultcache(key = 7)] + pub state_schema_generation: Option, + #[cultcache(key = 8)] + pub state_contract_sha256: Option, + #[cultcache(key = 9)] + pub runtime_id: String, + #[cultcache(key = 10)] + pub runtime_instance_id: String, + #[cultcache(key = 11)] + pub bound_endpoint: Option, + #[cultcache(key = 12)] + pub capabilities: Vec, + #[cultcache(key = 13)] + pub health_contract: String, + #[cultcache(key = 14)] + pub state: String, + #[cultcache(key = 15)] + pub detail: String, + #[cultcache(key = 16)] + pub write_lease_sha256: Option, + #[cultcache(key = 17)] + pub signer_identity_id: String, + #[cultcache(key = 18)] + pub publisher_sequence: u64, + #[cultcache(key = 19)] + pub observed_at_unix_millis: u64, + #[cultcache(key = 20)] + pub signature_algorithm: String, + #[cultcache(key = 21, bytes)] + pub signature: Vec, + #[cultcache(key = 22)] + pub activation_signer_identity_id: String, + #[cultcache(key = 23, bytes)] + pub activation_signature: Vec, +} + +impl GameCultRuntimePresenceHealthRecord { + /// Return the one canonical payload covered by both the stable provider + /// signature and the activation-scoped proof. Both signature fields are + /// empty in these bytes, avoiding circular signatures while binding every + /// authority-bearing statement field and both signer identities. + pub fn canonical_proof_payload(&self) -> Result> { + self.validate_shape(true)?; + let mut unsigned = self.clone(); + unsigned.signature.clear(); + unsigned.activation_signature.clear(); + Ok(rmp_serde::to_vec(&unsigned)?) + } + + /// Digest the complete signed presence record for an Odin correlation + /// receipt. An unsigned construction value is deliberately not digestible. + pub fn canonical_sha256(&self) -> Result { + self.validate()?; + Ok(prefixed_sha256(&rmp_serde::to_vec(self)?)) + } + + pub fn validate(&self) -> Result<()> { + self.validate_shape(false) + } + + fn validate_shape(&self, allow_unsigned: bool) -> Result<()> { + if self.schema_version != GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA { + bail!("runtime presence schema is unsupported"); + } + validate_authority_identifier(&self.target, "target")?; + validate_required_sha256( + &self.expected_projection_sha256, + "expected projection sha256", + )?; + validate_required_sha256(&self.plan_id, "plan id")?; + validate_authority_identifier(&self.incarnation_id, "incarnation id")?; + validate_required_sha256(&self.sealed_release_id, "sealed release id")?; + validate_required_sha256(&self.activation_witness_sha256, "activation witness sha256")?; + match (&self.state_schema_generation, &self.state_contract_sha256) { + (Some(generation), Some(contract_sha256)) => { + validate_authority_identifier(generation, "state schema generation")?; + validate_required_sha256(contract_sha256, "state contract sha256")?; + } + (None, None) => {} + _ => bail!("runtime presence state lineage is partial"), + } + validate_authority_identifier(&self.runtime_id, "runtime id")?; + validate_required_sha256(&self.runtime_instance_id, "runtime instance id")?; + validate_optional_endpoint(&self.bound_endpoint)?; + validate_runtime_capabilities(&self.capabilities)?; + validate_authority_identifier(&self.health_contract, "health contract")?; + validate_authority_identifier(&self.signer_identity_id, "signer identity id")?; + validate_authority_identifier( + &self.activation_signer_identity_id, + "activation signer identity id", + )?; + validate_optional_sha256(&self.write_lease_sha256, "write lease sha256")?; + + let signature_is_valid = + self.signature.len() == 64 || (allow_unsigned && self.signature.is_empty()); + let activation_signature_is_valid = self.activation_signature.len() == 64 + || (allow_unsigned && self.activation_signature.is_empty()); + if self.publisher_sequence == 0 + || self.observed_at_unix_millis == 0 + || !matches!( + self.state.as_str(), + "warming" | "active" | "degraded" | "failed" + ) + || self.detail.len() > 512 + || self.detail.chars().any(char::is_control) + || self.signature_algorithm != "ed25519" + || !signature_is_valid + || !activation_signature_is_valid + { + bail!("runtime presence state, sequence, detail, or signature is invalid"); + } + if self.state == "warming" && self.write_lease_sha256.is_some() { + bail!("warming runtime presence cannot claim a process write lease"); + } + if self.state_schema_generation.is_none() && self.write_lease_sha256.is_some() { + bail!("stateless runtime presence cannot claim a process write lease"); + } + Ok(()) + } +} + +/// Public route facts admitted for one incarnation. These are service-facing +/// endpoints only; actuator paths, unit files, container sockets, and runner +/// credentials do not belong in the Expected projection. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct IdunnExpectedRoute { + pub route_id: String, + pub transport: String, + pub stable_endpoint: String, + pub candidate_endpoint: String, +} + +impl IdunnExpectedRoute { + fn validate(&self) -> Result<()> { + validate_authority_identifier(&self.route_id, "route id")?; + validate_endpoint(&self.stable_endpoint, "stable route endpoint")?; + validate_endpoint(&self.candidate_endpoint, "candidate route endpoint")?; + if self.stable_endpoint == self.candidate_endpoint { + bail!("stable and candidate route endpoints are identical"); + } + match self.transport.as_str() { + "http" => { + if !(self.stable_endpoint.starts_with("http://") + || self.stable_endpoint.starts_with("https://")) + || !self.candidate_endpoint.starts_with("http://") + { + bail!("HTTP route endpoints use an unsupported scheme"); + } + } + "tcp" => { + if !self.stable_endpoint.starts_with("tcp://") + || !self.candidate_endpoint.starts_with("tcp://") + { + bail!("TCP route endpoints use an unsupported scheme"); + } + } + "rudp" => { + if !self.stable_endpoint.starts_with("rudp://") + || !self.candidate_endpoint.starts_with("rudp://") + { + bail!("RUDP route endpoints use an unsupported scheme"); + } + } + _ => bail!("route transport is unsupported"), + } + Ok(()) + } +} + +/// One capability requirement and Idunn's exact admitted provider selection. +/// Provider fields are all absent while unresolved. Managed providers bind an +/// Expected digest; external operator bindings instead bind an endpoint. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct IdunnExpectedDependency { + pub kind: String, + pub capability: String, + pub schema: String, + pub compatibility: String, + pub minimum_capacity: u32, + pub startup: String, + pub provider_id: Option, + pub provider_authority: Option, + pub provider_expected_projection_sha256: Option, + pub provider_endpoint: Option, +} + +impl IdunnExpectedDependency { + fn identity(&self) -> (&str, &str, &str) { + (&self.capability, &self.schema, &self.compatibility) + } + + fn validate(&self) -> Result<()> { + if !matches!( + self.kind.as_str(), + "bootstrap" + | "required" + | "optional" + | "shared-infrastructure" + | "private" + | "external-operator-binding" + ) { + bail!("dependency kind is unsupported"); + } + validate_authority_identifier(&self.capability, "dependency capability")?; + validate_authority_identifier(&self.schema, "dependency schema")?; + validate_authority_identifier(&self.compatibility, "dependency compatibility")?; + if self.minimum_capacity == 0 + || !matches!(self.startup.as_str(), "before-start" | "before-promotion") + { + bail!("dependency capacity or startup condition is invalid"); + } + if self.kind == "bootstrap" && self.startup != "before-start" { + bail!("bootstrap dependency must be satisfied before start"); + } + validate_optional_identifier(&self.provider_id, "dependency provider id")?; + validate_optional_identifier(&self.provider_authority, "dependency provider authority")?; + validate_optional_sha256( + &self.provider_expected_projection_sha256, + "dependency provider expected projection sha256", + )?; + validate_optional_endpoint_labeled( + &self.provider_endpoint, + "dependency provider endpoint", + )?; + + match ( + &self.provider_id, + &self.provider_authority, + &self.provider_expected_projection_sha256, + &self.provider_endpoint, + ) { + (None, None, None, None) => {} + (Some(_), Some(authority), Some(_), _) if authority == "managed-incarnation" => { + if self.kind == "external-operator-binding" { + bail!("external operator dependency cannot select a managed incarnation"); + } + } + (Some(_), Some(authority), None, Some(_)) + if authority == "external-operator-binding" + && self.kind == "external-operator-binding" => {} + _ => bail!("dependency provider binding is partial or authority-incoherent"), + } + Ok(()) + } +} + +/// Idunn-owned desired deployment identity published into CultMesh before +/// promotion. It describes the admitted incarnation and its public contracts, +/// never the private details needed to execute it on a particular host. +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache( + type = "idunn.expected_incarnation", + schema = "idunn.expected_incarnation.v2" +)] +pub struct IdunnExpectedIncarnationRecord { + #[cultcache(key = 0)] + pub schema_version: String, + #[cultcache(key = 1)] + pub target: String, + #[cultcache(key = 2)] + pub plan_id: String, + #[cultcache(key = 3)] + pub incarnation_id: String, + #[cultcache(key = 4)] + pub sealed_release_id: String, + #[cultcache(key = 5)] + pub source_repository: String, + #[cultcache(key = 6)] + pub source_revision: String, + #[cultcache(key = 7)] + pub recipe_sha256: String, + #[cultcache(key = 8)] + pub runtime_id: String, + #[cultcache(key = 9)] + pub expected_signer_identity_id: String, + #[cultcache(key = 10)] + pub health_contract: String, + #[cultcache(key = 11)] + pub artifact_sha256: String, + #[cultcache(key = 12)] + pub state_schema_generation: Option, + #[cultcache(key = 13)] + pub state_contract_sha256: Option, + #[cultcache(key = 14)] + pub write_lease_required: bool, + #[cultcache(key = 15)] + pub route: Option, + #[cultcache(key = 16)] + pub capabilities: Vec, + #[cultcache(key = 17)] + pub dependencies: Vec, +} + +impl IdunnExpectedIncarnationRecord { + pub fn decode_canonical(payload: &[u8]) -> Result { + if messagepack_array_len(payload) != Some(18) { + bail!("expected incarnation is not the 18-field positional contract"); + } + let expected: Self = + rmp_serde::from_slice(payload).context("decoding expected incarnation")?; + if rmp_serde::to_vec(&expected)? != payload { + bail!("expected incarnation is not canonical positional MessagePack"); + } + expected.validate()?; + Ok(expected) + } + + pub fn canonical_bytes(&self) -> Result> { + self.validate()?; + Ok(rmp_serde::to_vec(self)?) + } + + pub fn canonical_sha256(&self) -> Result { + Ok(prefixed_sha256(&self.canonical_bytes()?)) + } + + pub fn validate(&self) -> Result<()> { + if self.schema_version != IDUNN_EXPECTED_INCARNATION_SCHEMA { + bail!("expected incarnation schema is unsupported"); + } + validate_authority_identifier(&self.target, "target")?; + validate_required_sha256(&self.plan_id, "plan id")?; + validate_authority_identifier(&self.incarnation_id, "incarnation id")?; + validate_required_sha256(&self.sealed_release_id, "sealed release id")?; + validate_source_repository(&self.source_repository)?; + if !is_lower_hex(&self.source_revision, 40) { + bail!("source revision is not a lowercase Git object id"); + } + validate_required_sha256(&self.recipe_sha256, "recipe sha256")?; + validate_authority_identifier(&self.runtime_id, "runtime id")?; + validate_authority_identifier( + &self.expected_signer_identity_id, + "expected signer identity id", + )?; + validate_authority_identifier(&self.health_contract, "health contract")?; + validate_required_sha256(&self.artifact_sha256, "artifact sha256")?; + match (&self.state_schema_generation, &self.state_contract_sha256) { + (Some(generation), Some(contract_sha256)) => { + validate_authority_identifier(generation, "state schema generation")?; + validate_required_sha256(contract_sha256, "state contract sha256")?; + } + (None, None) => {} + _ => bail!("expected incarnation state lineage is partial"), + } + if self.write_lease_required && self.state_schema_generation.is_none() { + bail!("stateless incarnation cannot require a process write lease"); + } + if let Some(route) = &self.route { + route.validate()?; + } + validate_expected_capabilities(&self.capabilities)?; + validate_expected_dependencies(&self.dependencies)?; + Ok(()) + } +} + +/// Odin's observation of one dependency selected in the bound Expected +/// projection. Evidence digests identify the signed/provider-owned facts Odin +/// used; they do not copy those facts into Odin authority. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct OdinRuntimeDependencyEvidence { + pub kind: String, + pub capability: String, + pub schema: String, + pub compatibility: String, + pub provider_id: Option, + pub provider_authority: Option, + pub provider_expected_projection_sha256: Option, + pub provider_endpoint: Option, + pub observed_capacity: Option, + pub provider_evidence_sha256: Option, + pub ready: bool, +} + +impl OdinRuntimeDependencyEvidence { + fn identity(&self) -> (&str, &str, &str) { + (&self.capability, &self.schema, &self.compatibility) + } + + fn validate(&self) -> Result<()> { + if !matches!( + self.kind.as_str(), + "bootstrap" + | "required" + | "optional" + | "shared-infrastructure" + | "private" + | "external-operator-binding" + ) { + bail!("dependency evidence kind is unsupported"); + } + validate_authority_identifier(&self.kind, "dependency evidence kind")?; + validate_authority_identifier(&self.capability, "dependency evidence capability")?; + validate_authority_identifier(&self.schema, "dependency evidence schema")?; + validate_authority_identifier(&self.compatibility, "dependency evidence compatibility")?; + validate_optional_identifier(&self.provider_id, "dependency evidence provider id")?; + validate_optional_identifier( + &self.provider_authority, + "dependency evidence provider authority", + )?; + validate_optional_sha256( + &self.provider_expected_projection_sha256, + "dependency evidence provider expected projection sha256", + )?; + validate_optional_endpoint_labeled( + &self.provider_endpoint, + "dependency evidence provider endpoint", + )?; + validate_optional_sha256( + &self.provider_evidence_sha256, + "dependency provider evidence sha256", + )?; + if self.observed_capacity == Some(0) { + bail!("dependency evidence capacity is zero"); + } + if self.provider_id.is_none() + && (self.provider_authority.is_some() + || self.provider_expected_projection_sha256.is_some() + || self.provider_endpoint.is_some() + || self.observed_capacity.is_some() + || self.provider_evidence_sha256.is_some() + || self.ready) + { + bail!("dependency evidence has facts without a provider"); + } + if self.provider_id.is_some() && self.provider_authority.is_none() { + bail!("dependency evidence provider authority is missing"); + } + match ( + &self.provider_id, + &self.provider_authority, + &self.provider_expected_projection_sha256, + &self.provider_endpoint, + ) { + (None, None, None, None) => {} + (Some(_), Some(authority), Some(_), _) if authority == "managed-incarnation" => { + if self.kind == "external-operator-binding" { + bail!("external dependency evidence names a managed incarnation"); + } + } + (Some(_), Some(authority), None, Some(_)) + if authority == "external-operator-binding" + && self.kind == "external-operator-binding" => {} + _ => bail!("dependency evidence provider binding is authority-incoherent"), + } + if self.ready + && (self.observed_capacity.is_none() || self.provider_evidence_sha256.is_none()) + { + bail!("ready dependency lacks capacity or signed evidence"); + } + Ok(()) + } +} + +/// One explicit Expected/observed disagreement. Codes are unique and sorted so +/// the signed receipt has one canonical explanation for each disagreement. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct OdinTopologyDisagreement { + pub code: String, + pub expected: Option, + pub observed: Option, +} + +impl OdinTopologyDisagreement { + fn validate(&self) -> Result<()> { + validate_authority_identifier(&self.code, "topology disagreement code")?; + validate_optional_detail(&self.expected, "topology disagreement expected value")?; + validate_optional_detail(&self.observed, "topology disagreement observed value")?; + if self.expected.is_none() && self.observed.is_none() { + bail!("topology disagreement carries no expected or observed value"); + } + Ok(()) + } +} + +/// Odin-owned signed correlation of Idunn's exact Expected projection with +/// current Idunn activation and provider-signed presence. Presence state and +/// write-lease observation remain explicit so an opaque Ready bit cannot cross +/// Idunn's warming/fencing boundary. The signature covers the canonical +/// positional encoding with `signature` empty. +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache( + type = "odin.runtime_topology_correlation", + schema = "odin.runtime_topology_correlation.v2" +)] +pub struct OdinRuntimeTopologyCorrelationRecord { + #[cultcache(key = 0)] + pub schema_version: String, + #[cultcache(key = 1)] + pub target: String, + #[cultcache(key = 2)] + pub expected_projection_sha256: String, + #[cultcache(key = 3)] + pub expected: bool, + #[cultcache(key = 4)] + pub current_activation_sha256: Option, + #[cultcache(key = 5)] + pub signed_presence_sha256: Option, + #[cultcache(key = 6)] + pub observed_presence_state: Option, + #[cultcache(key = 7)] + pub observed_presence_publisher_sequence: Option, + #[cultcache(key = 8)] + pub observed_write_lease_sha256: Option, + #[cultcache(key = 9)] + pub observed_capabilities: Vec, + #[cultcache(key = 10)] + pub runtime_id: String, + #[cultcache(key = 11)] + pub runtime_instance_id: Option, + #[cultcache(key = 12)] + pub present: bool, + #[cultcache(key = 13)] + pub ready: bool, + #[cultcache(key = 14)] + pub dependencies: Vec, + #[cultcache(key = 15)] + pub disagreements: Vec, + #[cultcache(key = 16)] + pub signer_identity_id: String, + #[cultcache(key = 17)] + pub publisher_sequence: u64, + #[cultcache(key = 18)] + pub observed_at_unix_millis: u64, + #[cultcache(key = 19)] + pub signature_algorithm: String, + #[cultcache(key = 20, bytes)] + pub signature: Vec, +} + +impl OdinRuntimeTopologyCorrelationRecord { + pub fn decode_canonical_signed_payload(payload: &[u8]) -> Result<(Self, Vec)> { + if messagepack_array_len(payload) != Some(21) { + bail!("topology correlation is not the 21-field positional contract"); + } + let receipt: Self = + rmp_serde::from_slice(payload).context("decoding runtime topology correlation")?; + if rmp_serde::to_vec(&receipt)? != payload { + bail!("topology correlation is not canonical positional MessagePack"); + } + receipt.validate()?; + let unsigned = receipt.unsigned_signature_payload()?; + Ok((receipt, unsigned)) + } + + pub fn unsigned_signature_payload(&self) -> Result> { + self.validate_shape(true)?; + let mut unsigned = self.clone(); + unsigned.signature.clear(); + Ok(rmp_serde::to_vec(&unsigned)?) + } + + pub fn canonical_bytes(&self) -> Result> { + self.validate()?; + Ok(rmp_serde::to_vec(self)?) + } + + pub fn canonical_sha256(&self) -> Result { + Ok(prefixed_sha256(&self.canonical_bytes()?)) + } + + pub fn validate(&self) -> Result<()> { + self.validate_shape(false) + } + + pub fn validate_against_expected( + &self, + expected: &IdunnExpectedIncarnationRecord, + current_write_lease_sha256: Option<&str>, + ) -> Result<()> { + self.validate()?; + expected.validate()?; + if self.expected_projection_sha256 != expected.canonical_sha256()? + || self.target != expected.target + || self.runtime_id != expected.runtime_id + || self.dependencies.len() != expected.dependencies.len() + { + bail!("topology correlation substitutes or omits Expected authority"); + } + for (evidence, requirement) in self.dependencies.iter().zip(&expected.dependencies) { + if evidence.identity() != requirement.identity() + || evidence.kind != requirement.kind + || evidence.provider_id != requirement.provider_id + || evidence.provider_authority != requirement.provider_authority + || evidence.provider_expected_projection_sha256 + != requirement.provider_expected_projection_sha256 + || evidence.provider_endpoint != requirement.provider_endpoint + { + bail!("topology dependency evidence does not bind the Expected dependency"); + } + if evidence.ready + && evidence + .observed_capacity + .is_none_or(|capacity| capacity < requirement.minimum_capacity) + { + bail!("ready dependency does not meet Expected capacity"); + } + } + let mut capability_disagreements = Vec::new(); + correlate_capabilities( + &mut capability_disagreements, + &expected.capabilities, + &self.observed_capabilities, + ); + for disagreement in &capability_disagreements { + if !self.disagreements.contains(disagreement) { + bail!("topology correlation omits an observed capability disagreement"); + } + } + if let Some(current_write_lease_sha256) = current_write_lease_sha256 { + validate_required_sha256(current_write_lease_sha256, "current write lease sha256")?; + } + if expected.write_lease_required { + if self + .observed_write_lease_sha256 + .as_deref() + .is_some_and(|observed| Some(observed) != current_write_lease_sha256) + { + bail!("observed write lease is not the current Idunn grant"); + } + if self.ready + && (self.observed_write_lease_sha256.is_none() + || current_write_lease_sha256.is_none()) + { + bail!("Ready stateful topology lacks the exact current write lease"); + } + } else if self.observed_write_lease_sha256.is_some() || current_write_lease_sha256.is_some() + { + bail!("stateless Expected topology cannot carry a process write lease"); + } + Ok(()) + } + + fn validate_shape(&self, allow_unsigned: bool) -> Result<()> { + if self.schema_version != ODIN_RUNTIME_TOPOLOGY_CORRELATION_SCHEMA { + bail!("topology correlation schema is unsupported"); + } + validate_authority_identifier(&self.target, "target")?; + validate_required_sha256( + &self.expected_projection_sha256, + "correlation expected projection sha256", + )?; + if !self.expected { + bail!("topology correlation cannot deny the Expected record it binds"); + } + validate_optional_sha256(&self.current_activation_sha256, "current activation sha256")?; + validate_optional_sha256(&self.signed_presence_sha256, "signed presence sha256")?; + validate_optional_identifier(&self.observed_presence_state, "observed presence state")?; + if self.observed_presence_publisher_sequence == Some(0) { + bail!("observed presence publisher sequence is zero"); + } + validate_optional_sha256( + &self.observed_write_lease_sha256, + "observed write lease sha256", + )?; + validate_runtime_capabilities(&self.observed_capabilities)?; + if self.signed_presence_sha256.is_none() && !self.observed_capabilities.is_empty() { + bail!("topology correlation has capabilities without signed presence"); + } + match ( + &self.signed_presence_sha256, + &self.observed_presence_state, + self.observed_presence_publisher_sequence, + &self.observed_write_lease_sha256, + ) { + (None, None, None, None) => {} + (Some(_), Some(state), Some(_), write_lease) + if matches!(state.as_str(), "warming" | "active" | "degraded" | "failed") => + { + if state == "warming" && write_lease.is_some() { + bail!("warming topology presence cannot carry a process write lease"); + } + } + _ => bail!("signed presence state, sequence, or write-lease observation is partial"), + } + validate_authority_identifier(&self.runtime_id, "runtime id")?; + validate_optional_sha256(&self.runtime_instance_id, "runtime instance id")?; + let has_runtime_evidence = + self.current_activation_sha256.is_some() || self.signed_presence_sha256.is_some(); + if self.runtime_instance_id.is_some() != has_runtime_evidence { + bail!("runtime instance identity and observation evidence are partial"); + } + if self.present + && (self.current_activation_sha256.is_none() + || self.signed_presence_sha256.is_none() + || self.observed_presence_publisher_sequence.is_none() + || self.runtime_instance_id.is_none()) + { + bail!("Present topology state lacks an authenticated runtime session"); + } + validate_dependency_evidence(&self.dependencies)?; + validate_topology_disagreements(&self.disagreements)?; + let has_activation = self.current_activation_sha256.is_some(); + let has_presence = self.signed_presence_sha256.is_some(); + if has_activation && has_presence && !self.present { + bail!("authenticated runtime session cannot be projected as not Present"); + } + if has_activation != has_presence && self.disagreements.is_empty() { + bail!("partial or rejected runtime evidence lacks an explicit disagreement"); + } + if self.ready + && (!self.present + || self.observed_presence_state.as_deref() != Some("active") + || !self.disagreements.is_empty() + || self + .dependencies + .iter() + .any(|dependency| dependency.kind != "optional" && !dependency.ready)) + { + bail!("Ready topology state is not supported by correlated evidence"); + } + validate_authority_identifier(&self.signer_identity_id, "signer identity id")?; + let signature_is_valid = + self.signature.len() == 64 || (allow_unsigned && self.signature.is_empty()); + if self.publisher_sequence == 0 + || self.observed_at_unix_millis == 0 + || self.signature_algorithm != "ed25519" + || !signature_is_valid + { + bail!("topology correlation sequence, time, or signature is invalid"); + } + Ok(()) + } +} + +/// An in-memory activation-scoped signer reconstructed by the launched service +/// from its protected systemd credential. The stable service identity never +/// receives this key, and the public activation record never contains it. +pub struct IdunnRuntimeActivationSigner { + signing_key: SigningKey, +} + +impl IdunnRuntimeActivationSigner { + fn generate() -> Self { + let mut seed = Zeroizing::new([0_u8; 32]); + rand::rng().fill_bytes(seed.as_mut()); + Self { + signing_key: SigningKey::from_bytes(&*seed), + } + } + + /// Open the exact raw 32-byte Ed25519 seed supplied through Idunn's systemd + /// credential without retaining an ordinary heap copy. No persistent key + /// schema exists: this authority lives for one activation and is replaced + /// on every launch. + pub fn from_credential_reader(mut credential: impl Read) -> Result { + let mut seed = Zeroizing::new([0_u8; 32]); + credential + .read_exact(seed.as_mut()) + .context("reading runtime activation credential")?; + let mut extra = [0_u8; 1]; + if credential.read(&mut extra)? != 0 { + bail!("runtime activation credential is longer than 32 bytes"); + } + Ok(Self { + signing_key: SigningKey::from_bytes(&*seed), + }) + } + + fn write_credential(&self, mut destination: impl Write) -> Result<()> { + let seed = Zeroizing::new(self.signing_key.to_bytes()); + destination + .write_all(seed.as_ref()) + .context("writing runtime activation credential") + } + + pub fn public_key(&self) -> Vec { + self.signing_key.verifying_key().to_bytes().to_vec() + } + + pub fn identity_id(&self) -> String { + derive_idunn_runtime_activation_identity_id(&self.public_key()) + .expect("an Ed25519 activation signer always has a 32-byte public key") + } + + pub fn sign_presence_proof( + &self, + presence: &GameCultRuntimePresenceHealthRecord, + ) -> Result> { + if presence.activation_signer_identity_id != self.identity_id() { + bail!("runtime presence names a different activation signer"); + } + let payload = presence.canonical_proof_payload()?; + Ok(self + .signing_key + .sign(&runtime_activation_signing_message(&payload)) + .to_bytes() + .to_vec()) + } +} + +/// Idunn's one-shot launch material. Issuance always creates a fresh key, and +/// writing the protected credential consumes the only Idunn-side signer. +pub struct IdunnRuntimeActivationLaunch { + activation: IdunnRuntimeActivationRecord, + signer: IdunnRuntimeActivationSigner, +} + +impl IdunnRuntimeActivationLaunch { + pub fn issue( + expected: &IdunnExpectedIncarnationRecord, + runtime_instance_id: String, + issued_at_unix_millis: u64, + idunn_signer: &crate::ServiceIdentitySigner, + ) -> Result { + let signer = IdunnRuntimeActivationSigner::generate(); + let activation = IdunnRuntimeActivationRecord::issue_with_signer( + expected, + runtime_instance_id, + &signer, + issued_at_unix_millis, + idunn_signer, + )?; + Ok(Self { activation, signer }) + } + + pub fn activation(&self) -> &IdunnRuntimeActivationRecord { + &self.activation + } + + pub fn write_credential(self, destination: impl Write) -> Result { + self.signer.write_credential(destination)?; + Ok(self.activation) + } +} + +/// Derive the activation-scoped identity from the Idunn-issued public key. The +/// domain is dedicated to runtime activation and cannot be substituted for a +/// stable provider, Idunn, or Odin service identity. +pub fn derive_idunn_runtime_activation_identity_id(public_key: &[u8]) -> Result { + if public_key.len() != 32 { + bail!("runtime activation public key has invalid length"); + } + Ok(format!( + "{:x}", + Sha256::digest([IDUNN_RUNTIME_ACTIVATION_ID_DOMAIN, public_key].concat()) + )) +} + +/// Current Idunn authority for one launch. Idunn's stable signature binds the +/// complete Expected digest and activation key; Expected in turn pins the +/// provider signer identity. The provider public key is lookup material, not a +/// second authority: its profile-derived identity must equal Idunn's signed +/// selection exactly. +#[derive(Clone, Debug)] +pub struct VerifiedRuntimeAuthority { + expected: IdunnExpectedIncarnationRecord, + activation: IdunnRuntimeActivationRecord, + provider_signer_public_key: Vec, + expected_sha256: String, + activation_sha256: String, +} + +impl VerifiedRuntimeAuthority { + pub fn expected(&self) -> &IdunnExpectedIncarnationRecord { + &self.expected + } + + pub fn activation(&self) -> &IdunnRuntimeActivationRecord { + &self.activation + } + + pub fn expected_sha256(&self) -> &str { + &self.expected_sha256 + } + + pub fn activation_sha256(&self) -> &str { + &self.activation_sha256 + } +} + +pub fn verify_runtime_authority( + expected: &IdunnExpectedIncarnationRecord, + activation: &IdunnRuntimeActivationRecord, + idunn_anchor: &crate::ServiceIdentityTrustAnchor, + provider_signer_public_key: &[u8], +) -> Result { + expected.validate()?; + activation.verify_for_expected(expected, idunn_anchor)?; + if crate::derive_service_identity_id::( + provider_signer_public_key, + )? != expected.expected_signer_identity_id + { + bail!("provider public key is not Idunn's Expected signer selection"); + } + Ok(VerifiedRuntimeAuthority { + expected: expected.clone(), + activation: activation.clone(), + provider_signer_public_key: provider_signer_public_key.to_vec(), + expected_sha256: expected.canonical_sha256()?, + activation_sha256: activation.canonical_sha256()?, + }) +} + +#[derive(Clone, Copy, Debug)] +pub struct RuntimePresenceAuthenticationContext { + pub trusted_received_at_unix_millis: u64, + pub maximum_age_millis: u64, + pub maximum_future_skew_millis: u64, +} + +/// Canonical runtime bytes authenticated by both the Expected provider key and +/// the current activation key. No semantic claim has yet been accepted as +/// Present; Odin must correlate this observation against current authority. +#[derive(Clone, Debug)] +pub struct AuthenticatedRuntimePresenceClaim { + record: GameCultRuntimePresenceHealthRecord, + canonical_bytes: Vec, + signed_presence_sha256: String, + received_at_unix_millis: u64, +} + +impl AuthenticatedRuntimePresenceClaim { + pub fn record(&self) -> &GameCultRuntimePresenceHealthRecord { + &self.record + } + + pub fn canonical_bytes(&self) -> &[u8] { + &self.canonical_bytes + } + + pub fn signed_presence_sha256(&self) -> &str { + &self.signed_presence_sha256 + } + + pub fn received_at_unix_millis(&self) -> u64 { + self.received_at_unix_millis + } +} + +/// A signed Odin receipt authenticated against the exact current Expected, +/// activation, write lease, Idunn-admitted topology key, and trusted receipt +/// time. This is evidence, not a replay cursor or route-promotion grant. The +/// caller must source the key from Idunn's admitted Odin binding. Idunn's +/// durable admission owner must atomically check and advance Odin's publisher +/// sequence before treating the receipt as current. +#[derive(Clone, Debug)] +pub struct AuthenticatedOdinRuntimeTopologyCorrelation { + record: OdinRuntimeTopologyCorrelationRecord, + canonical_bytes: Vec, +} + +impl AuthenticatedOdinRuntimeTopologyCorrelation { + pub fn record(&self) -> &OdinRuntimeTopologyCorrelationRecord { + &self.record + } + + pub fn canonical_bytes(&self) -> &[u8] { + &self.canonical_bytes + } +} + +#[derive(Clone, Copy, Debug)] +pub struct OdinTopologyAuthenticationContext { + pub trusted_received_at_unix_millis: u64, + pub maximum_age_millis: u64, + pub maximum_future_skew_millis: u64, +} + +pub fn authenticate_odin_runtime_topology_correlation( + canonical_correlation: &[u8], + authority: &VerifiedRuntimeAuthority, + current_write_lease_sha256: Option<&str>, + admitted_odin_signer_public_key: &[u8], + context: OdinTopologyAuthenticationContext, +) -> Result { + ensure!( + context.trusted_received_at_unix_millis > 0 && context.maximum_age_millis > 0, + "Odin topology trusted time and maximum age must be positive" + ); + let (record, unsigned) = OdinRuntimeTopologyCorrelationRecord::decode_canonical_signed_payload( + canonical_correlation, + )?; + record.validate_against_expected(authority.expected(), current_write_lease_sha256)?; + if record.current_activation_sha256.as_deref() != Some(authority.activation_sha256()) + || record.runtime_instance_id.as_deref() + != Some(authority.activation.runtime_instance_id.as_str()) + { + bail!("topology correlation does not bind the current activation"); + } + let latest_trusted_timestamp = context + .trusted_received_at_unix_millis + .saturating_add(context.maximum_future_skew_millis); + if record.observed_at_unix_millis > latest_trusted_timestamp + || context + .trusted_received_at_unix_millis + .saturating_sub(record.observed_at_unix_millis) + > context.maximum_age_millis + { + bail!("Odin topology correlation is outside the trusted observation window"); + } + crate::verify_service_identity_signature_with_public_key::< + crate::OdinTopologyIdentity, + crate::OdinRuntimeTopologyCorrelationPurpose, + >( + admitted_odin_signer_public_key, + &unsigned, + &crate::ServiceIdentitySignature { + identity_id: record.signer_identity_id.clone(), + signature: record.signature.clone(), + }, + )?; + Ok(AuthenticatedOdinRuntimeTopologyCorrelation { + record, + canonical_bytes: canonical_correlation.to_vec(), + }) +} + +/// Odin's deterministic interpretation of an authenticated provider claim. +/// Authentication of the stable provider and current activation establishes +/// Present. Correlation disagreements remain explicit and deny Ready; they do +/// not erase the observed session that produced them. +#[derive(Clone, Debug)] +pub struct RuntimePresenceCorrelation { + presence: VerifiedRuntimePresence, + disagreements: Vec, +} + +impl RuntimePresenceCorrelation { + pub fn claim(&self) -> &AuthenticatedRuntimePresenceClaim { + &self.presence.claim + } + + pub fn disagreements(&self) -> &[OdinTopologyDisagreement] { + &self.disagreements + } + + pub fn into_present(self) -> VerifiedRuntimePresence { + self.presence + } + + pub fn into_undisputed_present(self) -> Result { + if !self.disagreements.is_empty() { + bail!("runtime presence disagrees with current Expected projection"); + } + Ok(self.presence) + } +} + +/// A dual-authenticated claim that matches current authority. This is the only +/// value that represents semantic Present. Replay admission deliberately does +/// not live here: Odin's durable store must atomically check and advance every +/// authenticated provider sequence, including disagreement-bearing claims, +/// before it signs a topology correlation. +#[derive(Clone, Debug)] +pub struct VerifiedRuntimePresence { + claim: AuthenticatedRuntimePresenceClaim, +} + +impl VerifiedRuntimePresence { + pub fn record(&self) -> &GameCultRuntimePresenceHealthRecord { + self.claim.record() + } + + pub fn canonical_bytes(&self) -> &[u8] { + self.claim.canonical_bytes() + } + + pub fn signed_presence_sha256(&self) -> &str { + self.claim.signed_presence_sha256() + } + + pub fn accepted_at_unix_millis(&self) -> u64 { + self.claim.received_at_unix_millis() + } +} + +pub fn authenticate_runtime_presence_claim( + canonical_presence: &[u8], + authority: &VerifiedRuntimeAuthority, + context: RuntimePresenceAuthenticationContext, +) -> Result { + ensure!( + context.trusted_received_at_unix_millis > 0 && context.maximum_age_millis > 0, + "runtime presence trusted time and maximum age must be positive" + ); + if messagepack_array_len(canonical_presence) != Some(24) { + bail!("runtime presence payload is not the 24-field positional contract"); + } + let presence: GameCultRuntimePresenceHealthRecord = + rmp_serde::from_slice(canonical_presence).context("decoding runtime presence health")?; + if rmp_serde::to_vec(&presence)? != canonical_presence { + bail!("runtime presence payload is not canonical positional MessagePack"); + } + presence.validate()?; + let latest_trusted_timestamp = context + .trusted_received_at_unix_millis + .saturating_add(context.maximum_future_skew_millis); + if presence.observed_at_unix_millis > latest_trusted_timestamp + || authority.activation.issued_at_unix_millis > latest_trusted_timestamp + || context + .trusted_received_at_unix_millis + .saturating_sub(presence.observed_at_unix_millis) + > context.maximum_age_millis + { + bail!("runtime presence is outside the trusted observation window"); + } + if presence.signer_identity_id != authority.expected.expected_signer_identity_id { + bail!("runtime presence signer is not Idunn's Expected signer selection"); + } + + let proof_payload = presence.canonical_proof_payload()?; + crate::verify_service_identity_signature_with_public_key::< + crate::GameCultProviderHealthIdentity, + crate::GameCultRuntimePresenceHealthPurpose, + >( + &authority.provider_signer_public_key, + &proof_payload, + &crate::ServiceIdentitySignature { + identity_id: presence.signer_identity_id.clone(), + signature: presence.signature.clone(), + }, + )?; + let activation_public_key: [u8; 32] = authority + .activation + .activation_signer_public_key + .as_slice() + .try_into() + .map_err(|_| anyhow::anyhow!("runtime activation public key has invalid length"))?; + let activation_signature = Signature::from_slice(&presence.activation_signature) + .map_err(|_| anyhow::anyhow!("runtime activation signature has invalid length"))?; + VerifyingKey::from_bytes(&activation_public_key)? + .verify( + &runtime_activation_signing_message(&proof_payload), + &activation_signature, + ) + .map_err(|_| anyhow::anyhow!("runtime activation proof verification failed"))?; + + Ok(AuthenticatedRuntimePresenceClaim { + record: presence, + canonical_bytes: canonical_presence.to_vec(), + signed_presence_sha256: prefixed_sha256(canonical_presence), + received_at_unix_millis: context.trusted_received_at_unix_millis, + }) +} + +pub fn correlate_runtime_presence_claim( + claim: AuthenticatedRuntimePresenceClaim, + authority: &VerifiedRuntimeAuthority, +) -> Result { + let expected = authority.expected(); + let activation = authority.activation(); + let presence = claim.record(); + let expected_endpoint = expected + .route + .as_ref() + .map(|route| route.candidate_endpoint.clone()); + let mut disagreements = Vec::new(); + push_disagreement( + &mut disagreements, + "activation-signer-identity", + Some(&activation.activation_signer_identity_id), + Some(&presence.activation_signer_identity_id), + ); + push_disagreement( + &mut disagreements, + "activation-witness", + Some(authority.activation_sha256()), + Some(&presence.activation_witness_sha256), + ); + push_disagreement( + &mut disagreements, + "bound-endpoint", + expected_endpoint.as_deref(), + presence.bound_endpoint.as_deref(), + ); + push_disagreement( + &mut disagreements, + "expected-projection", + Some(authority.expected_sha256()), + Some(&presence.expected_projection_sha256), + ); + push_disagreement( + &mut disagreements, + "health-contract", + Some(&expected.health_contract), + Some(&presence.health_contract), + ); + push_disagreement( + &mut disagreements, + "incarnation-id", + Some(&expected.incarnation_id), + Some(&presence.incarnation_id), + ); + push_disagreement( + &mut disagreements, + "plan-id", + Some(&expected.plan_id), + Some(&presence.plan_id), + ); + push_disagreement( + &mut disagreements, + "runtime-id", + Some(&expected.runtime_id), + Some(&presence.runtime_id), + ); + push_disagreement( + &mut disagreements, + "runtime-instance-id", + Some(&activation.runtime_instance_id), + Some(&presence.runtime_instance_id), + ); + push_disagreement( + &mut disagreements, + "sealed-release-id", + Some(&expected.sealed_release_id), + Some(&presence.sealed_release_id), + ); + push_disagreement( + &mut disagreements, + "signer-identity", + Some(&expected.expected_signer_identity_id), + Some(&presence.signer_identity_id), + ); + push_disagreement( + &mut disagreements, + "state-contract", + expected.state_contract_sha256.as_deref(), + presence.state_contract_sha256.as_deref(), + ); + push_disagreement( + &mut disagreements, + "state-schema-generation", + expected.state_schema_generation.as_deref(), + presence.state_schema_generation.as_deref(), + ); + push_disagreement( + &mut disagreements, + "target", + Some(&expected.target), + Some(&presence.target), + ); + if activation.issued_at_unix_millis > presence.observed_at_unix_millis { + disagreements.push(OdinTopologyDisagreement { + code: "activation-issued-after-presence".into(), + expected: Some(format!("at-most:{}", presence.observed_at_unix_millis)), + observed: Some(activation.issued_at_unix_millis.to_string()), + }); + } + correlate_capabilities( + &mut disagreements, + &expected.capabilities, + &presence.capabilities, + ); + disagreements.sort_by(|left, right| left.code.cmp(&right.code)); + validate_topology_disagreements(&disagreements)?; + Ok(RuntimePresenceCorrelation { + presence: VerifiedRuntimePresence { claim }, + disagreements, + }) +} + +fn push_disagreement( + disagreements: &mut Vec, + code: &str, + expected: Option<&str>, + observed: Option<&str>, +) { + if expected != observed { + disagreements.push(OdinTopologyDisagreement { + code: code.into(), + expected: expected.map(str::to_string), + observed: observed.map(str::to_string), + }); + } +} + +fn correlate_capabilities( + disagreements: &mut Vec, + expected: &[IdunnExpectedCapability], + observed: &[GameCultRuntimeCapability], +) { + for (index, required) in expected.iter().enumerate() { + let actual = observed + .iter() + .find(|actual| actual.identity() == required.identity()); + let requirement = format!( + "{}/{}/{} capacity>={}", + required.capability, required.schema, required.compatibility, required.minimum_capacity + ); + match actual { + None => disagreements.push(OdinTopologyDisagreement { + code: format!("expected-capability-{index:03}-missing"), + expected: Some(requirement), + observed: None, + }), + Some(actual) if actual.capacity < required.minimum_capacity => { + disagreements.push(OdinTopologyDisagreement { + code: format!("expected-capability-{index:03}-capacity"), + expected: Some(requirement), + observed: Some(format!("capacity={}", actual.capacity)), + }); + } + Some(_) => {} + } + } +} + +fn runtime_activation_signing_message(payload: &[u8]) -> Vec { + let purpose = GAMECULT_RUNTIME_ACTIVATION_PROOF_SIGNING_PURPOSE.as_bytes(); + let mut out = Vec::with_capacity( + IDUNN_RUNTIME_ACTIVATION_SIGNATURE_DOMAIN.len() + purpose.len() + payload.len() + 16, + ); + out.extend_from_slice(IDUNN_RUNTIME_ACTIVATION_SIGNATURE_DOMAIN); + out.extend_from_slice(&(purpose.len() as u64).to_be_bytes()); + out.extend_from_slice(purpose); + out.extend_from_slice(&(payload.len() as u64).to_be_bytes()); + out.extend_from_slice(payload); + out +} + +/// Idunn-issued launch identity. Idunn's stable service identity signs the +/// complete activation, including the ephemeral public key, before the private +/// credential is passed to one workload alongside Expected. Odin still treats +/// it as current only after Idunn publishes an observed native process. +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache( + type = "idunn.runtime_activation", + schema = "idunn.runtime_activation.v2" +)] +pub struct IdunnRuntimeActivationRecord { + #[cultcache(key = 0)] + pub schema_version: String, + #[cultcache(key = 1)] + pub expected_projection_sha256: String, + #[cultcache(key = 2)] + pub runtime_id: String, + #[cultcache(key = 3)] + pub runtime_instance_id: String, + #[cultcache(key = 4)] + pub activation_signer_identity_id: String, + #[cultcache(key = 5, bytes)] + pub activation_signer_public_key: Vec, + #[cultcache(key = 6)] + pub issued_at_unix_millis: u64, + #[cultcache(key = 7)] + pub idunn_signer_identity_id: String, + #[cultcache(key = 8)] + pub signature_algorithm: String, + #[cultcache(key = 9, bytes)] + pub signature: Vec, +} + +impl IdunnRuntimeActivationRecord { + fn issue_with_signer( + expected: &IdunnExpectedIncarnationRecord, + runtime_instance_id: String, + activation_signer: &IdunnRuntimeActivationSigner, + issued_at_unix_millis: u64, + idunn_signer: &crate::ServiceIdentitySigner, + ) -> Result { + expected.validate()?; + let mut activation = Self { + schema_version: IDUNN_RUNTIME_ACTIVATION_SCHEMA.into(), + expected_projection_sha256: expected.canonical_sha256()?, + runtime_id: expected.runtime_id.clone(), + runtime_instance_id, + activation_signer_identity_id: activation_signer.identity_id(), + activation_signer_public_key: activation_signer.public_key(), + issued_at_unix_millis, + idunn_signer_identity_id: idunn_signer.entry().identity_id.clone(), + signature_algorithm: "ed25519".into(), + signature: Vec::new(), + }; + let proof = idunn_signer.sign::( + &activation.unsigned_signature_payload()?, + ); + activation.signature = proof.signature; + activation.validate()?; + Ok(activation) + } + + pub fn decode_canonical(payload: &[u8]) -> Result { + if messagepack_array_len(payload) != Some(10) { + bail!("runtime activation is not the 10-field positional contract"); + } + let activation: Self = + rmp_serde::from_slice(payload).context("decoding runtime activation")?; + if rmp_serde::to_vec(&activation)? != payload { + bail!("runtime activation is not canonical positional MessagePack"); + } + activation.validate()?; + Ok(activation) + } + + pub fn canonical_bytes(&self) -> Result> { + self.validate()?; + Ok(rmp_serde::to_vec(self)?) + } + + pub fn canonical_sha256(&self) -> Result { + Ok(prefixed_sha256(&self.canonical_bytes()?)) + } + + pub fn unsigned_signature_payload(&self) -> Result> { + self.validate_shape(true)?; + let mut unsigned = self.clone(); + unsigned.signature.clear(); + Ok(rmp_serde::to_vec(&unsigned)?) + } + + pub fn verify_for_expected( + &self, + expected: &IdunnExpectedIncarnationRecord, + idunn_anchor: &crate::ServiceIdentityTrustAnchor, + ) -> Result<()> { + self.validate()?; + expected.validate()?; + if self.expected_projection_sha256 != expected.canonical_sha256()? + || self.runtime_id != expected.runtime_id + { + bail!("runtime activation does not bind the current Expected incarnation"); + } + crate::verify_service_identity_signature::< + crate::IdunnServiceIdentity, + crate::IdunnRuntimeActivationPurpose, + >( + idunn_anchor, + &self.unsigned_signature_payload()?, + &crate::ServiceIdentitySignature { + identity_id: self.idunn_signer_identity_id.clone(), + signature: self.signature.clone(), + }, + ) + .context("verifying Idunn runtime activation signature") + } + + pub fn validate(&self) -> Result<()> { + self.validate_shape(false) + } + + fn validate_shape(&self, allow_unsigned: bool) -> Result<()> { + if self.schema_version != IDUNN_RUNTIME_ACTIVATION_SCHEMA { + bail!("runtime activation schema is unsupported"); + } + validate_required_sha256( + &self.expected_projection_sha256, + "activation expected projection sha256", + )?; + validate_authority_identifier(&self.runtime_id, "activation runtime id")?; + validate_required_sha256(&self.runtime_instance_id, "runtime instance id")?; + validate_authority_identifier( + &self.activation_signer_identity_id, + "activation signer identity id", + )?; + if self.activation_signer_identity_id + != derive_idunn_runtime_activation_identity_id(&self.activation_signer_public_key)? + { + bail!("runtime activation signer identity does not match its public key"); + } + validate_authority_identifier( + &self.idunn_signer_identity_id, + "Idunn activation signer identity id", + )?; + let signature_is_valid = + self.signature.len() == 64 || (allow_unsigned && self.signature.is_empty()); + if self.issued_at_unix_millis == 0 + || self.signature_algorithm != "ed25519" + || !signature_is_valid + { + bail!("runtime activation issue time or signature is invalid"); + } + Ok(()) + } +} + +/// Unsigned Idunn-owned local file that grants one exact process incarnation +/// state-write authority. It is authoritative only when read through the +/// root-owned admitted path and lock protocol; its bytes carry no route or +/// traffic authority. +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache( + type = "idunn.process_write_lease", + schema = "idunn.process_write_lease.v1" +)] +pub struct IdunnProcessWriteLeaseRecord { + #[cultcache(key = 0)] + pub schema_version: String, + #[cultcache(key = 1)] + pub target: String, + #[cultcache(key = 2)] + pub expected_projection_sha256: String, + #[cultcache(key = 3)] + pub plan_id: String, + #[cultcache(key = 4)] + pub incarnation_id: String, + #[cultcache(key = 5)] + pub sealed_release_id: String, + #[cultcache(key = 6)] + pub activation_witness_sha256: String, + #[cultcache(key = 7)] + pub state_schema_generation: String, + #[cultcache(key = 8)] + pub state_contract_sha256: String, + #[cultcache(key = 9)] + pub runtime_id: String, + #[cultcache(key = 10)] + pub runtime_instance_id: String, + #[cultcache(key = 11)] + pub warming_presence_sha256: String, + #[cultcache(key = 12)] + pub lease_epoch: u64, + #[cultcache(key = 13)] + pub issued_at_unix_millis: u64, +} + +impl IdunnProcessWriteLeaseRecord { + pub fn decode_canonical(payload: &[u8]) -> Result { + if messagepack_array_len(payload) != Some(14) { + bail!("process write lease is not the 14-field positional contract"); + } + let lease: Self = rmp_serde::from_slice(payload).context("decoding process write lease")?; + if rmp_serde::to_vec(&lease)? != payload { + bail!("process write lease is not canonical positional MessagePack"); + } + lease.validate()?; + Ok(lease) + } + + pub fn canonical_bytes(&self) -> Result> { + self.validate()?; + Ok(rmp_serde::to_vec(self)?) + } + + pub fn canonical_sha256(&self) -> Result { + Ok(prefixed_sha256(&self.canonical_bytes()?)) + } + + pub fn validate(&self) -> Result<()> { + if self.schema_version != IDUNN_PROCESS_WRITE_LEASE_SCHEMA { + bail!("process write lease schema is unsupported"); + } + validate_authority_identifier(&self.target, "target")?; + validate_required_sha256( + &self.expected_projection_sha256, + "lease expected projection sha256", + )?; + validate_required_sha256(&self.plan_id, "plan id")?; + validate_authority_identifier(&self.incarnation_id, "incarnation id")?; + validate_required_sha256(&self.sealed_release_id, "sealed release id")?; + validate_required_sha256(&self.activation_witness_sha256, "activation witness sha256")?; + validate_authority_identifier(&self.state_schema_generation, "state schema generation")?; + validate_required_sha256(&self.state_contract_sha256, "state contract sha256")?; + validate_authority_identifier(&self.runtime_id, "runtime id")?; + validate_required_sha256(&self.runtime_instance_id, "runtime instance id")?; + validate_required_sha256(&self.warming_presence_sha256, "warming presence sha256")?; + if self.lease_epoch == 0 || self.issued_at_unix_millis == 0 { + bail!("process write lease epoch or issue time is invalid"); + } + Ok(()) + } +} + +fn messagepack_array_len(payload: &[u8]) -> Option { + match *payload.first()? { + marker @ 0x90..=0x9f => Some(usize::from(marker & 0x0f)), + 0xdc => Some(usize::from(u16::from_be_bytes([ + *payload.get(1)?, + *payload.get(2)?, + ]))), + 0xdd => usize::try_from(u32::from_be_bytes([ + *payload.get(1)?, + *payload.get(2)?, + *payload.get(3)?, + *payload.get(4)?, + ])) + .ok(), + _ => None, + } +} + +fn validate_authority_identifier(value: &str, label: &str) -> Result<()> { + if value.is_empty() + || value.len() > 256 + || value.trim() != value + || value.chars().any(char::is_control) + { + bail!("{label} is empty, oversized, padded, or contains control characters"); + } + Ok(()) +} + +fn validate_optional_identifier(value: &Option, label: &str) -> Result<()> { + if let Some(value) = value { + validate_authority_identifier(value, label)?; + } + Ok(()) +} + +fn validate_source_repository(value: &str) -> Result<()> { + validate_authority_identifier(value, "source repository")?; + let segments: Vec<&str> = value.split('/').collect(); + if segments.len() < 3 + || !segments[0].contains('.') + || segments.iter().any(|segment| { + segment.is_empty() + || matches!(*segment, "." | "..") + || !segment + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_' | b'.')) + }) + { + bail!("source repository is not a canonical host/owner/repository identity"); + } + Ok(()) +} + +fn validate_endpoint(value: &str, label: &str) -> Result<()> { + if value.is_empty() + || value.len() > 2048 + || value.trim() != value + || value.chars().any(char::is_control) + { + bail!("{label} is empty, oversized, padded, or contains control characters"); + } + Ok(()) +} + +fn validate_optional_endpoint_labeled(value: &Option, label: &str) -> Result<()> { + if let Some(endpoint) = value { + validate_endpoint(endpoint, label)?; + } + Ok(()) +} + +fn validate_optional_endpoint(value: &Option) -> Result<()> { + validate_optional_endpoint_labeled(value, "bound endpoint") +} + +fn validate_runtime_capabilities(capabilities: &[GameCultRuntimeCapability]) -> Result<()> { + if capabilities.len() > MAX_RUNTIME_CAPABILITIES { + bail!("runtime presence capability count exceeds its contract bound"); + } + let mut previous: Option<(&str, &str, &str)> = None; + for capability in capabilities { + validate_authority_identifier(&capability.capability, "capability")?; + validate_authority_identifier(&capability.schema, "capability schema")?; + validate_authority_identifier(&capability.compatibility, "capability compatibility")?; + if capability.capacity == 0 { + bail!("runtime capability capacity is zero"); + } + let identity = ( + capability.capability.as_str(), + capability.schema.as_str(), + capability.compatibility.as_str(), + ); + if previous.is_some_and(|previous| previous >= identity) { + bail!("runtime capabilities are not strictly sorted and unique"); + } + previous = Some(identity); + } + Ok(()) +} + +fn validate_expected_capabilities(capabilities: &[IdunnExpectedCapability]) -> Result<()> { + if capabilities.len() > MAX_RUNTIME_CAPABILITIES { + bail!("Expected capability count exceeds its contract bound"); + } + let mut previous: Option<(&str, &str, &str)> = None; + for capability in capabilities { + validate_authority_identifier(&capability.capability, "Expected capability")?; + validate_authority_identifier(&capability.schema, "Expected capability schema")?; + validate_authority_identifier( + &capability.compatibility, + "Expected capability compatibility", + )?; + if capability.minimum_capacity == 0 { + bail!("Expected capability minimum capacity is zero"); + } + let identity = capability.identity(); + if previous.is_some_and(|previous| previous >= identity) { + bail!("Expected capabilities are not strictly sorted and unique"); + } + previous = Some(identity); + } + Ok(()) +} + +fn validate_expected_dependencies(dependencies: &[IdunnExpectedDependency]) -> Result<()> { + if dependencies.len() > MAX_RUNTIME_DEPENDENCIES { + bail!("Expected dependency count exceeds its contract bound"); + } + let mut previous: Option<(&str, &str, &str)> = None; + for dependency in dependencies { + dependency.validate()?; + let identity = dependency.identity(); + if previous.is_some_and(|previous| previous >= identity) { + bail!("Expected dependencies are not strictly sorted and unique"); + } + previous = Some(identity); + } + Ok(()) +} + +fn validate_dependency_evidence(dependencies: &[OdinRuntimeDependencyEvidence]) -> Result<()> { + if dependencies.len() > MAX_RUNTIME_DEPENDENCIES { + bail!("dependency evidence count exceeds its contract bound"); + } + let mut previous: Option<(&str, &str, &str)> = None; + for dependency in dependencies { + dependency.validate()?; + let identity = dependency.identity(); + if previous.is_some_and(|previous| previous >= identity) { + bail!("dependency evidence is not strictly sorted and unique"); + } + previous = Some(identity); + } + Ok(()) +} + +fn validate_topology_disagreements(disagreements: &[OdinTopologyDisagreement]) -> Result<()> { + if disagreements.len() > MAX_TOPOLOGY_DISAGREEMENTS { + bail!("topology disagreement count exceeds its contract bound"); + } + let mut previous: Option<&str> = None; + for disagreement in disagreements { + disagreement.validate()?; + if previous.is_some_and(|previous| previous >= disagreement.code.as_str()) { + bail!("topology disagreements are not strictly sorted and unique by code"); + } + previous = Some(&disagreement.code); + } + Ok(()) +} + +fn validate_optional_detail(value: &Option, label: &str) -> Result<()> { + if value.as_deref().is_some_and(|value| { + value.is_empty() + || value.len() > 1024 + || value.trim() != value + || value.chars().any(char::is_control) + }) { + bail!("{label} is empty, oversized, padded, or contains control characters"); + } + Ok(()) +} + +fn validate_required_sha256(value: &str, label: &str) -> Result<()> { + if !is_sha256(value) { + bail!("{label} is malformed"); + } + Ok(()) +} + +fn validate_optional_sha256(value: &Option, label: &str) -> Result<()> { + if value.as_deref().is_some_and(|value| !is_sha256(value)) { + bail!("{label} is malformed"); + } + Ok(()) +} + +fn prefixed_sha256(bytes: &[u8]) -> String { + format!("sha256-{:x}", Sha256::digest(bytes)) +} + +fn is_sha256(value: &str) -> bool { + value + .strip_prefix("sha256-") + .is_some_and(|digest| is_lower_hex(digest, 64)) +} + +fn is_lower_hex(value: &str, length: usize) -> bool { + value.len() == length + && value + .bytes() + .all(|byte| byte.is_ascii_hexdigit() && !byte.is_ascii_uppercase()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{ + GAMECULT_SERVICE_TRUST_ANCHOR_SCHEMA, GameCultProviderHealthIdentity, + GameCultRuntimePresenceHealthPurpose, GameCultServiceTrustAnchorRecord, + IDUNN_SIGNED_DAEMON_HEALTH_SCHEMA, IdunnServiceIdentity, + OdinRuntimeTopologyCorrelationPurpose, OdinTopologyIdentity, ServiceIdentitySignature, + ServiceIdentitySigner, ServiceSignaturePurpose, derive_service_identity_id, + enroll_service_identity_at, verify_service_identity_signature, + }; + + struct WrongOdinTopologyPurpose; + + impl ServiceSignaturePurpose for WrongOdinTopologyPurpose { + const PURPOSE: &'static [u8] = b"odin.not-runtime-topology-correlation.v1"; + } + + fn digest(byte: char) -> String { + format!("sha256-{}", byte.to_string().repeat(64)) + } + + fn runtime_capability(name: &str) -> GameCultRuntimeCapability { + GameCultRuntimeCapability { + capability: name.into(), + schema: format!("{name}.v1"), + compatibility: "v1".into(), + capacity: 1, + } + } + + fn expected_capability(name: &str) -> IdunnExpectedCapability { + IdunnExpectedCapability { + capability: name.into(), + schema: format!("{name}.v1"), + compatibility: "v1".into(), + minimum_capacity: 1, + } + } + + fn runtime_presence() -> GameCultRuntimePresenceHealthRecord { + GameCultRuntimePresenceHealthRecord { + schema_version: GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA.into(), + target: "ghostlight".into(), + expected_projection_sha256: digest('9'), + plan_id: digest('e'), + incarnation_id: "ghostlight/2026-09-02/1".into(), + sealed_release_id: digest('f'), + activation_witness_sha256: digest('a'), + state_schema_generation: Some("world-v2".into()), + state_contract_sha256: Some(digest('b')), + runtime_id: "ghostlight-yggdrasil-1".into(), + runtime_instance_id: digest('8'), + bound_endpoint: Some("http://127.0.0.1:14103".into()), + capabilities: vec![ + runtime_capability("conversation"), + runtime_capability("state"), + ], + health_contract: "ghostlight.runtime-health.v1".into(), + state: "active".into(), + detail: "ready".into(), + write_lease_sha256: Some(digest('c')), + signer_identity_id: "provider-signing-key".into(), + publisher_sequence: 1, + observed_at_unix_millis: 100, + signature_algorithm: "ed25519".into(), + signature: vec![6; 64], + activation_signer_identity_id: "activation-signing-key".into(), + activation_signature: vec![7; 64], + } + } + + fn runtime_activation() -> IdunnRuntimeActivationRecord { + let signer = IdunnRuntimeActivationSigner::from_credential_reader(&[8; 32][..]).unwrap(); + IdunnRuntimeActivationRecord { + schema_version: IDUNN_RUNTIME_ACTIVATION_SCHEMA.into(), + expected_projection_sha256: digest('9'), + runtime_id: "ghostlight-yggdrasil-1".into(), + runtime_instance_id: digest('8'), + activation_signer_identity_id: signer.identity_id(), + activation_signer_public_key: signer.public_key(), + issued_at_unix_millis: 99, + idunn_signer_identity_id: "idunn-signing-key".into(), + signature_algorithm: "ed25519".into(), + signature: vec![5; 64], + } + } + + fn process_write_lease() -> IdunnProcessWriteLeaseRecord { + IdunnProcessWriteLeaseRecord { + schema_version: IDUNN_PROCESS_WRITE_LEASE_SCHEMA.into(), + target: "ghostlight".into(), + expected_projection_sha256: digest('9'), + plan_id: digest('e'), + incarnation_id: "ghostlight/2026-09-02/1".into(), + sealed_release_id: digest('f'), + activation_witness_sha256: digest('a'), + state_schema_generation: "world-v2".into(), + state_contract_sha256: digest('b'), + runtime_id: "ghostlight-yggdrasil-1".into(), + runtime_instance_id: digest('8'), + warming_presence_sha256: digest('d'), + lease_epoch: 1, + issued_at_unix_millis: 101, + } + } + + fn expected_dependency( + kind: &str, + capability: &str, + provider_id: Option<&str>, + ) -> IdunnExpectedDependency { + IdunnExpectedDependency { + kind: kind.into(), + capability: capability.into(), + schema: format!("{capability}.v1"), + compatibility: "v1".into(), + minimum_capacity: 1, + startup: if kind == "bootstrap" { + "before-start".into() + } else { + "before-promotion".into() + }, + provider_id: provider_id.map(str::to_string), + provider_authority: provider_id.map(|_| "managed-incarnation".into()), + provider_expected_projection_sha256: provider_id.map(|_| digest('1')), + provider_endpoint: provider_id.map(|_| "tcp://127.0.0.1:14100".into()), + } + } + + fn expected_incarnation() -> IdunnExpectedIncarnationRecord { + IdunnExpectedIncarnationRecord { + schema_version: IDUNN_EXPECTED_INCARNATION_SCHEMA.into(), + target: "ghostlight".into(), + plan_id: digest('2'), + incarnation_id: "ghostlight/2026-09-02/1".into(), + sealed_release_id: digest('3'), + source_repository: "github.com/GameCult/Ghostlight".into(), + source_revision: "4".repeat(40), + recipe_sha256: digest('5'), + runtime_id: "ghostlight-yggdrasil-1".into(), + expected_signer_identity_id: "ghostlight-provider-signing-key".into(), + health_contract: "ghostlight.runtime-health.v1".into(), + artifact_sha256: digest('6'), + state_schema_generation: Some("world-v2".into()), + state_contract_sha256: Some(digest('7')), + write_lease_required: true, + route: Some(IdunnExpectedRoute { + route_id: "ghostlight-public".into(), + transport: "http".into(), + stable_endpoint: "https://ghostlight.gamecult.net".into(), + candidate_endpoint: "http://127.0.0.1:14103".into(), + }), + capabilities: vec![ + expected_capability("conversation"), + expected_capability("state"), + ], + dependencies: vec![ + expected_dependency("shared-infrastructure", "rendezvous", Some("odin-ygg-1")), + expected_dependency("optional", "telemetry", None), + ], + } + } + + fn dependency_evidence(requirement: &IdunnExpectedDependency) -> OdinRuntimeDependencyEvidence { + let selected = requirement.provider_id.is_some(); + OdinRuntimeDependencyEvidence { + kind: requirement.kind.clone(), + capability: requirement.capability.clone(), + schema: requirement.schema.clone(), + compatibility: requirement.compatibility.clone(), + provider_id: requirement.provider_id.clone(), + provider_authority: requirement.provider_authority.clone(), + provider_expected_projection_sha256: requirement + .provider_expected_projection_sha256 + .clone(), + provider_endpoint: requirement.provider_endpoint.clone(), + observed_capacity: selected.then_some(1), + provider_evidence_sha256: selected.then(|| digest('8')), + ready: selected, + } + } + + fn topology_correlation( + expected: &IdunnExpectedIncarnationRecord, + ) -> OdinRuntimeTopologyCorrelationRecord { + OdinRuntimeTopologyCorrelationRecord { + schema_version: ODIN_RUNTIME_TOPOLOGY_CORRELATION_SCHEMA.into(), + target: expected.target.clone(), + expected_projection_sha256: expected.canonical_sha256().unwrap(), + expected: true, + current_activation_sha256: Some(digest('9')), + signed_presence_sha256: Some(digest('a')), + observed_presence_state: Some("active".into()), + observed_presence_publisher_sequence: Some(1), + observed_write_lease_sha256: Some(digest('c')), + observed_capabilities: expected + .capabilities + .iter() + .map(|capability| GameCultRuntimeCapability { + capability: capability.capability.clone(), + schema: capability.schema.clone(), + compatibility: capability.compatibility.clone(), + capacity: capability.minimum_capacity, + }) + .collect(), + runtime_id: expected.runtime_id.clone(), + runtime_instance_id: Some(digest('b')), + present: true, + ready: true, + dependencies: expected + .dependencies + .iter() + .map(dependency_evidence) + .collect(), + disagreements: vec![], + signer_identity_id: "odin-topology-signing-key".into(), + publisher_sequence: 1, + observed_at_unix_millis: 102, + signature_algorithm: "ed25519".into(), + signature: vec![7; 64], + } + } + + fn validate_against_current_lease( + receipt: &OdinRuntimeTopologyCorrelationRecord, + expected: &IdunnExpectedIncarnationRecord, + ) -> Result<()> { + let current_lease = digest('c'); + receipt.validate_against_expected(expected, Some(¤t_lease)) + } + + fn topology_anchor() -> GameCultServiceTrustAnchorRecord { + let public_key = vec![9; 32]; + GameCultServiceTrustAnchorRecord { + schema_version: GAMECULT_SERVICE_TRUST_ANCHOR_SCHEMA.into(), + trust_anchor_id: "root/odin/runtime-topology".into(), + service_id: "odin".into(), + runtime_id: "odin-yggdrasil-1".into(), + signer_identity_id: derive_service_identity_id::(&public_key) + .unwrap(), + signer_public_key: public_key, + signature_algorithm: "ed25519".into(), + signing_purpose: ODIN_RUNTIME_TOPOLOGY_CORRELATION_SIGNING_PURPOSE.into(), + signed_schema: ODIN_RUNTIME_TOPOLOGY_CORRELATION_SCHEMA.into(), + binding_authority: "root".into(), + bound_at_unix_millis: 100, + expires_at_unix_millis: Some(200), + private_state_exposed: false, + } + } + + fn runtime_presence_anchor() -> GameCultServiceTrustAnchorRecord { + let public_key = vec![7; 32]; + GameCultServiceTrustAnchorRecord { + schema_version: GAMECULT_SERVICE_TRUST_ANCHOR_SCHEMA.into(), + trust_anchor_id: "root/ghostlight/runtime-presence".into(), + service_id: "ghostlight".into(), + runtime_id: "ghostlight-yggdrasil-1".into(), + signer_identity_id: derive_service_identity_id::( + &public_key, + ) + .unwrap(), + signer_public_key: public_key, + signature_algorithm: "ed25519".into(), + signing_purpose: GAMECULT_RUNTIME_PRESENCE_HEALTH_SIGNING_PURPOSE.into(), + signed_schema: GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA.into(), + binding_authority: "root".into(), + bound_at_unix_millis: 100, + expires_at_unix_millis: Some(200), + private_state_exposed: false, + } + } + + struct DualProofFixture { + stable_signer: ServiceIdentitySigner, + activation_signer: IdunnRuntimeActivationSigner, + odin_signer: ServiceIdentitySigner, + idunn_anchor: crate::ServiceIdentityTrustAnchor, + stable_public_key: Vec, + odin_anchor: GameCultServiceTrustAnchorRecord, + expected: IdunnExpectedIncarnationRecord, + activation: IdunnRuntimeActivationRecord, + presence: GameCultRuntimePresenceHealthRecord, + } + + fn dual_proof_fixture(identity_path: &std::path::Path) -> Result { + let stable_signer = + enroll_service_identity_at::(identity_path)?; + let stable_public = stable_signer.trust_anchor()?; + let idunn_signer = enroll_service_identity_at::( + &identity_path.with_extension("idunn.cc"), + )?; + let idunn_anchor = idunn_signer.trust_anchor()?; + let odin_signer = enroll_service_identity_at::( + &identity_path.with_extension("odin.cc"), + )?; + let odin_public = odin_signer.trust_anchor()?; + let mut expected = expected_incarnation(); + expected.expected_signer_identity_id = stable_public.identity_id.clone(); + expected.capabilities[0].minimum_capacity = 2; + let launch = + IdunnRuntimeActivationLaunch::issue(&expected, digest('8'), 99, &idunn_signer)?; + let activation = launch.activation().clone(); + let mut activation_credential = Vec::new(); + assert_eq!( + launch.write_credential(&mut activation_credential)?, + activation + ); + let activation_signer = + IdunnRuntimeActivationSigner::from_credential_reader(&activation_credential[..])?; + + let stable_public_key = stable_public.public_key.clone(); + let odin_anchor = GameCultServiceTrustAnchorRecord { + schema_version: GAMECULT_SERVICE_TRUST_ANCHOR_SCHEMA.into(), + trust_anchor_id: "root/odin/runtime-topology".into(), + service_id: "odin".into(), + runtime_id: "odin-yggdrasil-1".into(), + signer_identity_id: odin_public.identity_id, + signer_public_key: odin_public.public_key, + signature_algorithm: "ed25519".into(), + signing_purpose: ODIN_RUNTIME_TOPOLOGY_CORRELATION_SIGNING_PURPOSE.into(), + signed_schema: ODIN_RUNTIME_TOPOLOGY_CORRELATION_SCHEMA.into(), + binding_authority: "root".into(), + bound_at_unix_millis: 1, + expires_at_unix_millis: Some(1_000), + private_state_exposed: false, + }; + + let mut presence = runtime_presence(); + presence.target = expected.target.clone(); + presence.plan_id = expected.plan_id.clone(); + presence.incarnation_id = expected.incarnation_id.clone(); + presence.sealed_release_id = expected.sealed_release_id.clone(); + presence.state_schema_generation = expected.state_schema_generation.clone(); + presence.state_contract_sha256 = expected.state_contract_sha256.clone(); + presence.bound_endpoint = expected + .route + .as_ref() + .map(|route| route.candidate_endpoint.clone()); + presence.capabilities = expected + .capabilities + .iter() + .map(|capability| GameCultRuntimeCapability { + capability: capability.capability.clone(), + schema: capability.schema.clone(), + compatibility: capability.compatibility.clone(), + capacity: capability.minimum_capacity, + }) + .collect(); + presence.health_contract = expected.health_contract.clone(); + presence.runtime_id = activation.runtime_id.clone(); + presence.runtime_instance_id = activation.runtime_instance_id.clone(); + presence.expected_projection_sha256 = activation.expected_projection_sha256.clone(); + presence.activation_witness_sha256 = activation.canonical_sha256()?; + presence.signer_identity_id = stable_public.identity_id; + presence.activation_signer_identity_id = activation_signer.identity_id(); + presence.observed_at_unix_millis = 110; + presence.signature.clear(); + presence.activation_signature.clear(); + let proof_payload = presence.canonical_proof_payload()?; + presence.signature = stable_signer + .sign::(&proof_payload) + .signature; + presence.activation_signature = activation_signer.sign_presence_proof(&presence)?; + + Ok(DualProofFixture { + stable_signer, + activation_signer, + odin_signer, + idunn_anchor, + stable_public_key, + odin_anchor, + expected, + activation, + presence, + }) + } + + fn resign_presence(fixture: &mut DualProofFixture) -> Result> { + fixture.presence.signature.clear(); + fixture.presence.activation_signature.clear(); + let proof_payload = fixture.presence.canonical_proof_payload()?; + fixture.presence.signature = fixture + .stable_signer + .sign::(&proof_payload) + .signature; + fixture.presence.activation_signature = fixture + .activation_signer + .sign_presence_proof(&fixture.presence)?; + Ok(rmp_serde::to_vec(&fixture.presence)?) + } + + fn fixture_authority(fixture: &DualProofFixture) -> Result { + verify_runtime_authority( + &fixture.expected, + &fixture.activation, + &fixture.idunn_anchor, + &fixture.stable_public_key, + ) + } + + fn authenticate_fixture( + fixture: &DualProofFixture, + ) -> Result { + authenticate_runtime_presence_claim( + &rmp_serde::to_vec(&fixture.presence)?, + &fixture_authority(fixture)?, + RuntimePresenceAuthenticationContext { + trusted_received_at_unix_millis: 120, + maximum_age_millis: 30, + maximum_future_skew_millis: 5, + }, + ) + } + + fn correlate_fixture(fixture: &DualProofFixture) -> Result { + correlate_runtime_presence_claim( + authenticate_fixture(fixture)?, + &fixture_authority(fixture)?, + ) + } + + fn verify_fixture(fixture: &DualProofFixture) -> Result { + correlate_fixture(fixture)?.into_undisputed_present() + } + + fn authenticated_topology_receipt( + fixture: &DualProofFixture, + presence: &VerifiedRuntimePresence, + present: bool, + disagreements: Vec, + ) -> Result { + let authority = fixture_authority(fixture)?; + let mut receipt = topology_correlation(&fixture.expected); + receipt.current_activation_sha256 = Some(authority.activation_sha256().into()); + receipt.signed_presence_sha256 = Some(presence.signed_presence_sha256().into()); + receipt.observed_presence_state = Some(presence.record().state.clone()); + receipt.observed_presence_publisher_sequence = Some(presence.record().publisher_sequence); + receipt.observed_write_lease_sha256 = presence.record().write_lease_sha256.clone(); + receipt.observed_capabilities = presence.record().capabilities.clone(); + receipt.runtime_instance_id = Some(fixture.activation.runtime_instance_id.clone()); + receipt.present = present; + receipt.ready = present && presence.record().state == "active"; + receipt.disagreements = disagreements; + receipt.signer_identity_id = fixture.odin_anchor.signer_identity_id.clone(); + receipt.signature.clear(); + receipt.signature = fixture + .odin_signer + .sign::(&receipt.unsigned_signature_payload()?) + .signature; + let encoded = receipt.canonical_bytes()?; + authenticate_odin_runtime_topology_correlation( + &encoded, + &authority, + presence.record().write_lease_sha256.as_deref(), + &fixture.odin_anchor.signer_public_key, + OdinTopologyAuthenticationContext { + trusted_received_at_unix_millis: 120, + maximum_age_millis: 30, + maximum_future_skew_millis: 5, + }, + ) + } + + #[test] + fn runtime_authority_contracts_round_trip_as_canonical_positional_records() -> Result<()> { + let presence = runtime_presence(); + presence.validate()?; + let encoded_presence = rmp_serde::to_vec(&presence)?; + assert_eq!(messagepack_array_len(&encoded_presence), Some(24)); + let decoded_presence: GameCultRuntimePresenceHealthRecord = + rmp_serde::from_slice(&encoded_presence)?; + assert_eq!(decoded_presence, presence); + let unsigned = presence.canonical_proof_payload()?; + let unsigned_presence: GameCultRuntimePresenceHealthRecord = + rmp_serde::from_slice(&unsigned)?; + assert!(unsigned_presence.signature.is_empty()); + assert!(unsigned_presence.activation_signature.is_empty()); + + let activation = runtime_activation(); + let encoded_activation = activation.canonical_bytes()?; + assert_eq!(messagepack_array_len(&encoded_activation), Some(10)); + assert_eq!( + IdunnRuntimeActivationRecord::decode_canonical(&encoded_activation)?, + activation + ); + assert_eq!( + activation.canonical_sha256()?, + prefixed_sha256(&encoded_activation) + ); + + let lease = process_write_lease(); + lease.validate()?; + let encoded_lease = lease.canonical_bytes()?; + assert_eq!(messagepack_array_len(&encoded_lease), Some(14)); + assert_eq!( + IdunnProcessWriteLeaseRecord::decode_canonical(&encoded_lease)?, + lease + ); + assert_eq!(lease.canonical_sha256()?, prefixed_sha256(&encoded_lease)); + Ok(()) + } + + #[test] + fn exact_stable_and_activation_proofs_admit_runtime_presence() -> Result<()> { + let temp = tempfile::tempdir()?; + let fixture = dual_proof_fixture(&temp.path().join("runtime-presence.cc"))?; + assert_eq!(fixture.activation.activation_signer_public_key.len(), 32); + let verified = verify_fixture(&fixture)?; + assert_eq!(verified.record(), &fixture.presence); + Ok(()) + } + + #[test] + fn authenticated_expected_disagreements_remain_typed_present_and_not_ready() -> Result<()> { + type Mutation = fn(&mut GameCultRuntimePresenceHealthRecord); + let cases: &[(&str, &str, Mutation)] = &[ + ("target", "target", |value| value.target = "epiphany".into()), + ("projection", "expected-projection", |value| { + value.expected_projection_sha256 = digest('0') + }), + ("plan", "plan-id", |value| value.plan_id = digest('0')), + ("incarnation", "incarnation-id", |value| { + value.incarnation_id = "ghostlight/other/1".into() + }), + ("release", "sealed-release-id", |value| { + value.sealed_release_id = digest('0') + }), + ("runtime", "runtime-id", |value| { + value.runtime_id = "ghostlight-other-runtime".into() + }), + ("process", "runtime-instance-id", |value| { + value.runtime_instance_id = digest('0') + }), + ("endpoint", "bound-endpoint", |value| { + value.bound_endpoint = Some("http://127.0.0.1:14104".into()) + }), + ("health", "health-contract", |value| { + value.health_contract = "ghostlight.other-health.v1".into() + }), + ("state-generation", "state-schema-generation", |value| { + value.state_schema_generation = Some("world-v3".into()) + }), + ("state-contract", "state-contract", |value| { + value.state_contract_sha256 = Some(digest('0')) + }), + ("activation", "activation-witness", |value| { + value.activation_witness_sha256 = digest('0') + }), + ]; + + let temp = tempfile::tempdir()?; + for (name, expected_code, mutate) in cases { + let mut fixture = dual_proof_fixture(&temp.path().join(format!("{name}.cc")))?; + mutate(&mut fixture.presence); + resign_presence(&mut fixture)?; + let correlation = correlate_fixture(&fixture)?; + assert!( + correlation + .disagreements() + .iter() + .any(|disagreement| disagreement.code == *expected_code), + "missing {expected_code} disagreement for {name}" + ); + assert_eq!( + correlation.claim().record().runtime_instance_id, + fixture.presence.runtime_instance_id + ); + assert!(correlation.into_undisputed_present().is_err()); + } + Ok(()) + } + + #[test] + fn provider_owns_observed_capacity_while_expected_owns_only_the_minimum() -> Result<()> { + let temp = tempfile::tempdir()?; + let mut fixture = dual_proof_fixture(&temp.path().join("runtime-presence.cc"))?; + fixture.presence.capabilities[0].capacity = 50; + fixture + .presence + .capabilities + .push(runtime_capability("telemetry")); + resign_presence(&mut fixture)?; + verify_fixture(&fixture)?; + + fixture.presence.capabilities[0].capacity = 1; + fixture + .presence + .capabilities + .retain(|capability| capability.capability != "state"); + resign_presence(&mut fixture)?; + let correlation = correlate_fixture(&fixture)?; + let codes = correlation + .disagreements() + .iter() + .map(|disagreement| disagreement.code.as_str()) + .collect::>(); + assert!(codes.contains(&"expected-capability-000-capacity")); + assert!(codes.contains(&"expected-capability-001-missing")); + assert!(!codes.iter().any(|code| code.contains("unexpected"))); + Ok(()) + } + + #[test] + fn stable_proof_and_provider_key_substitution_fail_closed() -> Result<()> { + let temp = tempfile::tempdir()?; + let mut fixture = dual_proof_fixture(&temp.path().join("runtime-presence.cc"))?; + fixture.presence.detail = "tampered after signing".into(); + assert!(authenticate_fixture(&fixture).is_err()); + + let other_idunn = enroll_service_identity_at::( + &temp.path().join("other-idunn.cc"), + )?; + assert!( + verify_runtime_authority( + &fixture.expected, + &fixture.activation, + &other_idunn.trust_anchor()?, + &fixture.stable_public_key, + ) + .is_err() + ); + + fixture.stable_public_key = vec![9; 32]; + assert!(fixture_authority(&fixture).is_err()); + Ok(()) + } + + #[test] + fn incumbent_stable_key_without_activation_key_cannot_claim_presence() -> Result<()> { + let temp = tempfile::tempdir()?; + let mut fixture = dual_proof_fixture(&temp.path().join("runtime-presence.cc"))?; + fixture.presence.activation_signature = vec![0; 64]; + let proof_payload = fixture.presence.canonical_proof_payload()?; + fixture.presence.signature = fixture + .stable_signer + .sign::(&proof_payload) + .signature; + + let error = verify_fixture(&fixture).unwrap_err(); + assert!(error.to_string().contains("activation proof verification")); + Ok(()) + } + + #[test] + fn activation_cannot_substitute_another_expected_runtime_id() -> Result<()> { + let temp = tempfile::tempdir()?; + let mut fixture = dual_proof_fixture(&temp.path().join("runtime-presence.cc"))?; + fixture.activation.runtime_id = "ghostlight-yggdrasil-2".into(); + fixture.presence.runtime_id = fixture.activation.runtime_id.clone(); + fixture.presence.activation_witness_sha256 = fixture.activation.canonical_sha256()?; + + // The stable service key can re-sign the altered canonical statement, + // but the activation proof still came from the other runtime id. + let proof_payload = fixture.presence.canonical_proof_payload()?; + fixture.presence.signature = fixture + .stable_signer + .sign::(&proof_payload) + .signature; + let error = fixture_authority(&fixture).unwrap_err(); + assert!(error.to_string().contains("current Expected")); + Ok(()) + } + + #[test] + fn idunn_signature_prevents_provider_minted_activation() -> Result<()> { + let temp = tempfile::tempdir()?; + let mut fixture = dual_proof_fixture(&temp.path().join("runtime-presence.cc"))?; + fixture.activation.issued_at_unix_millis += 1; + fixture.presence.activation_witness_sha256 = fixture.activation.canonical_sha256()?; + resign_presence(&mut fixture)?; + let error = fixture_authority(&fixture).unwrap_err(); + assert!( + error + .to_string() + .contains("Idunn runtime activation signature") + ); + Ok(()) + } + + #[test] + fn authenticated_odin_receipt_is_fresh_evidence_not_replay_authority() -> Result<()> { + let temp = tempfile::tempdir()?; + let fixture = dual_proof_fixture(&temp.path().join("runtime-presence.cc"))?; + let authority = fixture_authority(&fixture)?; + let presence = verify_fixture(&fixture)?; + let accepted = authenticated_topology_receipt(&fixture, &presence, true, Vec::new())?; + assert_eq!( + accepted.record().observed_presence_publisher_sequence, + Some(fixture.presence.publisher_sequence) + ); + assert_eq!( + accepted.record().signed_presence_sha256.as_deref(), + Some(presence.signed_presence_sha256()) + ); + + let mut tampered = accepted.record().clone(); + tampered.signature = vec![0; 64]; + assert!( + authenticate_odin_runtime_topology_correlation( + &tampered.canonical_bytes()?, + &authority, + presence.record().write_lease_sha256.as_deref(), + &fixture.odin_anchor.signer_public_key, + OdinTopologyAuthenticationContext { + trusted_received_at_unix_millis: 120, + maximum_age_millis: 30, + maximum_future_skew_millis: 5, + }, + ) + .is_err() + ); + assert!( + authenticate_odin_runtime_topology_correlation( + accepted.canonical_bytes(), + &authority, + presence.record().write_lease_sha256.as_deref(), + &fixture.odin_anchor.signer_public_key, + OdinTopologyAuthenticationContext { + trusted_received_at_unix_millis: 200, + maximum_age_millis: 30, + maximum_future_skew_millis: 5, + }, + ) + .is_err() + ); + Ok(()) + } + + #[test] + fn separate_launches_receive_distinct_activation_identities() -> Result<()> { + let temp = tempfile::tempdir()?; + let idunn = + enroll_service_identity_at::(&temp.path().join("idunn.cc"))?; + let expected = expected_incarnation(); + let first = IdunnRuntimeActivationLaunch::issue(&expected, digest('8'), 99, &idunn)?; + let second = IdunnRuntimeActivationLaunch::issue(&expected, digest('9'), 100, &idunn)?; + assert_ne!( + first.activation().activation_signer_identity_id, + second.activation().activation_signer_identity_id + ); + Ok(()) + } + + #[test] + fn trusted_time_rejects_stale_and_future_presence() -> Result<()> { + let temp = tempfile::tempdir()?; + let mut fixture = dual_proof_fixture(&temp.path().join("runtime-presence.cc"))?; + let authority = fixture_authority(&fixture)?; + let canonical = rmp_serde::to_vec(&fixture.presence)?; + assert!( + authenticate_runtime_presence_claim( + &canonical, + &authority, + RuntimePresenceAuthenticationContext { + trusted_received_at_unix_millis: 200, + maximum_age_millis: 10, + maximum_future_skew_millis: 5, + }, + ) + .is_err() + ); + + fixture.presence.observed_at_unix_millis = 126; + resign_presence(&mut fixture)?; + assert!( + authenticate_runtime_presence_claim( + &rmp_serde::to_vec(&fixture.presence)?, + &authority, + RuntimePresenceAuthenticationContext { + trusted_received_at_unix_millis: 120, + maximum_age_millis: 30, + maximum_future_skew_millis: 5, + }, + ) + .is_err() + ); + + Ok(()) + } + + #[test] + fn runtime_authority_decoders_reject_malformed_and_noncanonical_bytes() -> Result<()> { + let temp = tempfile::tempdir()?; + let fixture = dual_proof_fixture(&temp.path().join("runtime-presence.cc"))?; + assert!( + authenticate_runtime_presence_claim( + &[0x91, 0xc0], + &fixture_authority(&fixture)?, + RuntimePresenceAuthenticationContext { + trusted_received_at_unix_millis: 120, + maximum_age_millis: 30, + maximum_future_skew_millis: 5, + }, + ) + .is_err() + ); + assert!(IdunnRuntimeActivationRecord::decode_canonical(&[0x91, 0xc0]).is_err()); + assert!(IdunnProcessWriteLeaseRecord::decode_canonical(&[0x91, 0xc0]).is_err()); + + let mut legacy_presence = fixture.presence.clone(); + legacy_presence.schema_version = "gamecult.runtime_presence_health.v1".into(); + assert!( + authenticate_runtime_presence_claim( + &rmp_serde::to_vec(&legacy_presence)?, + &fixture_authority(&fixture)?, + RuntimePresenceAuthenticationContext { + trusted_received_at_unix_millis: 120, + maximum_age_millis: 30, + maximum_future_skew_millis: 5, + }, + ) + .is_err() + ); + let mut legacy_activation = fixture.activation.clone(); + legacy_activation.schema_version = "idunn.runtime_activation.v1".into(); + assert!( + IdunnRuntimeActivationRecord::decode_canonical(&rmp_serde::to_vec(&legacy_activation)?) + .is_err() + ); + let mut legacy_expected = fixture.expected.clone(); + legacy_expected.schema_version = "idunn.expected_incarnation.v1".into(); + assert!( + IdunnExpectedIncarnationRecord::decode_canonical(&rmp_serde::to_vec(&legacy_expected)?) + .is_err() + ); + let mut legacy_topology = topology_correlation(&fixture.expected); + legacy_topology.schema_version = "odin.runtime_topology_correlation.v1".into(); + assert!( + OdinRuntimeTopologyCorrelationRecord::decode_canonical_signed_payload( + &rmp_serde::to_vec(&legacy_topology)?, + ) + .is_err() + ); + + let presence = rmp_serde::to_vec(&runtime_presence())?; + assert_eq!(&presence[..3], &[0xdc, 0, 24]); + let mut noncanonical_presence = vec![0xdd, 0, 0, 0, 24]; + noncanonical_presence.extend_from_slice(&presence[3..]); + assert!( + authenticate_runtime_presence_claim( + &noncanonical_presence, + &fixture_authority(&fixture)?, + RuntimePresenceAuthenticationContext { + trusted_received_at_unix_millis: 120, + maximum_age_millis: 30, + maximum_future_skew_millis: 5, + }, + ) + .is_err() + ); + + let activation = runtime_activation().canonical_bytes()?; + assert_eq!(activation[0], 0x9a); + let mut noncanonical_activation = vec![0xdc, 0, 10]; + noncanonical_activation.extend_from_slice(&activation[1..]); + assert!(IdunnRuntimeActivationRecord::decode_canonical(&noncanonical_activation).is_err()); + + let lease = process_write_lease().canonical_bytes()?; + assert_eq!(lease[0], 0x9e); + let mut noncanonical_lease = vec![0xdc, 0, 14]; + noncanonical_lease.extend_from_slice(&lease[1..]); + assert!(IdunnProcessWriteLeaseRecord::decode_canonical(&noncanonical_lease).is_err()); + Ok(()) + } + + #[test] + fn runtime_presence_refuses_capability_and_warming_lease_ambiguity() { + let mut value = runtime_presence(); + value.capabilities.reverse(); + assert!(value.validate().is_err()); + + let mut value = runtime_presence(); + let mut duplicate = runtime_capability("state"); + duplicate.capacity = 2; + value.capabilities.push(duplicate); + assert!(value.validate().is_err()); + + let mut value = runtime_presence(); + value.capabilities[0].capacity = 0; + assert!(value.validate().is_err()); + + let mut value = runtime_presence(); + value.state = "warming".into(); + assert!(value.validate().is_err()); + value.write_lease_sha256 = None; + assert!(value.validate().is_ok()); + + let mut value = runtime_presence(); + value.state_schema_generation = None; + assert!(value.validate().is_err()); + value.state_contract_sha256 = None; + assert!(value.validate().is_err()); + value.write_lease_sha256 = None; + assert!(value.validate().is_ok()); + } + + #[test] + fn runtime_authority_refuses_malformed_digest_process_and_epoch_bindings() { + let mut value = runtime_presence(); + value.activation_witness_sha256 = digest('A'); + assert!(value.validate().is_err()); + + let mut value = runtime_presence(); + value.plan_id = "plan-1".into(); + assert!(value.validate().is_err()); + + let mut value = runtime_presence(); + value.sealed_release_id = "release-1".into(); + assert!(value.validate().is_err()); + + let mut value = runtime_presence(); + value.runtime_instance_id = "systemd-invocation-1".into(); + assert!(value.validate().is_err()); + + let mut value = runtime_presence(); + value.expected_projection_sha256 = "expected-1".into(); + assert!(value.validate().is_err()); + + let mut value = runtime_presence(); + value.write_lease_sha256 = Some("lease-1".into()); + assert!(value.validate().is_err()); + + let mut value = process_write_lease(); + value.warming_presence_sha256 = digest('D'); + assert!(value.validate().is_err()); + + let mut value = process_write_lease(); + value.plan_id = "plan-1".into(); + assert!(value.validate().is_err()); + + let mut value = process_write_lease(); + value.runtime_instance_id = "pid-4242".into(); + assert!(value.validate().is_err()); + + let mut value = process_write_lease(); + value.lease_epoch = 0; + assert!(value.validate().is_err()); + + let mut value = runtime_activation(); + value.runtime_instance_id = digest('A'); + assert!(value.validate().is_err()); + + let mut value = runtime_activation(); + value.expected_projection_sha256 = "expected-1".into(); + assert!(value.validate().is_err()); + } + + #[test] + fn runtime_presence_trust_anchor_is_provider_profile_and_purpose_bound() -> Result<()> { + let anchor = runtime_presence_anchor(); + anchor.validate()?; + + let mut value = anchor.clone(); + value.signing_purpose = "idunn.signed_daemon_health.v1".into(); + assert!(value.validate().is_err()); + + let mut value = anchor.clone(); + value.signed_schema = IDUNN_SIGNED_DAEMON_HEALTH_SCHEMA.into(); + assert!(value.validate().is_err()); + + let mut value = anchor.clone(); + value.signing_purpose = "gamecult.runtime_presence_health.v1".into(); + value.signed_schema = "gamecult.runtime_presence_health.v1".into(); + assert!(value.validate().is_err()); + + let mut value = anchor; + value.signer_identity_id = + derive_service_identity_id::(&value.signer_public_key)?; + assert!(value.validate().is_err()); + Ok(()) + } + + #[test] + fn process_write_lease_cannot_encode_route_or_traffic_authority() -> Result<()> { + let canonical = process_write_lease().canonical_bytes()?; + assert_eq!(messagepack_array_len(&canonical), Some(14)); + + for forbidden in ["route", "traffic"] { + let mut with_extra_authority = vec![0xdc, 0, 15]; + with_extra_authority.extend_from_slice(&canonical[1..]); + with_extra_authority.extend_from_slice(&rmp_serde::to_vec(&forbidden)?); + assert!(IdunnProcessWriteLeaseRecord::decode_canonical(&with_extra_authority).is_err()); + } + Ok(()) + } + + #[test] + fn expected_and_topology_correlation_round_trip_canonically() -> Result<()> { + let expected = expected_incarnation(); + let expected_bytes = expected.canonical_bytes()?; + assert_eq!(&expected_bytes[..3], &[0xdc, 0, 18]); + assert_eq!( + IdunnExpectedIncarnationRecord::decode_canonical(&expected_bytes)?, + expected + ); + assert_eq!( + expected.canonical_sha256()?, + prefixed_sha256(&expected_bytes) + ); + + let receipt = topology_correlation(&expected); + validate_against_current_lease(&receipt, &expected)?; + let receipt_bytes = receipt.canonical_bytes()?; + assert_eq!(&receipt_bytes[..3], &[0xdc, 0, 21]); + let (decoded, unsigned) = + OdinRuntimeTopologyCorrelationRecord::decode_canonical_signed_payload(&receipt_bytes)?; + assert_eq!(decoded, receipt); + let unsigned_receipt: OdinRuntimeTopologyCorrelationRecord = + rmp_serde::from_slice(&unsigned)?; + assert!(unsigned_receipt.signature.is_empty()); + assert_eq!(unsigned, receipt.unsigned_signature_payload()?); + assert_eq!(receipt.canonical_sha256()?, prefixed_sha256(&receipt_bytes)); + Ok(()) + } + + #[test] + fn expected_and_topology_decoders_reject_noncanonical_or_extended_records() -> Result<()> { + assert!(IdunnExpectedIncarnationRecord::decode_canonical(&[0x91, 0xc0]).is_err()); + assert!( + OdinRuntimeTopologyCorrelationRecord::decode_canonical_signed_payload(&[0x91, 0xc0]) + .is_err() + ); + + let expected = expected_incarnation().canonical_bytes()?; + let mut noncanonical_expected = vec![0xdd, 0, 0, 0, 18]; + noncanonical_expected.extend_from_slice(&expected[3..]); + assert!(IdunnExpectedIncarnationRecord::decode_canonical(&noncanonical_expected).is_err()); + + let receipt = topology_correlation(&expected_incarnation()).canonical_bytes()?; + let mut noncanonical_receipt = vec![0xdd, 0, 0, 0, 21]; + noncanonical_receipt.extend_from_slice(&receipt[3..]); + assert!( + OdinRuntimeTopologyCorrelationRecord::decode_canonical_signed_payload( + &noncanonical_receipt, + ) + .is_err() + ); + + for forbidden in ["private-host-path", "runner-secret"] { + let mut extended = vec![0xdc, 0, 19]; + extended.extend_from_slice(&expected[3..]); + extended.extend_from_slice(&rmp_serde::to_vec(&forbidden)?); + assert!(IdunnExpectedIncarnationRecord::decode_canonical(&extended).is_err()); + } + Ok(()) + } + + #[test] + fn expected_refuses_partial_state_and_noncanonical_claim_sets() { + let mut value = expected_incarnation(); + value.state_contract_sha256 = None; + assert!(value.validate().is_err()); + + let mut value = expected_incarnation(); + value.state_schema_generation = None; + assert!(value.validate().is_err()); + + let mut value = expected_incarnation(); + value.state_schema_generation = None; + value.state_contract_sha256 = None; + assert!(value.validate().is_err()); + value.write_lease_required = false; + assert!(value.validate().is_ok()); + + let mut value = expected_incarnation(); + value.capabilities.reverse(); + assert!(value.validate().is_err()); + + let mut value = expected_incarnation(); + value.capabilities.push(expected_capability("state")); + assert!(value.validate().is_err()); + + let mut value = expected_incarnation(); + value.dependencies.reverse(); + assert!(value.validate().is_err()); + + let mut value = expected_incarnation(); + value.dependencies.push(value.dependencies[1].clone()); + assert!(value.validate().is_err()); + + let mut value = expected_incarnation(); + let mut duplicate = value.dependencies[1].clone(); + duplicate.kind = "required".into(); + value.dependencies.push(duplicate); + assert!(value.validate().is_err()); + + let mut value = expected_incarnation(); + value.dependencies[0].provider_authority = None; + assert!(value.validate().is_err()); + + let mut value = expected_incarnation(); + value.source_revision = "A".repeat(40); + assert!(value.validate().is_err()); + + for private_source in [ + "F:\\Projects\\Ghostlight", + "/srv/build/Ghostlight", + "https://token@github.com/GameCult/Ghostlight", + ] { + let mut value = expected_incarnation(); + value.source_repository = private_source.into(); + assert!(value.validate().is_err()); + } + + let mut value = expected_incarnation(); + let stable_endpoint = value.route.as_ref().unwrap().stable_endpoint.clone(); + value.route.as_mut().unwrap().candidate_endpoint = stable_endpoint; + assert!(value.validate().is_err()); + } + + #[test] + fn expected_route_keeps_rudp_distinct_and_scheme_exact() { + let mut value = expected_incarnation(); + value.route = Some(IdunnExpectedRoute { + route_id: "odin-rendezvous".into(), + transport: "rudp".into(), + stable_endpoint: "rudp://10.77.0.1:17871".into(), + candidate_endpoint: "rudp://127.0.0.1:24171".into(), + }); + assert!(value.validate().is_ok()); + + value.route.as_mut().unwrap().candidate_endpoint = "tcp://127.0.0.1:24171".into(); + assert!(value.validate().is_err()); + + value.route.as_mut().unwrap().candidate_endpoint = "udp://127.0.0.1:24171".into(); + assert!(value.validate().is_err()); + } + + #[test] + fn topology_correlation_refuses_expected_substitution_and_dependency_omission() { + let expected = expected_incarnation(); + + let mut value = topology_correlation(&expected); + value.target = "epiphany".into(); + assert!(value.validate().is_ok()); + assert!(validate_against_current_lease(&value, &expected).is_err()); + + let mut value = topology_correlation(&expected); + value.runtime_id = "ghostlight-other-runtime".into(); + assert!(validate_against_current_lease(&value, &expected).is_err()); + + let mut value = topology_correlation(&expected); + value.expected_projection_sha256 = digest('c'); + assert!(validate_against_current_lease(&value, &expected).is_err()); + + let mut value = topology_correlation(&expected); + value.dependencies.pop(); + assert!(value.validate().is_ok()); + assert!(validate_against_current_lease(&value, &expected).is_err()); + + let mut value = topology_correlation(&expected); + value.dependencies[0].provider_id = Some("odin-substitute".into()); + assert!(value.validate().is_ok()); + assert!(validate_against_current_lease(&value, &expected).is_err()); + + let mut value = topology_correlation(&expected); + value.dependencies[0].kind = "required".into(); + assert!(value.validate().is_ok()); + assert!(validate_against_current_lease(&value, &expected).is_err()); + + let mut value = topology_correlation(&expected); + value.dependencies[0].observed_capacity = Some(0); + assert!(value.validate().is_err()); + + let mut value = topology_correlation(&expected); + value.dependencies[0].observed_capacity = Some(1); + let mut stricter = expected.clone(); + stricter.dependencies[0].minimum_capacity = 2; + value.expected_projection_sha256 = stricter.canonical_sha256().unwrap(); + assert!(validate_against_current_lease(&value, &stricter).is_err()); + } + + #[test] + fn topology_correlation_requires_explicit_coherent_partial_states() { + let expected = expected_incarnation(); + let mut value = topology_correlation(&expected); + value.signed_presence_sha256 = None; + value.observed_presence_state = None; + value.observed_presence_publisher_sequence = None; + value.observed_write_lease_sha256 = None; + value.observed_capabilities.clear(); + value.present = false; + value.ready = false; + assert!(value.validate().is_err()); + + value.disagreements.push(OdinTopologyDisagreement { + code: "presence-missing".into(), + expected: Some("signed-runtime-presence".into()), + observed: None, + }); + assert!(value.validate().is_ok()); + + let mut value = topology_correlation(&expected); + value.runtime_instance_id = None; + assert!(value.validate().is_err()); + + let mut value = topology_correlation(&expected); + value.current_activation_sha256 = None; + value.present = true; + value.ready = false; + value.disagreements.push(OdinTopologyDisagreement { + code: "activation-missing".into(), + expected: Some("current-activation".into()), + observed: None, + }); + assert!(value.validate().is_err()); + + let mut value = topology_correlation(&expected); + value.expected = false; + value.present = false; + value.ready = false; + assert!(value.validate().is_err()); + } + + #[test] + fn topology_correlation_refuses_unsorted_or_duplicate_claims() { + let expected = expected_incarnation(); + let mut value = topology_correlation(&expected); + value.dependencies.reverse(); + assert!(value.validate().is_err()); + + let mut value = topology_correlation(&expected); + value.dependencies.push(value.dependencies[1].clone()); + assert!(value.validate().is_err()); + + let mut value = topology_correlation(&expected); + value.ready = false; + value.disagreements = vec![ + OdinTopologyDisagreement { + code: "schema-mismatch".into(), + expected: Some("state.v2".into()), + observed: Some("state.v1".into()), + }, + OdinTopologyDisagreement { + code: "endpoint-mismatch".into(), + expected: Some("route-a".into()), + observed: Some("route-b".into()), + }, + ]; + assert!(value.validate().is_err()); + + value.disagreements[1].code = "schema-mismatch".into(); + assert!(value.validate().is_err()); + } + + #[test] + fn topology_correlation_preserves_warming_and_exact_write_lease_authority() { + let expected = expected_incarnation(); + let value = topology_correlation(&expected); + validate_against_current_lease(&value, &expected).unwrap(); + assert!( + value + .validate_against_expected(&expected, Some(&digest('d'))) + .is_err() + ); + assert!(value.validate_against_expected(&expected, None).is_err()); + + let mut digest_only_ready = topology_correlation(&expected); + digest_only_ready.observed_write_lease_sha256 = None; + digest_only_ready.validate().unwrap(); + assert!( + digest_only_ready + .validate_against_expected(&expected, Some(&digest('c'))) + .is_err() + ); + + let mut warming = topology_correlation(&expected); + warming.observed_presence_state = Some("warming".into()); + warming.ready = false; + assert!(warming.validate().is_err()); + warming.observed_write_lease_sha256 = None; + warming.validate().unwrap(); + warming.validate_against_expected(&expected, None).unwrap(); + warming.ready = true; + assert!(warming.validate().is_err()); + + let mut partial = topology_correlation(&expected); + partial.observed_presence_state = None; + assert!(partial.validate().is_err()); + let mut partial = topology_correlation(&expected); + partial.signed_presence_sha256 = None; + assert!(partial.validate().is_err()); + + let mut stateless = expected.clone(); + stateless.state_schema_generation = None; + stateless.state_contract_sha256 = None; + stateless.write_lease_required = false; + let mut stateless_receipt = topology_correlation(&stateless); + stateless_receipt.observed_write_lease_sha256 = None; + stateless_receipt + .validate_against_expected(&stateless, None) + .unwrap(); + stateless_receipt.observed_write_lease_sha256 = Some(digest('c')); + assert!( + stateless_receipt + .validate_against_expected(&stateless, Some(&digest('c'))) + .is_err() + ); + } + + #[test] + fn topology_correlation_cannot_manufacture_ready() { + let expected = expected_incarnation(); + let mut value = topology_correlation(&expected); + value.dependencies[0].ready = false; + assert!(value.validate().is_err()); + + let mut value = topology_correlation(&expected); + value.disagreements.push(OdinTopologyDisagreement { + code: "capability-mismatch".into(), + expected: Some("conversation.v1".into()), + observed: Some("conversation.v0".into()), + }); + assert!(value.validate().is_err()); + + let mut value = topology_correlation(&expected); + value.present = false; + value.disagreements.push(OdinTopologyDisagreement { + code: "identity-mismatch".into(), + expected: Some("expected-runtime".into()), + observed: Some("other-runtime".into()), + }); + assert!(value.validate().is_err()); + + let mut value = topology_correlation(&expected); + value.dependencies[0].provider_evidence_sha256 = None; + assert!(value.validate().is_err()); + + let mut missing_capability = topology_correlation(&expected); + missing_capability.observed_capabilities.pop(); + assert!(missing_capability.validate().is_ok()); + assert!( + missing_capability + .validate_against_expected(&expected, Some(&digest('c'))) + .is_err() + ); + missing_capability.present = true; + missing_capability.ready = false; + missing_capability.disagreements = vec![OdinTopologyDisagreement { + code: "generic-mismatch".into(), + expected: Some("expected".into()), + observed: Some("observed".into()), + }]; + assert!( + missing_capability + .validate_against_expected(&expected, Some(&digest('c'))) + .is_err() + ); + correlate_capabilities( + &mut missing_capability.disagreements, + &expected.capabilities, + &missing_capability.observed_capabilities, + ); + missing_capability + .disagreements + .sort_by(|left, right| left.code.cmp(&right.code)); + missing_capability + .validate_against_expected(&expected, Some(&digest('c'))) + .unwrap(); + + let mut additional_capability = topology_correlation(&expected); + additional_capability + .observed_capabilities + .push(runtime_capability("telemetry")); + additional_capability + .validate_against_expected(&expected, Some(&digest('c'))) + .unwrap(); + } + + #[test] + fn topology_signature_is_exact_and_purpose_bound() -> Result<()> { + let temp = tempfile::tempdir()?; + let signer = enroll_service_identity_at::( + &temp.path().join("odin-topology.cc"), + )?; + let anchor = signer.trust_anchor()?; + let expected = expected_incarnation(); + let mut receipt = topology_correlation(&expected); + receipt.signer_identity_id = anchor.identity_id.clone(); + receipt.signature.clear(); + let unsigned = receipt.unsigned_signature_payload()?; + let proof = signer.sign::(&unsigned); + receipt.signature = proof.signature.clone(); + + let encoded = receipt.canonical_bytes()?; + let (decoded, decoded_unsigned) = + OdinRuntimeTopologyCorrelationRecord::decode_canonical_signed_payload(&encoded)?; + verify_service_identity_signature::< + OdinTopologyIdentity, + OdinRuntimeTopologyCorrelationPurpose, + >( + &anchor, + &decoded_unsigned, + &ServiceIdentitySignature { + identity_id: decoded.signer_identity_id, + signature: decoded.signature, + }, + )?; + + let mut substituted = decoded_unsigned.clone(); + substituted.push(0); + assert!( + verify_service_identity_signature::< + OdinTopologyIdentity, + OdinRuntimeTopologyCorrelationPurpose, + >(&anchor, &substituted, &proof) + .is_err() + ); + + let wrong_purpose = signer.sign::(&decoded_unsigned); + assert!( + verify_service_identity_signature::< + OdinTopologyIdentity, + OdinRuntimeTopologyCorrelationPurpose, + >(&anchor, &decoded_unsigned, &wrong_purpose) + .is_err() + ); + Ok(()) + } + + #[test] + fn topology_trust_anchor_is_odin_profile_and_purpose_bound() -> Result<()> { + let anchor = topology_anchor(); + anchor.validate()?; + + let mut value = anchor.clone(); + value.service_id = "idunn".into(); + assert!(value.validate().is_err()); + + let mut value = anchor.clone(); + value.signing_purpose = GAMECULT_RUNTIME_PRESENCE_HEALTH_SIGNING_PURPOSE.into(); + assert!(value.validate().is_err()); + + let mut value = anchor.clone(); + value.signed_schema = GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA.into(); + assert!(value.validate().is_err()); + + let mut value = anchor; + value.signer_identity_id = + derive_service_identity_id::(&value.signer_public_key)?; + assert!(value.validate().is_err()); + Ok(()) + } +} diff --git a/packages/cultnet-rs/src/service_identity.rs b/packages/cultnet-rs/src/service_identity.rs new file mode 100644 index 00000000..6f3e58d8 --- /dev/null +++ b/packages/cultnet-rs/src/service_identity.rs @@ -0,0 +1,959 @@ +use anyhow::{Context, Result, anyhow, bail}; +use cultcache_rs::{CacheBackingStore, CultCacheEnvelope, SingleFileMessagePackBackingStore}; +use ed25519_dalek::{Signature, Signer, SigningKey, Verifier, VerifyingKey}; +use rand::RngCore; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use std::io::{Read, Write}; +use std::marker::PhantomData; +use std::path::Path; +#[cfg(windows)] +use zeroize::Zeroize; +use zeroize::Zeroizing; + +pub const WINDOWS_SERVICE_IDENTITY_ASSURANCE: &str = "os_user_installation_bound_best_effort"; +pub const LINUX_SERVICE_IDENTITY_ASSURANCE: &str = "os_installation_file_bound_cloneable_baseline"; + +/// Fixes every authority-bearing name at the call site. A service cannot open +/// another service's identity by supplying a convenient runtime string. +pub trait ServiceIdentityProfile: Send + Sync + 'static { + const PRIVATE_TYPE: &'static str; + const PRIVATE_SCHEMA: &'static str; + const PRIVATE_KEY: &'static str; + const TRUST_ANCHOR_TYPE: &'static str; + const TRUST_ANCHOR_SCHEMA: &'static str; + const TRUST_ANCHOR_KEY: &'static str; + const ID_DOMAIN: &'static [u8]; + const SIGNATURE_DOMAIN: &'static [u8]; + const PROTECTOR_CONTEXT: &'static str; +} + +/// A purpose is a type, not caller-provided prose. Protocol implementations +/// define one zero-sized type per signed statement kind. +pub trait ServiceSignaturePurpose: Send + Sync + 'static { + const PURPOSE: &'static [u8]; +} + +pub enum IdunnServiceIdentity {} + +/// Odin-owned identity dedicated to signed Expected/Present/Ready topology +/// correlation. It is not a provider-health key and cannot issue Idunn +/// admission or lifecycle authority. +pub enum OdinTopologyIdentity {} + +pub struct OdinRuntimeTopologyCorrelationPurpose; + +impl ServiceSignaturePurpose for OdinRuntimeTopologyCorrelationPurpose { + const PURPOSE: &'static [u8] = b"odin.runtime_topology_correlation.v2"; +} + +impl ServiceIdentityProfile for OdinTopologyIdentity { + const PRIVATE_TYPE: &'static str = "odin.topology_identity.private.v1"; + const PRIVATE_SCHEMA: &'static str = "odin.topology_identity.private.v1"; + const PRIVATE_KEY: &'static str = "odin-topology-identity"; + const TRUST_ANCHOR_TYPE: &'static str = "odin.topology_identity.trust_anchor.v1"; + const TRUST_ANCHOR_SCHEMA: &'static str = "odin.topology_identity.trust_anchor.v1"; + const TRUST_ANCHOR_KEY: &'static str = "odin-topology-identity-public"; + const ID_DOMAIN: &'static [u8] = b"odin.topology-identity.id.v1\0"; + const SIGNATURE_DOMAIN: &'static [u8] = b"odin.topology-identity.signature.v1\0"; + const PROTECTOR_CONTEXT: &'static str = "odin-topology-identity-v1"; +} + +/// Root-operated identity dedicated to deployment-brake release grants. The +/// Idunn daemon receives only this profile's public trust anchor; opening the +/// private store is a provisioning operation, never daemon startup work. +pub enum IdunnDeploymentBrakeOperatorIdentity {} + +pub struct IdunnDeploymentBrakeReleasePurpose; + +impl ServiceSignaturePurpose + for IdunnDeploymentBrakeReleasePurpose +{ + const PURPOSE: &'static [u8] = b"idunn.deployment-release.v1"; +} + +impl ServiceIdentityProfile for IdunnDeploymentBrakeOperatorIdentity { + const PRIVATE_TYPE: &'static str = "idunn.deployment_brake_operator.private.v1"; + const PRIVATE_SCHEMA: &'static str = "idunn.deployment_brake_operator.private.v1"; + const PRIVATE_KEY: &'static str = "idunn-deployment-brake-operator"; + const TRUST_ANCHOR_TYPE: &'static str = "idunn.deployment_brake_operator.trust_anchor.v1"; + const TRUST_ANCHOR_SCHEMA: &'static str = "idunn.deployment_brake_operator.trust_anchor.v1"; + const TRUST_ANCHOR_KEY: &'static str = "idunn-deployment-brake-operator-public"; + const ID_DOMAIN: &'static [u8] = b"idunn.deployment-brake-operator.id.v1\0"; + const SIGNATURE_DOMAIN: &'static [u8] = b"idunn.deployment-brake-operator.signature.v1\0"; + const PROTECTOR_CONTEXT: &'static str = "idunn-deployment-brake-operator-v1"; +} + +/// Dedicated provider-owned identity for generic daemon-health statements. +/// It is intentionally independent of any one provider runtime or repository. +pub enum GameCultProviderHealthIdentity {} + +/// The only generic health statement purpose accepted from provider-health +/// identities by Idunn admission. +pub struct IdunnSignedDaemonHealthPurpose; + +impl ServiceSignaturePurpose for IdunnSignedDaemonHealthPurpose { + const PURPOSE: &'static [u8] = b"idunn.signed_daemon_health.v1"; +} + +/// Provider-owned runtime presence and health. This is deliberately distinct +/// from the legacy daemon-health purpose so an old signature cannot acquire +/// deployment-incarnation authority by substitution. +pub struct GameCultRuntimePresenceHealthPurpose; + +impl ServiceSignaturePurpose + for GameCultRuntimePresenceHealthPurpose +{ + const PURPOSE: &'static [u8] = b"gamecult.runtime_presence_health.v2"; +} + +impl ServiceIdentityProfile for GameCultProviderHealthIdentity { + const PRIVATE_TYPE: &'static str = "gamecult.provider_health_identity.private.v1"; + const PRIVATE_SCHEMA: &'static str = "gamecult.provider_health_identity.private.v1"; + const PRIVATE_KEY: &'static str = "gamecult-provider-health-identity"; + const TRUST_ANCHOR_TYPE: &'static str = "gamecult.provider_health_identity.trust_anchor.v1"; + const TRUST_ANCHOR_SCHEMA: &'static str = "gamecult.provider_health_identity.trust_anchor.v1"; + const TRUST_ANCHOR_KEY: &'static str = "gamecult-provider-health-identity-public"; + const ID_DOMAIN: &'static [u8] = b"gamecult.provider-health.identity.v1\0"; + const SIGNATURE_DOMAIN: &'static [u8] = b"gamecult.provider-health.signature.v1\0"; + const PROTECTOR_CONTEXT: &'static str = "gamecult-provider-health-identity-v1"; +} + +/// The only signing purpose accepted for Idunn's public projection of a +/// provider-authenticated health admission. Keeping this profile here makes +/// purpose selection a compile-time protocol choice rather than caller text. +pub struct IdunnAuthenticatedProviderHealthProjectionPurpose; + +impl ServiceSignaturePurpose + for IdunnAuthenticatedProviderHealthProjectionPurpose +{ + const PURPOSE: &'static [u8] = b"idunn.authenticated-provider-health-projection.v1"; +} + +/// Idunn's stable identity signs every activation before its ephemeral key is +/// handed to a workload. This makes the launch witness portable without +/// allowing a configuration file or provider to mint its own activation. +pub struct IdunnRuntimeActivationPurpose; + +impl ServiceSignaturePurpose for IdunnRuntimeActivationPurpose { + const PURPOSE: &'static [u8] = b"idunn.runtime_activation.v2"; +} + +impl ServiceIdentityProfile for IdunnServiceIdentity { + const PRIVATE_TYPE: &'static str = "idunn.service_identity.private.v1"; + const PRIVATE_SCHEMA: &'static str = "idunn.service_identity.private.v1"; + const PRIVATE_KEY: &'static str = "idunn-service-identity"; + const TRUST_ANCHOR_TYPE: &'static str = "idunn.service_identity.trust_anchor.v1"; + const TRUST_ANCHOR_SCHEMA: &'static str = "idunn.service_identity.trust_anchor.v1"; + const TRUST_ANCHOR_KEY: &'static str = "idunn-service-identity-public"; + const ID_DOMAIN: &'static [u8] = b"idunn.service-identity.id.v1\0"; + const SIGNATURE_DOMAIN: &'static [u8] = b"idunn.service-identity.signature.v1\0"; + const PROTECTOR_CONTEXT: &'static str = "idunn-service-identity-v1"; +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct ServiceIdentityPrivateEntry { + pub schema_version: String, + pub identity_id: String, + pub public_key: Vec, + pub protected_private_seed: Vec, + pub protector_kind: String, + pub protector_binding: String, + pub protector_version: String, + pub assurance: String, + pub created_at: String, + pub enrollment_nonce: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct ServiceIdentityTrustAnchor { + pub schema_version: String, + pub identity_id: String, + pub public_key: Vec, + pub assurance: String, + pub identity_created_at: String, + pub source_identity_record_sha256: String, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ServiceIdentitySignature { + pub identity_id: String, + pub signature: Vec, +} + +/// The only value holding unprotected key material. It exposes signing, never +/// seed export or access to the underlying signing key. +pub struct ServiceIdentitySigner { + entry: ServiceIdentityPrivateEntry, + signing_key: SigningKey, + profile: PhantomData

, +} + +impl ServiceIdentitySigner

{ + pub fn entry(&self) -> &ServiceIdentityPrivateEntry { + &self.entry + } + + pub fn trust_anchor(&self) -> Result { + Ok(ServiceIdentityTrustAnchor { + schema_version: P::TRUST_ANCHOR_SCHEMA.into(), + identity_id: self.entry.identity_id.clone(), + public_key: self.entry.public_key.clone(), + assurance: self.entry.assurance.clone(), + identity_created_at: self.entry.created_at.clone(), + source_identity_record_sha256: format!( + "sha256-{}", + hex(&Sha256::digest(rmp_serde::to_vec(&self.entry)?)) + ), + }) + } + + pub fn sign>(&self, payload: &[u8]) -> ServiceIdentitySignature { + ServiceIdentitySignature { + identity_id: self.entry.identity_id.clone(), + signature: self + .signing_key + .sign(&signing_message::(payload)) + .to_bytes() + .to_vec(), + } + } +} + +pub fn enroll_service_identity_at( + path: &Path, +) -> Result> { + if path.exists() { + bail!( + "service identity store {} already exists; enrollment is immutable", + path.display() + ); + } + prepare_private_parent(path)?; + let mut seed = Zeroizing::new([0u8; 32]); + rand::rng().fill_bytes(seed.as_mut()); + let signing_key = SigningKey::from_bytes(&*seed); + let mut nonce = [0u8; 32]; + rand::rng().fill_bytes(&mut nonce); + let public_key = signing_key.verifying_key().to_bytes(); + let binding = platform_binding::

()?; + let entry = ServiceIdentityPrivateEntry { + schema_version: P::PRIVATE_SCHEMA.into(), + identity_id: identity_id::

(&public_key), + public_key: public_key.to_vec(), + protected_private_seed: protect_seed::

(&*seed, &binding)?, + protector_kind: platform_protector_kind().into(), + protector_binding: binding, + protector_version: "v1".into(), + assurance: platform_assurance().into(), + created_at: chrono::Utc::now().to_rfc3339(), + enrollment_nonce: nonce.to_vec(), + }; + validate_private::

(&entry)?; + let envelope = private_envelope::

(&entry)?; + atomic_create_private_store(path, &envelope)?; + harden_store_permissions(path)?; + open_service_identity_at::

(path) +} + +pub fn open_service_identity_at( + path: &Path, +) -> Result> { + if !path.is_file() { + bail!("service identity store {} does not exist", path.display()); + } + let entries = SingleFileMessagePackBackingStore::new(path).pull_all()?; + open_service_identity_envelopes::

(&entries) +} + +/// Opens an immutable service-identity credential from a read-only stream. +/// +/// Unlike [`open_service_identity_at`], this path has no sibling-lock or +/// writable-filesystem requirement. It is intended for immutable credential +/// copies such as systemd `LoadCredential` files. +pub fn open_service_identity_credential_reader( + mut reader: impl Read, +) -> Result> { + let mut bytes = Zeroizing::new(Vec::new()); + reader + .read_to_end(&mut bytes) + .context("failed to read immutable service identity credential")?; + open_service_identity_credential_bytes::

(&bytes) +} + +/// Opens an immutable service-identity credential from its exact bytes. +pub fn open_service_identity_credential_bytes( + bytes: &[u8], +) -> Result> { + let mut decoder = rmp_serde::Deserializer::new(std::io::Cursor::new(bytes)); + let entries = Vec::::deserialize(&mut decoder) + .context("immutable service identity credential is malformed MessagePack")?; + if decoder.position() != bytes.len() as u64 { + bail!("immutable service identity credential contains trailing data"); + } + open_service_identity_envelopes::

(&entries) +} + +fn open_service_identity_envelopes( + entries: &[CultCacheEnvelope], +) -> Result> { + let [envelope] = entries else { + bail!("service identity store must contain exactly one immutable envelope"); + }; + if envelope.r#type != P::PRIVATE_TYPE + || envelope.key != P::PRIVATE_KEY + || envelope.schema_id.as_deref() != Some(P::PRIVATE_SCHEMA) + { + bail!("service identity store belongs to a different profile or schema"); + } + let entry: ServiceIdentityPrivateEntry = rmp_serde::from_slice(&envelope.payload) + .context("service identity private payload is malformed MessagePack")?; + validate_private::

(&entry)?; + if entry.protector_kind != platform_protector_kind() || entry.assurance != platform_assurance() + { + bail!("service identity protector does not belong to this platform implementation"); + } + let binding = platform_binding::

()?; + if entry.protector_binding != binding { + bail!("service identity protector binding does not match this OS installation or profile"); + } + let seed = unprotect_seed::

(&entry.protected_private_seed, &binding)?; + let seed: &[u8; 32] = seed + .as_slice() + .try_into() + .map_err(|_| anyhow!("unprotected service identity seed has invalid length"))?; + let signing_key = SigningKey::from_bytes(seed); + if signing_key.verifying_key().to_bytes().as_slice() != entry.public_key.as_slice() { + bail!("service identity private seed does not match enrolled public key"); + } + Ok(ServiceIdentitySigner { + entry, + signing_key, + profile: PhantomData, + }) +} + +pub fn export_service_identity_trust_anchor( + signer: &ServiceIdentitySigner

, + path: &Path, +) -> Result { + let anchor = signer.trust_anchor()?; + ensure_parent(path)?; + let envelope = CultCacheEnvelope { + key: P::TRUST_ANCHOR_KEY.into(), + r#type: P::TRUST_ANCHOR_TYPE.into(), + payload: rmp_serde::to_vec(&anchor)?, + stored_at: anchor.identity_created_at.clone(), + schema_id: Some(P::TRUST_ANCHOR_SCHEMA.into()), + }; + let mut backing = SingleFileMessagePackBackingStore::new(path); + match backing.pull_all()?.as_slice() { + [] => backing.push(&envelope)?, + [current] if current == &envelope => {} + _ => bail!("service identity trust anchor output already contains different state"), + } + Ok(anchor) +} + +pub fn verify_service_identity_signature( + anchor: &ServiceIdentityTrustAnchor, + payload: &[u8], + proof: &ServiceIdentitySignature, +) -> Result<()> +where + P: ServiceIdentityProfile, + S: ServiceSignaturePurpose

, +{ + validate_anchor::

(anchor)?; + if proof.identity_id != anchor.identity_id { + bail!("service identity signature names a different identity"); + } + let key: [u8; 32] = anchor + .public_key + .as_slice() + .try_into() + .map_err(|_| anyhow!("service identity public key has invalid length"))?; + let signature = Signature::from_slice(&proof.signature) + .map_err(|_| anyhow!("service identity signature has invalid length"))?; + VerifyingKey::from_bytes(&key)? + .verify(&signing_message::(payload), &signature) + .map_err(|_| anyhow!("service identity signature verification failed")) +} + +/// Verify against a public key already admitted by an owning trust store. +/// The profile derives the identity, so callers cannot substitute a convenient +/// identity string while retaining the same signature bytes. +pub fn verify_service_identity_signature_with_public_key( + public_key: &[u8], + payload: &[u8], + proof: &ServiceIdentitySignature, +) -> Result<()> +where + P: ServiceIdentityProfile, + S: ServiceSignaturePurpose

, +{ + if proof.identity_id != derive_service_identity_id::

(public_key)? { + bail!("service identity signature names a different identity"); + } + let key: [u8; 32] = public_key + .try_into() + .map_err(|_| anyhow!("service identity public key has invalid length"))?; + let signature = Signature::from_slice(&proof.signature) + .map_err(|_| anyhow!("service identity signature has invalid length"))?; + VerifyingKey::from_bytes(&key)? + .verify(&signing_message::(payload), &signature) + .map_err(|_| anyhow!("service identity signature verification failed")) +} + +fn validate_private(entry: &ServiceIdentityPrivateEntry) -> Result<()> { + if entry.schema_version != P::PRIVATE_SCHEMA + || entry.public_key.len() != 32 + || entry.enrollment_nonce.len() != 32 + || entry.protected_private_seed.is_empty() + || entry.protector_version != "v1" + { + bail!("service identity private entry violates its profile schema"); + } + chrono::DateTime::parse_from_rfc3339(&entry.created_at) + .map_err(|_| anyhow!("service identity created_at is not RFC3339"))?; + if identity_id::

(&entry.public_key) != entry.identity_id { + bail!("service identity id does not match public key"); + } + Ok(()) +} + +fn validate_anchor(anchor: &ServiceIdentityTrustAnchor) -> Result<()> { + if anchor.schema_version != P::TRUST_ANCHOR_SCHEMA + || anchor.public_key.len() != 32 + || derive_service_identity_id::

(&anchor.public_key)? != anchor.identity_id + { + bail!("service identity trust anchor violates its profile schema"); + } + Ok(()) +} + +/// Derives the profile-bound identity id from an exact Ed25519 public key. +/// Consumers use this instead of accepting a caller-supplied identity string. +pub fn derive_service_identity_id(public_key: &[u8]) -> Result { + if public_key.len() != 32 { + bail!("service identity public key has invalid length"); + } + Ok(hex(&Sha256::digest([P::ID_DOMAIN, public_key].concat()))) +} + +fn identity_id(key: &[u8]) -> String { + derive_service_identity_id::

(key).expect("internal service identity key is Ed25519") +} + +fn signing_message>( + payload: &[u8], +) -> Vec { + let mut out = + Vec::with_capacity(P::SIGNATURE_DOMAIN.len() + S::PURPOSE.len() + payload.len() + 16); + out.extend_from_slice(P::SIGNATURE_DOMAIN); + out.extend_from_slice(&(S::PURPOSE.len() as u64).to_be_bytes()); + out.extend_from_slice(S::PURPOSE); + out.extend_from_slice(&(payload.len() as u64).to_be_bytes()); + out.extend_from_slice(payload); + out +} + +fn private_envelope( + entry: &ServiceIdentityPrivateEntry, +) -> Result { + Ok(CultCacheEnvelope { + key: P::PRIVATE_KEY.into(), + r#type: P::PRIVATE_TYPE.into(), + payload: rmp_serde::to_vec(entry)?, + stored_at: entry.created_at.clone(), + schema_id: Some(P::PRIVATE_SCHEMA.into()), + }) +} + +fn ensure_parent(path: &Path) -> Result<()> { + let parent = path + .parent() + .ok_or_else(|| anyhow!("service identity path has no parent"))?; + std::fs::create_dir_all(parent)?; + Ok(()) +} + +/// Like [`ensure_parent`], but also locks the directory down to owner-only +/// access. Reserved for paths that hold private key material: a public trust +/// anchor is routinely exported into a directory another service already +/// owns and reads from, and forcing that directory to `0700` would lock the +/// legitimate reader out rather than protect anything. +fn prepare_private_parent(path: &Path) -> Result<()> { + ensure_parent(path)?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let parent = path + .parent() + .ok_or_else(|| anyhow!("service identity path has no parent"))?; + std::fs::set_permissions(parent, std::fs::Permissions::from_mode(0o700))?; + } + Ok(()) +} + +/// Enrollment is a one-shot authority transition. `create_new` makes two +/// simultaneous enrollments unable to overwrite one another, while the encoded +/// envelope vector remains a supported CultCache single-file representation. +fn atomic_create_private_store(path: &Path, envelope: &CultCacheEnvelope) -> Result<()> { + let bytes = rmp_serde::to_vec(&vec![envelope.clone()]) + .context("failed to encode immutable service identity store")?; + let mut file = std::fs::OpenOptions::new() + .create_new(true) + .write(true) + .open(path) + .with_context(|| { + format!( + "service identity store {} already exists or cannot be created; enrollment is immutable", + path.display() + ) + })?; + if let Err(error) = file.write_all(&bytes).and_then(|()| file.sync_all()) { + drop(file); + let _ = std::fs::remove_file(path); + return Err(error).context("failed to persist immutable service identity store"); + } + Ok(()) +} +fn harden_store_permissions(_path: &Path) -> Result<()> { + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(_path, std::fs::Permissions::from_mode(0o600))?; + } + Ok(()) +} + +#[cfg(windows)] +fn platform_protector_kind() -> &'static str { + "windows_dpapi_current_user" +} +#[cfg(target_os = "linux")] +fn platform_protector_kind() -> &'static str { + "linux_file_mode_machine_id_binding" +} +#[cfg(windows)] +fn platform_assurance() -> &'static str { + WINDOWS_SERVICE_IDENTITY_ASSURANCE +} +#[cfg(target_os = "linux")] +fn platform_assurance() -> &'static str { + LINUX_SERVICE_IDENTITY_ASSURANCE +} + +#[cfg(windows)] +fn platform_binding() -> Result { + Ok(format!("dpapi-current-user:{}", P::PROTECTOR_CONTEXT)) +} +#[cfg(target_os = "linux")] +fn platform_binding() -> Result { + let raw = std::fs::read_to_string("/etc/machine-id") + .or_else(|_| std::fs::read_to_string("/var/lib/dbus/machine-id")) + .context("Linux machine-id is unavailable")?; + let id = raw.trim(); + if id.is_empty() { + bail!("Linux machine-id is empty"); + } + Ok(format!( + "{}:machine-id-sha256:{}", + P::PROTECTOR_CONTEXT, + hex(&Sha256::digest(id.as_bytes())) + )) +} + +#[cfg(windows)] +fn protect_seed(seed: &[u8; 32], binding: &str) -> Result> { + Ok(dpapi::

(seed, binding, true)?.to_vec()) +} +#[cfg(windows)] +fn unprotect_seed( + seed: &[u8], + binding: &str, +) -> Result>> { + dpapi::

(seed, binding, false) +} +#[cfg(windows)] +fn dpapi( + input: &[u8], + binding: &str, + protect: bool, +) -> Result>> { + use windows_sys::Win32::Foundation::LocalFree; + use windows_sys::Win32::Security::Cryptography::{ + CRYPT_INTEGER_BLOB, CRYPTPROTECT_UI_FORBIDDEN, CryptProtectData, CryptUnprotectData, + }; + let mut input = Zeroizing::new(input.to_vec()); + let mut entropy = Zeroizing::new( + Sha256::digest( + [ + b"gamecult-service-identity-dpapi-v1\0".as_slice(), + P::PROTECTOR_CONTEXT.as_bytes(), + binding.as_bytes(), + ] + .concat(), + ) + .to_vec(), + ); + let ib = CRYPT_INTEGER_BLOB { + cbData: input.len() as u32, + pbData: input.as_mut_ptr(), + }; + let eb = CRYPT_INTEGER_BLOB { + cbData: entropy.len() as u32, + pbData: entropy.as_mut_ptr(), + }; + let mut output = CRYPT_INTEGER_BLOB { + cbData: 0, + pbData: std::ptr::null_mut(), + }; + let ok = unsafe { + if protect { + CryptProtectData( + &ib, + std::ptr::null(), + &eb, + std::ptr::null(), + std::ptr::null(), + CRYPTPROTECT_UI_FORBIDDEN, + &mut output, + ) + } else { + CryptUnprotectData( + &ib, + std::ptr::null_mut(), + &eb, + std::ptr::null(), + std::ptr::null(), + CRYPTPROTECT_UI_FORBIDDEN, + &mut output, + ) + } + }; + if ok == 0 { + return Err(std::io::Error::last_os_error()) + .context("DPAPI service identity operation failed"); + } + if output.pbData.is_null() { + bail!("DPAPI service identity operation returned a null output"); + } + let output = unsafe { std::slice::from_raw_parts_mut(output.pbData, output.cbData as usize) }; + let bytes = Zeroizing::new(output.to_vec()); + output.zeroize(); + unsafe { + LocalFree(output.as_mut_ptr().cast()); + } + Ok(bytes) +} + +#[cfg(target_os = "linux")] +fn protect_seed(seed: &[u8; 32], binding: &str) -> Result> { + xor_linux::

(seed, binding) +} +#[cfg(target_os = "linux")] +fn unprotect_seed( + seed: &[u8], + binding: &str, +) -> Result>> { + if seed.len() != 32 { + bail!("protected Linux service seed has invalid length"); + } + let mask = Sha256::digest( + [ + b"gamecult-linux-service-seed-v1\0".as_slice(), + P::PROTECTOR_CONTEXT.as_bytes(), + binding.as_bytes(), + ] + .concat(), + ); + let mut clear_seed = Zeroizing::new(Vec::with_capacity(seed.len())); + clear_seed.extend(seed.iter().zip(mask).map(|(a, b)| a ^ b)); + Ok(clear_seed) +} +#[cfg(target_os = "linux")] +fn xor_linux(seed: &[u8], binding: &str) -> Result> { + let mask = Sha256::digest( + [ + b"gamecult-linux-service-seed-v1\0".as_slice(), + P::PROTECTOR_CONTEXT.as_bytes(), + binding.as_bytes(), + ] + .concat(), + ); + Ok(seed.iter().zip(mask).map(|(a, b)| a ^ b).collect()) +} + +fn hex(bytes: &[u8]) -> String { + const D: &[u8; 16] = b"0123456789abcdef"; + let mut s = String::with_capacity(bytes.len() * 2); + for b in bytes { + s.push(D[(b >> 4) as usize] as char); + s.push(D[(b & 15) as usize] as char); + } + s +} + +#[cfg(test)] +mod tests { + use super::*; + enum OtherPurpose {} + impl ServiceSignaturePurpose for OtherPurpose { + const PURPOSE: &'static [u8] = b"idunn.other.v1"; + } + + #[test] + fn immutable_enrollment_reopens_and_is_purpose_and_payload_bound() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = temp.path().join("idunn.ccmp"); + let signer = enroll_service_identity_at::(&path)?; + let proof = signer.sign::(b"healthy"); + let anchor = signer.trust_anchor()?; + verify_service_identity_signature::< + IdunnServiceIdentity, + IdunnAuthenticatedProviderHealthProjectionPurpose, + >(&anchor, b"healthy", &proof)?; + assert!( + verify_service_identity_signature::< + IdunnServiceIdentity, + IdunnAuthenticatedProviderHealthProjectionPurpose, + >(&anchor, b"unhealthy", &proof) + .is_err() + ); + assert!( + verify_service_identity_signature::( + &anchor, b"healthy", &proof + ) + .is_err() + ); + assert!(enroll_service_identity_at::(&path).is_err()); + assert_eq!( + derive_service_identity_id::(&anchor.public_key)?, + anchor.identity_id + ); + assert!(derive_service_identity_id::(&[0; 31]).is_err()); + assert_eq!( + open_service_identity_at::(&path)?.entry(), + signer.entry() + ); + Ok(()) + } + + #[test] + fn immutable_credential_reader_opens_without_a_path_and_retains_signing_authority() -> Result<()> + { + let temp = tempfile::tempdir()?; + let path = temp.path().join("provider-health.cc"); + let enrolled = enroll_service_identity_at::(&path)?; + let bytes = std::fs::read(path)?; + + let from_reader = open_service_identity_credential_reader::( + std::io::Cursor::new(bytes.clone()), + )?; + let from_bytes = + open_service_identity_credential_bytes::(&bytes)?; + assert_eq!(from_reader.entry(), enrolled.entry()); + assert_eq!(from_bytes.entry(), enrolled.entry()); + + let payload = b"runtime-presence"; + let proof = from_reader.sign::(payload); + verify_service_identity_signature::< + GameCultProviderHealthIdentity, + GameCultRuntimePresenceHealthPurpose, + >(&enrolled.trust_anchor()?, payload, &proof)?; + Ok(()) + } + + #[test] + fn immutable_credential_reader_rejects_ambiguous_profile_or_machine_state() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = temp.path().join("idunn.ccmp"); + enroll_service_identity_at::(&path)?; + let bytes = std::fs::read(path)?; + + assert!( + open_service_identity_credential_bytes::(&bytes) + .is_err() + ); + + let mut envelopes: Vec = rmp_serde::from_slice(&bytes)?; + let mut duplicated = envelopes.clone(); + duplicated.push(envelopes[0].clone()); + assert!( + open_service_identity_credential_bytes::(&rmp_serde::to_vec( + &duplicated + )?) + .is_err() + ); + + let mut trailing = bytes.clone(); + trailing.push(0xc0); + assert!(open_service_identity_credential_bytes::(&trailing).is_err()); + + let mut wrong_seed_envelopes = envelopes.clone(); + let mut wrong_seed: ServiceIdentityPrivateEntry = + rmp_serde::from_slice(&wrong_seed_envelopes[0].payload)?; + wrong_seed.protected_private_seed[0] ^= 1; + wrong_seed_envelopes[0].payload = rmp_serde::to_vec(&wrong_seed)?; + assert!( + open_service_identity_credential_bytes::(&rmp_serde::to_vec( + &wrong_seed_envelopes + )?) + .is_err() + ); + + let mut entry: ServiceIdentityPrivateEntry = rmp_serde::from_slice(&envelopes[0].payload)?; + entry.protector_binding.push_str(":another-machine"); + envelopes[0].payload = rmp_serde::to_vec(&entry)?; + assert!( + open_service_identity_credential_bytes::(&rmp_serde::to_vec( + &envelopes + )?) + .is_err() + ); + Ok(()) + } + + #[test] + fn malformed_or_profile_substituted_state_fails_closed_without_replacement() -> Result<()> { + enum Alien {} + impl ServiceIdentityProfile for Alien { + const PRIVATE_TYPE: &'static str = "alien.private.v1"; + const PRIVATE_SCHEMA: &'static str = "alien.private.v1"; + const PRIVATE_KEY: &'static str = "alien"; + const TRUST_ANCHOR_TYPE: &'static str = "alien.anchor.v1"; + const TRUST_ANCHOR_SCHEMA: &'static str = "alien.anchor.v1"; + const TRUST_ANCHOR_KEY: &'static str = "alien-public"; + const ID_DOMAIN: &'static [u8] = b"alien.id\0"; + const SIGNATURE_DOMAIN: &'static [u8] = b"alien.sig\0"; + const PROTECTOR_CONTEXT: &'static str = "alien-v1"; + } + let temp = tempfile::tempdir()?; + let malformed = temp.path().join("bad.ccmp"); + std::fs::write(&malformed, b"rot")?; + let before = std::fs::read(&malformed)?; + assert!(open_service_identity_at::(&malformed).is_err()); + assert!(enroll_service_identity_at::(&malformed).is_err()); + assert_eq!(before, std::fs::read(&malformed)?); + let good = temp.path().join("good.ccmp"); + enroll_service_identity_at::(&good)?; + assert!(open_service_identity_at::(&good).is_err()); + Ok(()) + } + + #[test] + fn exporting_a_public_trust_anchor_does_not_narrow_its_parent_permissions() -> Result<()> { + let temp = tempfile::tempdir()?; + let signer = + enroll_service_identity_at::(&temp.path().join("private.ccmp"))?; + let public_parent = temp.path().join("public"); + std::fs::create_dir(&public_parent)?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(&public_parent, std::fs::Permissions::from_mode(0o750))?; + } + let output = public_parent.join("public.ccmp"); + export_service_identity_trust_anchor(&signer, &output)?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + assert_eq!( + std::fs::metadata(&public_parent)?.permissions().mode() & 0o777, + 0o750, + "exporting a public trust anchor must not reassign its parent directory's permissions" + ); + } + Ok(()) + } + + #[test] + fn exported_anchor_is_public_only_immutable_and_rejects_identity_substitution() -> Result<()> { + let temp = tempfile::tempdir()?; + let signer = + enroll_service_identity_at::(&temp.path().join("private.ccmp"))?; + let output = temp.path().join("public.ccmp"); + let anchor = export_service_identity_trust_anchor(&signer, &output)?; + let bytes = std::fs::read(&output)?; + assert!( + !bytes + .windows(signer.entry().protected_private_seed.len()) + .any(|w| w == signer.entry().protected_private_seed) + ); + export_service_identity_trust_anchor(&signer, &output)?; + assert_eq!(bytes, std::fs::read(&output)?); + let other = + enroll_service_identity_at::(&temp.path().join("other.ccmp"))?; + let proof = other.sign::(b"healthy"); + assert!( + verify_service_identity_signature::< + IdunnServiceIdentity, + IdunnAuthenticatedProviderHealthProjectionPurpose, + >(&anchor, b"healthy", &proof) + .is_err() + ); + Ok(()) + } + + #[test] + fn provider_health_profile_is_domain_separated_and_purpose_bound() -> Result<()> { + let temp = tempfile::tempdir()?; + let signer = enroll_service_identity_at::( + &temp.path().join("provider-health.cc"), + )?; + let proof = signer.sign::(b"signed-health-record"); + let anchor = signer.trust_anchor()?; + verify_service_identity_signature::< + GameCultProviderHealthIdentity, + IdunnSignedDaemonHealthPurpose, + >(&anchor, b"signed-health-record", &proof)?; + verify_service_identity_signature_with_public_key::< + GameCultProviderHealthIdentity, + IdunnSignedDaemonHealthPurpose, + >(&anchor.public_key, b"signed-health-record", &proof)?; + assert_eq!( + anchor.schema_version, + "gamecult.provider_health_identity.trust_anchor.v1" + ); + assert_ne!( + derive_service_identity_id::(&anchor.public_key)?, + derive_service_identity_id::(&anchor.public_key)? + ); + Ok(()) + } + + #[test] + fn runtime_presence_and_legacy_health_purposes_cannot_be_substituted() -> Result<()> { + let temp = tempfile::tempdir()?; + let signer = enroll_service_identity_at::( + &temp.path().join("provider-runtime-presence.cc"), + )?; + let anchor = signer.trust_anchor()?; + let payload = b"runtime-presence"; + + let presence = signer.sign::(payload); + verify_service_identity_signature::< + GameCultProviderHealthIdentity, + GameCultRuntimePresenceHealthPurpose, + >(&anchor, payload, &presence)?; + assert!( + verify_service_identity_signature::< + GameCultProviderHealthIdentity, + IdunnSignedDaemonHealthPurpose, + >(&anchor, payload, &presence) + .is_err() + ); + + let legacy = signer.sign::(payload); + assert!( + verify_service_identity_signature::< + GameCultProviderHealthIdentity, + GameCultRuntimePresenceHealthPurpose, + >(&anchor, payload, &legacy) + .is_err() + ); + Ok(()) + } +} diff --git a/packages/cultnet-rs/src/snapshot_query.rs b/packages/cultnet-rs/src/snapshot_query.rs new file mode 100644 index 00000000..f6150d1d --- /dev/null +++ b/packages/cultnet-rs/src/snapshot_query.rs @@ -0,0 +1,339 @@ +use anyhow::{Result, anyhow}; +use std::collections::{BTreeMap, BTreeSet}; + +use crate::{CultNetDocumentRegistry, CultNetMessage, CultNetRawDocumentRecord}; + +/// Read-only backing surface for a CultNet snapshot server. +/// +/// Records returned here are already the stored wire records. The snapshot +/// server selects and clones them; it never decodes payloads, rewrites source +/// metadata, or applies them to another cache. +pub trait CultNetRawSnapshotSource { + fn raw_snapshot(&self) -> Result>; +} + +impl CultNetRawSnapshotSource for Vec { + fn raw_snapshot(&self) -> Result> { + Ok(self.clone()) + } +} + +impl CultNetRawSnapshotSource for F +where + F: Fn() -> Result>, +{ + fn raw_snapshot(&self) -> Result> { + self() + } +} + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct CultNetReadOnlySnapshotPolicy { + allowed_records: BTreeSet<(String, String)>, +} + +impl CultNetReadOnlySnapshotPolicy { + pub fn new() -> Self { + Self::default() + } + + pub fn allow( + &mut self, + schema_id: impl Into, + record_key: impl Into, + ) -> Result<&mut Self> { + let schema_id = schema_id.into(); + let record_key = record_key.into(); + require_non_empty(&schema_id, "schema_id")?; + require_non_empty(&record_key, "record_key")?; + self.allowed_records.insert((schema_id, record_key)); + Ok(self) + } + + fn allows(&self, schema_id: &str, record_key: &str) -> bool { + self.allowed_records + .contains(&(schema_id.to_string(), record_key.to_string())) + } +} + +/// Serve one raw snapshot request from an explicitly bounded, read-only view. +/// +/// The document registry proves that every exposed schema belongs to the +/// runtime. The policy grants individual `(schema_id, record_key)` pairs. The +/// source owns the bytes and metadata; this function only selects records. +pub fn serve_read_only_raw_snapshot( + document_registry: &CultNetDocumentRegistry, + policy: &CultNetReadOnlySnapshotPolicy, + source: &S, + request: &CultNetMessage, +) -> Result { + let CultNetMessage::SnapshotRequest { + message_id, + schema_ids, + record_keys, + } = request + else { + return Err(anyhow!("expected cultnet.snapshot_request.v0")); + }; + require_non_empty(message_id, "message_id")?; + reject_duplicates(schema_ids.as_deref(), "requested schema id")?; + reject_duplicates(record_keys.as_deref(), "requested record key")?; + + let requested_schemas = schema_ids + .as_ref() + .map(|values| values.iter().map(String::as_str).collect::>()); + let requested_keys = record_keys + .as_ref() + .map(|values| values.iter().map(String::as_str).collect::>()); + let mut selected = Vec::new(); + let mut seen = BTreeSet::new(); + + for document in source.raw_snapshot()? { + let identity = (document.schema_id.clone(), document.record_key.clone()); + if !seen.insert(identity.clone()) { + return Err(anyhow!( + "snapshot source contains duplicate record {:?}/{:?}", + identity.0, + identity.1 + )); + } + if document_registry + .binding_by_schema_id(&document.schema_id) + .is_none() + { + return Err(anyhow!( + "snapshot source record {:?}/{:?} uses an unregistered schema", + document.schema_id, + document.record_key + )); + } + if !policy.allows(&document.schema_id, &document.record_key) { + continue; + } + if requested_schemas + .as_ref() + .is_some_and(|schemas| !schemas.contains(document.schema_id.as_str())) + { + continue; + } + if requested_keys + .as_ref() + .is_some_and(|keys| !keys.contains(document.record_key.as_str())) + { + continue; + } + selected.push(document); + } + + Ok(CultNetMessage::SnapshotResponseRaw { + message_id: message_id.clone(), + documents: selected, + }) +} + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct CultNetSnapshotSourceExpectation { + pub runtime_id: Option, + pub agent_id: Option, + pub role: Option, + pub tags: Option>, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CultNetSnapshotRecordExpectation { + pub schema_id: String, + pub record_key: String, + pub source: CultNetSnapshotSourceExpectation, +} + +impl CultNetSnapshotRecordExpectation { + pub fn new(schema_id: impl Into, record_key: impl Into) -> Self { + Self { + schema_id: schema_id.into(), + record_key: record_key.into(), + source: CultNetSnapshotSourceExpectation::default(), + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CultNetRawSnapshotQuery { + message_id: String, + expectations: BTreeMap<(String, String), CultNetSnapshotSourceExpectation>, +} + +impl CultNetRawSnapshotQuery { + pub fn new( + message_id: impl Into, + records: Vec, + ) -> Result { + let message_id = message_id.into(); + require_non_empty(&message_id, "message_id")?; + if records.is_empty() { + return Err(anyhow!("snapshot query must request at least one record")); + } + let mut expectations = BTreeMap::new(); + for record in records { + require_non_empty(&record.schema_id, "schema_id")?; + require_non_empty(&record.record_key, "record_key")?; + let identity = (record.schema_id, record.record_key); + if expectations + .insert(identity.clone(), record.source) + .is_some() + { + return Err(anyhow!( + "snapshot query contains duplicate requested record {:?}/{:?}", + identity.0, + identity.1 + )); + } + } + Ok(Self { + message_id, + expectations, + }) + } + + pub fn request(&self) -> CultNetMessage { + CultNetMessage::SnapshotRequest { + message_id: self.message_id.clone(), + schema_ids: Some( + self.expectations + .keys() + .map(|(schema_id, _)| schema_id.clone()) + .collect::>() + .into_iter() + .collect(), + ), + record_keys: Some( + self.expectations + .keys() + .map(|(_, record_key)| record_key.clone()) + .collect::>() + .into_iter() + .collect(), + ), + } + } + + /// Validate a response and return records without mutating any local cache. + pub fn accept_response( + &self, + response: CultNetMessage, + ) -> Result> { + let CultNetMessage::SnapshotResponseRaw { + message_id, + documents, + } = response + else { + return Err(anyhow!("expected cultnet.snapshot_response_raw.v0")); + }; + if message_id != self.message_id { + return Err(anyhow!( + "snapshot response message id {:?} does not match request {:?}", + message_id, + self.message_id + )); + } + + let mut seen = BTreeSet::new(); + for document in &documents { + let identity = (document.schema_id.clone(), document.record_key.clone()); + if !seen.insert(identity.clone()) { + return Err(anyhow!( + "snapshot response contains duplicate record {:?}/{:?}", + identity.0, + identity.1 + )); + } + let expected = self.expectations.get(&identity).ok_or_else(|| { + anyhow!( + "snapshot response contains unexpected record {:?}/{:?}", + identity.0, + identity.1 + ) + })?; + require_expected_metadata( + "source_runtime_id", + expected.runtime_id.as_deref(), + document.source_runtime_id.as_deref(), + &identity, + )?; + require_expected_metadata( + "source_agent_id", + expected.agent_id.as_deref(), + document.source_agent_id.as_deref(), + &identity, + )?; + require_expected_metadata( + "source_role", + expected.role.as_deref(), + document.source_role.as_deref(), + &identity, + )?; + if let Some(expected_tags) = expected.tags.as_ref() + && document.tags.as_ref() != Some(expected_tags) + { + return Err(anyhow!( + "snapshot response record {:?}/{:?} has unexpected tags: expected {:?}, got {:?}", + identity.0, + identity.1, + expected_tags, + document.tags + )); + } + } + Ok(documents) + } +} + +pub fn query_read_only_raw_snapshot( + query: &CultNetRawSnapshotQuery, + exchange: F, +) -> Result> +where + F: FnOnce(CultNetMessage) -> Result, +{ + query.accept_response(exchange(query.request())?) +} + +fn require_expected_metadata( + field: &str, + expected: Option<&str>, + actual: Option<&str>, + identity: &(String, String), +) -> Result<()> { + if let Some(expected) = expected + && actual != Some(expected) + { + return Err(anyhow!( + "snapshot response record {:?}/{:?} has unexpected {field}: expected {:?}, got {:?}", + identity.0, + identity.1, + expected, + actual + )); + } + Ok(()) +} + +fn reject_duplicates(values: Option<&[String]>, label: &str) -> Result<()> { + let Some(values) = values else { + return Ok(()); + }; + let mut seen = BTreeSet::new(); + for value in values { + require_non_empty(value, label)?; + if !seen.insert(value) { + return Err(anyhow!("duplicate {label} {value:?}")); + } + } + Ok(()) +} + +fn require_non_empty(value: &str, field: &str) -> Result<()> { + if value.trim().is_empty() { + return Err(anyhow!("{field} must be non-empty")); + } + Ok(()) +} diff --git a/packages/cultnet-rs/src/transport.rs b/packages/cultnet-rs/src/transport.rs index d7aad261..d097aa3c 100644 --- a/packages/cultnet-rs/src/transport.rs +++ b/packages/cultnet-rs/src/transport.rs @@ -19,6 +19,13 @@ pub struct CultNetTransportStats { pub bytes_sent: u64, pub frames_received: u64, pub frames_sent: u64, + /// Reliable packets dropped by a send-side expiry deadline instead of being + /// retransmitted. Always zero for transports without reliable expiry. + pub reliable_packets_expired: u64, + /// Incomplete fragment sets discarded to admit newer ones. Each is one + /// payload that lost a fragment on the wire. Zero for non-fragmenting + /// transports. + pub fragment_sets_evicted: u64, } #[derive(Clone, Debug, PartialEq, Eq)] @@ -201,6 +208,7 @@ pub fn create_tcp_framed_transport_profile( max_payload_bytes: options.max_payload_bytes, max_fragment_bytes: options.max_fragment_bytes, max_pending_reliable_packets: None, + reliable_expire_after_ms: None, }], }], } @@ -258,6 +266,27 @@ where let mut header = [0_u8; FRAME_HEADER_BYTES]; self.stream.read_exact(&mut header)?; let payload_len = u32::from_be_bytes(header) as usize; + // Refuse before allocating. The length is the peer's claim, so honouring + // it without checking the advertised maximum lets one header reserve as + // much memory as the peer cares to name. + if let Some(max_payload_bytes) = self + .profile + .transports + .iter() + .find(|transport| transport.protocol == CultNetTransportProtocol::TcpFramed) + .and_then(|transport| { + transport + .channels + .iter() + .find(|channel| channel.channel_id == "schema") + }) + .and_then(|channel| channel.max_payload_bytes) + && payload_len > max_payload_bytes as usize + { + return Err(anyhow!( + "tcp_framed payload length {payload_len} exceeds advertised maximum {max_payload_bytes}" + )); + } let mut payload = vec![0_u8; payload_len]; self.stream.read_exact(&mut payload)?; self.stats.bytes_received += (FRAME_HEADER_BYTES + payload_len) as u64; diff --git a/packages/cultnet-rs/tests/byte_slot_migration.rs b/packages/cultnet-rs/tests/byte_slot_migration.rs new file mode 100644 index 00000000..0acdcd4d --- /dev/null +++ b/packages/cultnet-rs/tests/byte_slot_migration.rs @@ -0,0 +1,113 @@ +//! Does adding `#[cultcache(key = N, bytes)]` to a field break reading the +//! records already written without it? +//! +//! The attribute changes a slot from a MessagePack array-of-integers (serde's +//! default for `Vec`) to a real `bin`. Six of cultnet-rs's byte fields +//! already carry it and four do not, so converting the remaining four is a +//! stored-data question before it is a signature question: every record +//! written before the flip holds an array in that slot. +//! +//! Both directions are tested, because a migration has to survive a rollback +//! as well as a roll-forward. + +use cultcache_rs::DatabaseEntry; +use serde::Serialize; + +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache(type = "test.byte_slot", schema = "test.byte_slot.v1")] +struct ArrayEncoded { + #[cultcache(key = 0)] + schema_version: String, + #[cultcache(key = 1)] + payload: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache(type = "test.byte_slot", schema = "test.byte_slot.v1")] +struct BinEncoded { + #[cultcache(key = 0)] + schema_version: String, + #[cultcache(key = 1, bytes)] + payload: Vec, +} + +fn sample() -> Vec { + (0u8..=255).collect() +} + +#[test] +fn the_attribute_is_what_switches_the_wire_shape() { + let array = rmp_serde::to_vec(&ArrayEncoded { + schema_version: "test.byte_slot.v1".into(), + payload: sample(), + }) + .expect("array encodes"); + let bin = rmp_serde::to_vec(&BinEncoded { + schema_version: "test.byte_slot.v1".into(), + payload: sample(), + }) + .expect("bin encodes"); + + // 256 bytes as `bin` is a 3-byte header plus the payload. As an array it + // is one or two bytes per element, because every value above 0x7f needs a + // u8 marker of its own. + println!("array-encoded {} bytes, bin-encoded {} bytes", array.len(), bin.len()); + assert!( + array.len() > bin.len(), + "array {} should exceed bin {}", + array.len(), + bin.len() + ); +} + +#[test] +fn a_bytes_reader_still_reads_records_written_before_the_attribute() { + let written_before = rmp_serde::to_vec(&ArrayEncoded { + schema_version: "test.byte_slot.v1".into(), + payload: sample(), + }) + .expect("array encodes"); + + // serde_bytes::ByteBuf accepts an array marker as well as bin, so the + // flip is NOT a hard read break. This was worth measuring rather than + // assuming: the opposite would have forced a dual-read path or a rewrite + // migration onto every field with persisted records. + let read_after = + rmp_serde::from_slice::(&written_before).expect("bytes reader reads an array"); + assert_eq!(read_after.payload, sample()); +} + +#[test] +fn a_pre_attribute_reader_still_reads_records_written_after_it() { + let written_after = rmp_serde::to_vec(&BinEncoded { + schema_version: "test.byte_slot.v1".into(), + payload: sample(), + }) + .expect("bin encodes"); + + // The rollback direction, so reverting the binary is survivable. + let read_by_old = + rmp_serde::from_slice::(&written_after).expect("plain reader reads bin"); + assert_eq!(read_by_old.payload, sample()); +} + +#[test] +fn re_encoding_an_old_record_under_the_attribute_does_not_reproduce_its_bytes() { + // Where the cost actually lands. Reading is compatible; reproducing is + // not. Idunn byte-exact-checks its control store by re-encoding what it + // decoded, and canonical_sha256 hashes the same serialization -- so a + // record written before the flip can be read, but can never re-encode to + // the bytes it was stored as, and its canonical digest changes. + let written_before = rmp_serde::to_vec(&ArrayEncoded { + schema_version: "test.byte_slot.v1".into(), + payload: sample(), + }) + .expect("array encodes"); + let decoded = rmp_serde::from_slice::(&written_before).expect("reads"); + let re_encoded = rmp_serde::to_vec(&decoded).expect("re-encodes"); + + assert_ne!( + written_before, re_encoded, + "identical bytes would mean the attribute changed nothing" + ); +} diff --git a/packages/cultnet-rs/tests/cultnet.rs b/packages/cultnet-rs/tests/cultnet.rs index 6511fdbd..0b76990f 100644 --- a/packages/cultnet-rs/tests/cultnet.rs +++ b/packages/cultnet-rs/tests/cultnet.rs @@ -4,6 +4,7 @@ use chrono::Utc; use cultcache_rs::CultCache; use cultcache_rs::DatabaseEntry; use cultcache_rs::SingleFileMessagePackBackingStore; +use cultnet_rs::CULTNET_RUDP_RELIABLE_SEND_WINDOW_PACKETS; use cultnet_rs::CultMesh; use cultnet_rs::CultMeshAuthorityLease; use cultnet_rs::CultMeshPeerCard; @@ -60,12 +61,14 @@ use cultnet_rs::encode_frame; use cultnet_rs::encode_rudp_packet; use pretty_assertions::assert_eq; use std::cell::RefCell; +use std::collections::VecDeque; use std::net::UdpSocket; use std::rc::Rc; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex}; use std::thread; use std::time::Duration as StdDuration; +use std::time::Instant as StdInstant; const TS_HELLO_FRAME: &[u8] = include_bytes!("fixtures/cultnet-ts-hello.frame"); const TS_LEGACY_LOGIN_FRAME: &[u8] = include_bytes!("fixtures/cultnet-ts-legacy-login.frame"); @@ -255,6 +258,7 @@ fn cultnet_schema_messages_round_trip_through_messagepack_frames() -> Result<()> max_payload_bytes: None, max_fragment_bytes: None, max_pending_reliable_packets: None, + reliable_expire_after_ms: None, }], }], }]), @@ -355,6 +359,7 @@ fn rudp_transport_profile_advertises_state_and_realtime_channels() { let profile = create_rudp_transport_profile( "rust-rudp", cultnet_rs::RudpTransportProfileOptions { + media_delivery: None, transport_id: Some("public-rudp".to_string()), host: Some("127.0.0.1".to_string()), port: Some(7777), @@ -362,6 +367,7 @@ fn rudp_transport_profile_advertises_state_and_realtime_channels() { max_fragment_bytes: Some(1000), max_pending_reliable_packets: Some(64), reconnect_policy: None, + media_reliable_expire_after_ms: None, }, ); @@ -422,6 +428,12 @@ fn rudp_transport_profile_advertises_state_and_realtime_channels() { CultNetTransportOrdering::Unordered, Some(64) ), + ( + "audio", + CultNetTransportDelivery::Reliable, + CultNetTransportOrdering::Unordered, + Some(64) + ), ] ); } @@ -754,6 +766,133 @@ fn rudp_session_pings_and_detects_receive_timeout() -> Result<()> { Ok(()) } +#[test] +fn rudp_lossy_packets_cannot_create_reliable_ordered_gaps() -> Result<()> { + let mut client = CultNetRudpSession::new(CultNetRudpSessionOptions { + connection_id: 198, + initial_sequence: 1, + ..CultNetRudpSessionOptions::default() + }); + let mut server = CultNetRudpSession::new(CultNetRudpSessionOptions { + connection_id: 198, + initial_sequence: 100, + ..CultNetRudpSessionOptions::default() + }); + let connect = client.create_connect(0, Vec::new())?; + let accept = server.accept_connect(&connect, 1, Vec::new())?; + client.receive(&accept, 2)?; + + let ping = client.create_ping(b"pulse".to_vec()); + let pong = server + .receive(&ping, 3)? + .reply + .expect("ping should produce pong"); + let realtime = server.send( + "realtime", + b"discarded realtime".to_vec(), + CultNetRudpSendOptions::default(), + )?; + let latest = server.send( + "latest", + b"discarded latest state".to_vec(), + CultNetRudpSendOptions { + sequenced: true, + ..CultNetRudpSendOptions::default() + }, + )?; + let schema = server.send( + "schema", + b"committed response".to_vec(), + CultNetRudpSendOptions { + reliable: true, + ordered: true, + ..CultNetRudpSendOptions::default() + }, + )?; + + assert_eq!(ping.sequence, 0); + assert_eq!(pong.sequence, 0); + assert_eq!(realtime.sequence, 0); + assert_eq!(latest.sequence, 1); + assert_eq!(schema.sequence, accept.sequence + 1); + assert_eq!( + client.receive(&schema, 4)?.delivered[0].payload, + b"committed response" + ); + Ok(()) +} + +#[test] +fn rudp_unreliable_sequenced_delivery_is_scoped_to_its_channel() -> Result<()> { + let mut sender = CultNetRudpSession::new(CultNetRudpSessionOptions { + connection_id: 197, + initial_sequence: 50, + ..CultNetRudpSessionOptions::default() + }); + let mut receiver = CultNetRudpSession::new(CultNetRudpSessionOptions { + connection_id: 197, + initial_sequence: 100, + ..CultNetRudpSessionOptions::default() + }); + let connect = sender.create_connect(0, Vec::new())?; + let accept = receiver.accept_connect(&connect, 1, Vec::new())?; + sender.receive(&accept, 2)?; + let options = CultNetRudpSendOptions { + sequenced: true, + ..CultNetRudpSendOptions::default() + }; + let older = sender.send("latest", b"older".to_vec(), options.clone())?; + let newer = sender.send("latest", b"newer".to_vec(), options)?; + + assert_eq!(older.sequence, 1); + assert_eq!(newer.sequence, 2); + assert_eq!(receiver.receive(&newer, 3)?.delivered[0].payload, b"newer"); + assert!(receiver.receive(&older, 4)?.delivered.is_empty()); + Ok(()) +} + +#[test] +fn rudp_rejects_unreliable_ordered_delivery() -> Result<()> { + let mut session = CultNetRudpSession::new(CultNetRudpSessionOptions { + connection_id: 196, + ..CultNetRudpSessionOptions::default() + }); + session.receive( + &CultNetRudpPacket { + packet_type: CultNetRudpPacketType::Accept, + connection_id: 196, + sequence: 50, + ack: 0, + ack_mask: 0, + channel_id: "control".to_string(), + reliable: false, + ordered: false, + sequenced: false, + fragment_id: 0, + fragment_index: 0, + fragment_count: 0, + payload: Vec::new(), + }, + 0, + )?; + let error = session + .send( + "schema", + b"cannot order what will not retransmit".to_vec(), + CultNetRudpSendOptions { + ordered: true, + ..CultNetRudpSendOptions::default() + }, + ) + .unwrap_err(); + assert!( + error + .to_string() + .contains("ordered delivery requires reliability") + ); + Ok(()) +} + #[test] fn rudp_session_bounds_pending_reliable_packets_before_enqueue() -> Result<()> { let mut session = CultNetRudpSession::new(CultNetRudpSessionOptions { @@ -1116,6 +1255,7 @@ fn rudp_socket_transport_handshakes_and_carries_reliable_ordered_schema_frames() let connection_id = 0x1020_3040; let mut server = CultNetRudpSocketTransportConnection::new(CultNetRudpSocketTransportOptions { + media_delivery: None, runtime_id: "rust-rudp-server".to_string(), socket: server_socket, mode: CultNetRudpSocketMode::Server, @@ -1128,9 +1268,11 @@ fn rudp_socket_transport_handshakes_and_carries_reliable_ordered_schema_frames() max_fragment_bytes: None, max_pending_reliable_packets: None, reconnect_policy: None, + media_reliable_expire_after_ms: None, })?; let mut client = CultNetRudpSocketTransportConnection::new(CultNetRudpSocketTransportOptions { + media_delivery: None, runtime_id: "rust-rudp-client".to_string(), socket: client_socket, mode: CultNetRudpSocketMode::Client, @@ -1143,6 +1285,7 @@ fn rudp_socket_transport_handshakes_and_carries_reliable_ordered_schema_frames() max_fragment_bytes: None, max_pending_reliable_packets: None, reconnect_policy: None, + media_reliable_expire_after_ms: None, })?; client.connect(b"join".to_vec())?; @@ -1206,6 +1349,7 @@ fn rudp_socket_transport_carries_fragmented_reliable_ordered_schema_frames() -> let connection_id = 0x1020_3041; let mut server = CultNetRudpSocketTransportConnection::new(CultNetRudpSocketTransportOptions { + media_delivery: None, runtime_id: "rust-rudp-fragment-server".to_string(), socket: server_socket, mode: CultNetRudpSocketMode::Server, @@ -1218,9 +1362,11 @@ fn rudp_socket_transport_carries_fragmented_reliable_ordered_schema_frames() -> max_fragment_bytes: Some(8), max_pending_reliable_packets: None, reconnect_policy: None, + media_reliable_expire_after_ms: None, })?; let mut client = CultNetRudpSocketTransportConnection::new(CultNetRudpSocketTransportOptions { + media_delivery: None, runtime_id: "rust-rudp-fragment-client".to_string(), socket: client_socket, mode: CultNetRudpSocketMode::Client, @@ -1233,6 +1379,7 @@ fn rudp_socket_transport_carries_fragmented_reliable_ordered_schema_frames() -> max_fragment_bytes: Some(8), max_pending_reliable_packets: None, reconnect_policy: None, + media_reliable_expire_after_ms: None, })?; client.connect(b"join".to_vec())?; @@ -1248,6 +1395,164 @@ fn rudp_socket_transport_carries_fragmented_reliable_ordered_schema_frames() -> Ok(()) } +#[test] +fn rudp_session_advances_large_fragment_sets_through_a_bounded_reliable_window() -> Result<()> { + let connection_id = 0x1020_3042; + let mut sender = CultNetRudpSession::new(CultNetRudpSessionOptions { + connection_id, + initial_sequence: 1, + resend_delay_ms: 25, + max_pending_reliable_packets: None, + }); + let mut receiver = CultNetRudpSession::new(CultNetRudpSessionOptions { + connection_id, + initial_sequence: 100, + resend_delay_ms: 25, + max_pending_reliable_packets: None, + }); + for session in [&mut sender, &mut receiver] { + session.receive( + &CultNetRudpPacket { + packet_type: CultNetRudpPacketType::Accept, + connection_id, + sequence: 0, + ack: 0, + ack_mask: 0, + channel_id: "control".into(), + reliable: false, + ordered: false, + sequenced: false, + fragment_id: 0, + fragment_index: 0, + fragment_count: 0, + payload: Vec::new(), + }, + 0, + )?; + } + + let fragment_count = CULTNET_RUDP_RELIABLE_SEND_WINDOW_PACKETS + 17; + let payload = (0..fragment_count * 8) + .map(|index| (index % 251) as u8) + .collect::>(); + let initial = sender.send_many( + "schema", + payload.clone(), + CultNetRudpSendOptions { + reliable: true, + ordered: true, + now_ms: 1, + ..CultNetRudpSendOptions::default() + }, + Some(8), + )?; + assert_eq!(initial.len(), CULTNET_RUDP_RELIABLE_SEND_WINDOW_PACKETS); + let oldest_sequence = initial[0].sequence; + assert_eq!(sender.pending_reliable_sequences().len(), initial.len()); + assert_eq!(sender.queued_reliable_packet_count(), 17); + + let mut wire = VecDeque::from(initial); + let mut delivered = Vec::new(); + while !wire.is_empty() { + let admitted = wire.len(); + for _ in 0..admitted { + let packet = wire.pop_front().expect("admitted packet remains queued"); + let received = receiver.receive(&packet, 2)?; + delivered.extend(received.delivered); + } + let acknowledged = sender.receive(&receiver.create_ack(), 3)?; + wire.extend(acknowledged.ready_to_send); + } + + assert_eq!(sender.outstanding_reliable_packet_count(), 0); + assert_eq!(delivered.len(), 1); + assert_eq!(delivered[0].payload, payload); + let old_ack = receiver.create_ack_for_received(oldest_sequence); + assert_eq!(old_ack.ack, oldest_sequence); + assert_eq!(old_ack.ack_mask, 0); + Ok(()) +} + +#[test] +fn rudp_socket_flush_waits_for_large_fragment_delivery() -> Result<()> { + let server_socket = bind_udp_socket()?; + let client_socket = bind_udp_socket()?; + let server_addr = server_socket.local_addr()?; + let connection_id = 0x1020_3043; + let mut server = + CultNetRudpSocketTransportConnection::new(CultNetRudpSocketTransportOptions { + media_delivery: None, + runtime_id: "rust-rudp-window-server".into(), + socket: server_socket, + mode: CultNetRudpSocketMode::Server, + remote_addr: None, + connection_id, + initial_sequence: 100, + resend_delay_ms: 25, + transport_id: None, + max_payload_bytes: None, + max_fragment_bytes: Some(8), + max_pending_reliable_packets: None, + reconnect_policy: None, + media_reliable_expire_after_ms: None, + })?; + let mut client = + CultNetRudpSocketTransportConnection::new(CultNetRudpSocketTransportOptions { + media_delivery: None, + runtime_id: "rust-rudp-window-client".into(), + socket: client_socket, + mode: CultNetRudpSocketMode::Client, + remote_addr: Some(server_addr), + connection_id, + initial_sequence: 1, + resend_delay_ms: 25, + transport_id: None, + max_payload_bytes: None, + max_fragment_bytes: Some(8), + max_pending_reliable_packets: None, + reconnect_policy: None, + media_reliable_expire_after_ms: None, + })?; + client.connect(b"join".to_vec())?; + pump_rudp_handshake(&mut client, &mut server)?; + + let payload = vec![0x5a; (CULTNET_RUDP_RELIABLE_SEND_WINDOW_PACKETS + 17) * 8]; + client.send("schema", payload.clone())?; + assert_eq!( + client.pending_reliable_packet_count(), + CULTNET_RUDP_RELIABLE_SEND_WINDOW_PACKETS + ); + assert_eq!( + client.outstanding_reliable_packet_count(), + CULTNET_RUDP_RELIABLE_SEND_WINDOW_PACKETS + 17 + ); + + let server_worker = thread::spawn(move || -> Result { + let delivery_deadline = StdInstant::now() + StdDuration::from_secs(5); + let delivered = loop { + if let Some(frame) = server.receive_once()? { + break frame; + } + if StdInstant::now() >= delivery_deadline { + anyhow::bail!("RUDP large socket frame was not delivered"); + } + }; + let linger_deadline = StdInstant::now() + StdDuration::from_millis(250); + while StdInstant::now() < linger_deadline { + let _ = server.receive_once()?; + server.poll_resends()?; + } + Ok(delivered) + }); + client.flush_reliable(StdDuration::from_secs(5))?; + let delivered = server_worker + .join() + .map_err(|_| anyhow::anyhow!("RUDP server worker panicked"))??; + assert_eq!(delivered.payload, payload); + assert_eq!(client.outstanding_reliable_packet_count(), 0); + Ok(()) +} + #[test] fn cultmesh_facade_creates_rudp_client_from_peer_endpoint() -> Result<()> { let connection_id = 0x2030_4050; @@ -1645,7 +1950,7 @@ fn document_registry_replicates_typed_cultcache_state() -> Result<()> { let mut origin = CultCache::new(); origin.register_entry_type::()?; - origin.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&origin_store)); + origin.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&origin_store))?; origin.pull_all_backing_stores()?; origin.put("epiphany.persona", &payload)?; @@ -1653,7 +1958,7 @@ fn document_registry_replicates_typed_cultcache_state() -> Result<()> { let mut target = CultCache::new(); target.register_entry_type::()?; - target.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&target_store)); + target.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&target_store))?; target.pull_all_backing_stores()?; let applied = registry.apply_snapshot_response::(&mut target, &snapshot)?; @@ -1702,7 +2007,7 @@ fn raw_snapshot_replication_preserves_messagepack_payload_bytes() -> Result<()> let mut origin = CultCache::new(); origin.register_entry_type::()?; - origin.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&origin_store)); + origin.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&origin_store))?; origin.pull_all_backing_stores()?; origin.put("epiphany.persona", &payload)?; @@ -1713,7 +2018,7 @@ fn raw_snapshot_replication_preserves_messagepack_payload_bytes() -> Result<()> let mut target = CultCache::new(); target.register_entry_type::()?; - target.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&target_store)); + target.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&target_store))?; target.pull_all_backing_stores()?; let applied = registry .apply_raw_snapshot_response::(&mut target, &raw_snapshot)?; @@ -1749,7 +2054,7 @@ fn raw_snapshot_replication_hydrates_same_schema_rust_aliases() -> Result<()> { )); let mut origin = CultCache::new(); origin.register_entry_type::()?; - origin.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&origin_store)); + origin.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&origin_store))?; origin.pull_all_backing_stores()?; origin.put("epiphany.persona", &canonical)?; let raw_snapshot = origin_registry.create_raw_snapshot_response( @@ -1768,7 +2073,7 @@ fn raw_snapshot_replication_hydrates_same_schema_rust_aliases() -> Result<()> { )); let mut target = CultCache::new(); target.register_entry_type::()?; - target.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&target_store)); + target.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&target_store))?; target.pull_all_backing_stores()?; let applied = alias_registry @@ -1808,7 +2113,7 @@ fn reactive_document_coalesces_direct_same_schema_alias_member_writes() -> Resul { let mut cache = cache.lock().expect("cache mutex"); cache.register_entry_type::()?; - cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&target_store)); + cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&target_store))?; cache.pull_all_backing_stores()?; cache.put( "note:reactive", @@ -1873,7 +2178,7 @@ fn reactive_document_tracks_canonical_reconciliation_delta() -> Result<()> { { let mut cache = cache.lock().expect("cache mutex"); cache.register_entry_type::()?; - cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&target_store)); + cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&target_store))?; cache.pull_all_backing_stores()?; cache.put( "note:reconcile", @@ -2133,3 +2438,514 @@ fn schema_discovery_round_trips_over_legacy_gamecult_contract_when_inline_schema ); Ok(()) } + +#[test] +fn rudp_sequenced_channel_tracking_is_bounded() -> Result<()> { + let mut sender = CultNetRudpSession::new(CultNetRudpSessionOptions { + connection_id: 401, + initial_sequence: 1, + ..CultNetRudpSessionOptions::default() + }); + let mut receiver = CultNetRudpSession::new(CultNetRudpSessionOptions { + connection_id: 401, + initial_sequence: 100, + ..CultNetRudpSessionOptions::default() + }); + let connect = sender.create_connect(0, Vec::new())?; + let accept = receiver.accept_connect(&connect, 1, Vec::new())?; + sender.receive(&accept, 2)?; + + // A peer picks its own channel ids, so the sequenced map is attacker-sized. + let mut error = None; + for channel in 0..256u32 { + let packet = sender.send( + &format!("channel-{channel}"), + b"state".to_vec(), + CultNetRudpSendOptions { + sequenced: true, + ..CultNetRudpSendOptions::default() + }, + )?; + if let Err(refused) = receiver.receive(&packet, 3) { + error = Some(refused); + break; + } + } + let error = error.expect("unbounded sequenced channel tracking"); + assert!( + error.to_string().contains("too many sequenced channels"), + "unexpected refusal: {error}" + ); + Ok(()) +} + +#[test] +fn rudp_ordered_channel_tracking_is_bounded() -> Result<()> { + let mut sender = CultNetRudpSession::new(CultNetRudpSessionOptions { + connection_id: 402, + initial_sequence: 1, + ..CultNetRudpSessionOptions::default() + }); + let mut receiver = CultNetRudpSession::new(CultNetRudpSessionOptions { + connection_id: 402, + initial_sequence: 100, + ..CultNetRudpSessionOptions::default() + }); + let connect = sender.create_connect(0, Vec::new())?; + let accept = receiver.accept_connect(&connect, 1, Vec::new())?; + sender.receive(&accept, 2)?; + + let mut error = None; + for channel in 0..256u32 { + let packet = sender.send( + &format!("ordered-{channel}"), + b"state".to_vec(), + CultNetRudpSendOptions { + reliable: true, + ordered: true, + ..CultNetRudpSendOptions::default() + }, + )?; + match receiver.receive(&packet, 3) { + Ok(_) => { + sender.receive(&receiver.create_ack_for(packet.sequence), 3)?; + } + Err(refused) => { + error = Some(refused); + break; + } + } + } + let error = error.expect("unbounded ordered channel tracking"); + assert!( + error.to_string().contains("too many ordered channels"), + "unexpected refusal: {error}" + ); + Ok(()) +} + +#[test] +fn rudp_ordered_hold_buffer_is_bounded() -> Result<()> { + let mut sender = CultNetRudpSession::new(CultNetRudpSessionOptions { + connection_id: 403, + initial_sequence: 1, + ..CultNetRudpSessionOptions::default() + }); + let mut receiver = CultNetRudpSession::new(CultNetRudpSessionOptions { + connection_id: 403, + initial_sequence: 100, + ..CultNetRudpSessionOptions::default() + }); + let connect = sender.create_connect(0, Vec::new())?; + let accept = receiver.accept_connect(&connect, 1, Vec::new())?; + sender.receive(&accept, 2)?; + + let options = CultNetRudpSendOptions { + reliable: true, + ordered: true, + ..CultNetRudpSendOptions::default() + }; + // Withhold the first frame so every later one has to be held. + let withheld = sender.send("schema", b"first".to_vec(), options.clone())?; + sender.receive(&receiver.create_ack_for(withheld.sequence), 3)?; + + let mut error = None; + for step in 0..4_096u32 { + let packet = sender.send("schema", vec![(step % 251) as u8], options.clone())?; + match receiver.receive(&packet, 4) { + Ok(result) => { + assert!(result.delivered.is_empty(), "gap should hold every frame"); + sender.receive(&receiver.create_ack_for(packet.sequence), 4)?; + } + Err(refused) => { + error = Some(refused); + break; + } + } + } + let error = error.expect("unbounded ordered hold buffer"); + assert!( + error.to_string().contains("ordered hold buffer is full"), + "unexpected refusal: {error}" + ); + Ok(()) +} + +#[test] +fn rudp_far_ahead_sequence_cannot_starve_the_session() -> Result<()> { + let mut sender = CultNetRudpSession::new(CultNetRudpSessionOptions { + connection_id: 404, + initial_sequence: 1, + ..CultNetRudpSessionOptions::default() + }); + let mut receiver = CultNetRudpSession::new(CultNetRudpSessionOptions { + connection_id: 404, + initial_sequence: 100, + ..CultNetRudpSessionOptions::default() + }); + let connect = sender.create_connect(0, Vec::new())?; + let accept = receiver.accept_connect(&connect, 1, Vec::new())?; + sender.receive(&accept, 2)?; + + let options = CultNetRudpSendOptions { + reliable: true, + ..CultNetRudpSendOptions::default() + }; + let first = sender.send("schema", b"before".to_vec(), options.clone())?; + assert_eq!(receiver.receive(&first, 3)?.delivered.len(), 1); + sender.receive(&receiver.create_ack_for(first.sequence), 3)?; + + // A forged packet far past the current highest. Recording it would push the + // receive state beyond every sequence still in flight. + let forged = CultNetRudpPacket { + packet_type: CultNetRudpPacketType::Data, + connection_id: 404, + sequence: first.sequence.saturating_add(500_000), + ack: 0, + ack_mask: 0, + channel_id: "schema".to_string(), + reliable: true, + ordered: false, + sequenced: false, + fragment_id: 0, + fragment_index: 0, + fragment_count: 0, + payload: b"hostile".to_vec(), + }; + assert!(receiver.receive(&forged, 4)?.delivered.is_empty()); + + // The session must still accept the next legitimate packet. + let second = sender.send("schema", b"after".to_vec(), options)?; + assert_eq!( + receiver.receive(&second, 5)?.delivered[0].payload, + b"after".to_vec() + ); + Ok(()) +} + +#[test] +fn rudp_reliable_sequence_exhaustion_is_reported_not_fatal() -> Result<()> { + let mut sender = CultNetRudpSession::new(CultNetRudpSessionOptions { + connection_id: 405, + // One sequence left: the handshake takes it, so the next send is the + // first allocation with nowhere to go. + initial_sequence: u32::MAX - 1, + ..CultNetRudpSessionOptions::default() + }); + let mut receiver = CultNetRudpSession::new(CultNetRudpSessionOptions { + connection_id: 405, + initial_sequence: 100, + ..CultNetRudpSessionOptions::default() + }); + let connect = sender.create_connect(0, Vec::new())?; + let accept = receiver.accept_connect(&connect, 1, Vec::new())?; + sender.receive(&accept, 2)?; + + let error = sender + .send( + "schema", + b"one past the end".to_vec(), + CultNetRudpSendOptions { + reliable: true, + ..CultNetRudpSendOptions::default() + }, + ) + .unwrap_err(); + assert!( + error.to_string().contains("sequence space is exhausted"), + "unexpected error: {error}" + ); + Ok(()) +} + +#[test] +fn tcp_framed_receive_refuses_a_length_beyond_the_advertised_maximum() -> Result<()> { + let profile = create_tcp_framed_transport_profile( + "rust-transport", + TcpFramedTransportProfileOptions { + transport_id: Some("bounded-tcp".to_string()), + max_payload_bytes: Some(64), + ..TcpFramedTransportProfileOptions::default() + }, + ); + let advertised = profile + .transports + .iter() + .find(|transport| transport.protocol == CultNetTransportProtocol::TcpFramed) + .and_then(|transport| { + transport + .channels + .iter() + .find(|channel| channel.channel_id == "schema") + }) + .and_then(|channel| channel.max_payload_bytes) + .expect("tcp_framed schema channel advertises a maximum payload"); + + // Only a header, declaring more than the profile allows. Honouring it would + // reserve the peer's number before a single payload byte arrives. + let mut header = ((advertised as u64 + 1) as u32).to_be_bytes().to_vec(); + header.extend_from_slice(b"short"); + let mut receiver = TcpFramedTransportConnection::new(std::io::Cursor::new(header), profile); + let error = receiver.receive().unwrap_err(); + assert!( + error.to_string().contains("exceeds advertised maximum"), + "unexpected error: {error}" + ); + Ok(()) +} + +#[test] +fn rudp_reliable_expiry_drops_late_packets_and_leaves_durable_sends_alone() -> Result<()> { + let mut sender = CultNetRudpSession::new(CultNetRudpSessionOptions { + connection_id: 7, + initial_sequence: 10, + resend_delay_ms: 100, + ..CultNetRudpSessionOptions::default() + }); + accept_rudp_session(&mut sender, 7)?; + + let expiring = sender.send( + "media", + b"late".to_vec(), + CultNetRudpSendOptions { + reliable: true, + now_ms: 0, + reliable_expire_after_ms: Some(50), + ..CultNetRudpSendOptions::default() + }, + )?; + let durable = sender.send( + "schema", + b"state".to_vec(), + CultNetRudpSendOptions { + reliable: true, + now_ms: 0, + ..CultNetRudpSendOptions::default() + }, + )?; + assert_eq!(sender.pending_reliable_sequences().len(), 2); + assert_eq!(sender.reliable_packets_expired(), 0); + + // One millisecond before the deadline nothing is dropped: the boundary is + // inclusive, so a packet is never discarded early. + sender.send( + "schema", + b"tick".to_vec(), + CultNetRudpSendOptions { + reliable: false, + now_ms: 50, + ..CultNetRudpSendOptions::default() + }, + )?; + assert_eq!(sender.reliable_packets_expired(), 0); + + sender.send( + "schema", + b"tick".to_vec(), + CultNetRudpSendOptions { + reliable: false, + now_ms: 51, + ..CultNetRudpSendOptions::default() + }, + )?; + assert_eq!(sender.reliable_packets_expired(), 1); + assert_eq!( + sender.pending_reliable_sequences(), + vec![durable.sequence], + "a send without an expiry deadline is never dropped" + ); + assert_ne!(expiring.sequence, durable.sequence); + Ok(()) +} + +#[test] +fn rudp_reliable_expiry_also_reclaims_the_send_backlog() -> Result<()> { + let mut sender = CultNetRudpSession::new(CultNetRudpSessionOptions { + connection_id: 9, + initial_sequence: 10, + resend_delay_ms: 100, + ..CultNetRudpSessionOptions::default() + }); + accept_rudp_session(&mut sender, 9)?; + + let fragment_count = CULTNET_RUDP_RELIABLE_SEND_WINDOW_PACKETS + 5; + let payload = (0..fragment_count * 8) + .map(|index| (index % 251) as u8) + .collect::>(); + sender.send_many( + "media", + payload, + CultNetRudpSendOptions { + reliable: true, + now_ms: 0, + reliable_expire_after_ms: Some(30), + ..CultNetRudpSendOptions::default() + }, + Some(8), + )?; + assert_eq!( + sender.pending_reliable_sequences().len(), + CULTNET_RUDP_RELIABLE_SEND_WINDOW_PACKETS + ); + assert_eq!(sender.queued_reliable_packet_count(), 5); + + // The deadline is fixed at admission, so waiting in the backlog spends it + // exactly as waiting for an acknowledgement does. A queued packet promoted + // after its moment has passed would be as useless as a retransmitted one. + sender.send( + "schema", + b"tick".to_vec(), + CultNetRudpSendOptions { + reliable: false, + now_ms: 31, + ..CultNetRudpSendOptions::default() + }, + )?; + assert_eq!(sender.pending_reliable_sequences().len(), 0); + assert_eq!(sender.queued_reliable_packet_count(), 0); + assert_eq!( + sender.reliable_packets_expired(), + fragment_count as u64, + "both the in-flight window and the backlog are counted" + ); + Ok(()) +} + +fn accept_rudp_session(session: &mut CultNetRudpSession, connection_id: u32) -> Result<()> { + session.receive( + &CultNetRudpPacket { + packet_type: CultNetRudpPacketType::Accept, + connection_id, + sequence: 0, + ack: 0, + ack_mask: 0, + channel_id: "control".into(), + reliable: false, + ordered: false, + sequenced: false, + fragment_id: 0, + fragment_index: 0, + fragment_count: 0, + payload: Vec::new(), + }, + 0, + )?; + Ok(()) +} + +#[test] +fn rudp_profile_expires_only_the_media_channels() -> Result<()> { + let socket = std::net::UdpSocket::bind("127.0.0.1:0")?; + let peer = socket.local_addr()?; + let options = CultNetRudpSocketTransportOptions::client("expiry-profile", socket, peer, 11); + let connection = CultNetRudpSocketTransportConnection::new(options)?; + + let channels = connection + .profile + .transports + .iter() + .flat_map(|transport| transport.channels.iter()) + .map(|channel| (channel.channel_id.as_str(), channel.reliable_expire_after_ms)) + .collect::>(); + + for media_channel in ["media", "audio"] { + assert!( + channels.contains(&(media_channel, Some(75))), + "the {media_channel} channel advertises its expiry so a peer can see it: {channels:?}" + ); + } + for (channel_id, expiry) in &channels { + if !matches!(*channel_id, "media" | "audio") { + assert_eq!( + *expiry, None, + "state channel {channel_id} must never expire a reliable send" + ); + } + } + Ok(()) +} + +fn connected_rudp_pair() -> Result<(CultNetRudpSession, CultNetRudpSession)> { + let mut client = CultNetRudpSession::new(CultNetRudpSessionOptions { + connection_id: 311, + initial_sequence: 1, + ..CultNetRudpSessionOptions::default() + }); + let mut server = CultNetRudpSession::new(CultNetRudpSessionOptions { + connection_id: 311, + initial_sequence: 100, + ..CultNetRudpSessionOptions::default() + }); + let connect = client.create_connect(0, Vec::new())?; + let accept = server.accept_connect(&connect, 1, Vec::new())?; + client.receive(&accept, 2)?; + Ok((client, server)) +} + +#[test] +fn rudp_evicts_stalest_incomplete_fragment_set_instead_of_dying() -> Result<()> { + let (mut client, mut server) = connected_rudp_pair()?; + client.set_max_pending_fragment_sets(4)?; + let fragmented = |server: &mut CultNetRudpSession, fill: u8| { + server.send_many( + "media", + vec![fill; 2500], + CultNetRudpSendOptions::default(), + Some(1000), + ) + }; + + // Eight payloads each lose their last fragment on the wire. Before, the + // fifth new set errored the receiver out and every later fragmented + // payload with it. + let mut stranded = Vec::new(); + for fill in 1..=8u8 { + let packets = fragmented(&mut server, fill)?; + assert_eq!(packets.len(), 3); + for packet in &packets[..2] { + assert!(client.receive(packet, 10)?.delivered.is_empty()); + } + stranded.push(packets[2].clone()); + } + assert_eq!(client.fragment_sets_evicted(), 4); + + // A complete payload still lands. + let complete = fragmented(&mut server, 9)?; + let mut delivered = Vec::new(); + for packet in &complete { + delivered.extend(client.receive(packet, 11)?.delivered); + } + assert_eq!(delivered.len(), 1); + assert_eq!(delivered[0].payload, vec![9u8; 2500]); + assert_eq!(client.fragment_sets_evicted(), 5); + + // The eviction order is stalest-first: set 8 is still pending and completes + // when its late fragment shows up; set 1 was evicted long ago, so its late + // fragment opens a fresh set and delivers nothing. + assert_eq!( + client.receive(&stranded[7], 12)?.delivered[0].payload, + vec![8u8; 2500] + ); + assert!(client.receive(&stranded[0], 13)?.delivered.is_empty()); + Ok(()) +} + +#[test] +fn rudp_fragment_ids_wrap_instead_of_saturating() -> Result<()> { + let (_client, mut server) = connected_rudp_pair()?; + let mut ids = Vec::new(); + for _ in 0..(u16::MAX as usize + 1) { + let packets = server.send_many( + "media", + vec![0u8; 1500], + CultNetRudpSendOptions::default(), + Some(1000), + )?; + ids.push(packets[0].fragment_id); + } + assert_eq!(ids[0], 1); + assert_eq!(ids[u16::MAX as usize - 1], u16::MAX); + // The 65,536th set must not share an id with the 65,535th. + assert_eq!(ids[u16::MAX as usize], 1); + Ok(()) +} diff --git a/packages/cultnet-rs/tests/media_channel_delivery.rs b/packages/cultnet-rs/tests/media_channel_delivery.rs new file mode 100644 index 00000000..499be0d3 --- /dev/null +++ b/packages/cultnet-rs/tests/media_channel_delivery.rs @@ -0,0 +1,92 @@ +//! Media delivery is the caller's choice, and the advertised profile is where +//! that choice lives. +//! +//! Before this, `media` was hardcoded `Reliable` in two places — the profile +//! builder and the per-channel send options — so no caller could say otherwise. +//! A producer asking for an unreliable media lane got a reliable one that +//! retransmits under loss, which adds load exactly when the link has least to +//! give. Muninn asked for that lane by passing no expiry, meaning "unreliable, +//! so expiry is moot", and got reliable-with-no-expiry instead: retransmit +//! forever. +//! +//! The default stays `Reliable` so every existing consumer is untouched. + +use std::net::UdpSocket; + +use cultnet_rs::{ + CultNetRudpSocketTransportConnection, CultNetRudpSocketTransportOptions, + CultNetTransportDelivery, +}; + +fn media_channel_delivery( + media_delivery: Option, +) -> (CultNetTransportDelivery, Option) { + let socket = UdpSocket::bind("127.0.0.1:0").expect("binds"); + let endpoint = "127.0.0.1:5204".parse().expect("parses"); + let mut options = CultNetRudpSocketTransportOptions::client("test-media", socket, endpoint, 7); + options.media_delivery = media_delivery; + options.media_reliable_expire_after_ms = None; + + let transport = CultNetRudpSocketTransportConnection::new(options).expect("opens"); + let channel = transport + .profile + .transports + .first() + .expect("a transport") + .channels + .iter() + .find(|channel| channel.channel_id == "media") + .expect("a media channel"); + (channel.delivery, channel.reliable_expire_after_ms) +} + +#[test] +fn media_defaults_to_reliable_so_existing_consumers_do_not_move() { + let (delivery, _) = media_channel_delivery(None); + assert_eq!(delivery, CultNetTransportDelivery::Reliable); +} + +#[test] +fn a_caller_can_ask_for_an_unreliable_media_lane() { + let (delivery, expiry) = media_channel_delivery(Some(CultNetTransportDelivery::Unreliable)); + assert_eq!(delivery, CultNetTransportDelivery::Unreliable); + assert_eq!( + expiry, None, + "an unreliable channel has nothing to expire; the pairing should stay honest" + ); +} + +#[test] +fn asking_for_reliable_explicitly_matches_the_default() { + let (explicit, _) = media_channel_delivery(Some(CultNetTransportDelivery::Reliable)); + let (defaulted, _) = media_channel_delivery(None); + assert_eq!(explicit, defaulted); +} + +/// The channels whose semantics are fixed by design must not drift with this. +#[test] +fn the_other_channels_keep_their_fixed_delivery() { + let socket = UdpSocket::bind("127.0.0.1:0").expect("binds"); + let endpoint = "127.0.0.1:5204".parse().expect("parses"); + let mut options = CultNetRudpSocketTransportOptions::client("test-media", socket, endpoint, 8); + options.media_delivery = Some(CultNetTransportDelivery::Unreliable); + + let transport = CultNetRudpSocketTransportConnection::new(options).expect("opens"); + let channels = &transport + .profile + .transports + .first() + .expect("a transport") + .channels; + + let delivery_of = |id: &str| { + channels + .iter() + .find(|channel| channel.channel_id == id) + .unwrap_or_else(|| panic!("channel {id}")) + .delivery + }; + + assert_eq!(delivery_of("schema"), CultNetTransportDelivery::Reliable); + assert_eq!(delivery_of("realtime"), CultNetTransportDelivery::Unreliable); +} diff --git a/packages/cultnet-rs/tests/media_stream_contracts.rs b/packages/cultnet-rs/tests/media_stream_contracts.rs new file mode 100644 index 00000000..963d78a2 --- /dev/null +++ b/packages/cultnet-rs/tests/media_stream_contracts.rs @@ -0,0 +1,217 @@ +//! The CultMesh Media Stream records are a wire contract shared by producers and +//! consumers that never link each other. These tests pin the properties a +//! consumer is entitled to rely on: positional layout, declared field order, and +//! tolerance of records written before an appended field existed. + +use cultcache_rs::DatabaseEntry; +use cultnet_rs::{ + GAMECULT_MEDIA_AUDIO_PACKET_SCHEMA, GAMECULT_MEDIA_RECEIVER_FEEDBACK_SCHEMA, + GAMECULT_MEDIA_VIDEO_ACCESS_UNIT_SCHEMA, GAMECULT_MEDIA_VIDEO_PARITY_SHARD_SCHEMA, + GameCultMediaAudioPacketRecord, GameCultMediaReceiverFeedbackRecord, + GameCultMediaVideoAccessUnitRecord, GameCultMediaVideoParityShardRecord, +}; +use rmpv::Value; + +fn access_unit() -> GameCultMediaVideoAccessUnitRecord { + GameCultMediaVideoAccessUnitRecord { + stream_id: "raven-primary-av".to_string(), + session_id: "session-1".to_string(), + frame_id: 4242, + codec: "h264".to_string(), + pts_ticks: 90_000, + duration_ticks: 3_000, + timebase_num: 1, + timebase_den: 90_000, + keyframe: true, + dependency_frame_id: None, + deadline_ticks: 108_000, + chunk_index: 2, + chunk_count: 5, + payload: vec![0xDE, 0xAD, 0xBE, 0xEF], + } +} + +fn audio_packet() -> GameCultMediaAudioPacketRecord { + GameCultMediaAudioPacketRecord { + stream_id: "raven-primary-av".to_string(), + session_id: "session-1".to_string(), + packet_id: 77, + codec: "opus".to_string(), + pts_ticks: 48_000, + duration_ticks: 960, + timebase_num: 1, + timebase_den: 48_000, + deadline_ticks: 96_000, + payload: vec![1, 2, 3], + } +} + +fn decode_as_value(bytes: &[u8]) -> Value { + rmp_serde::from_slice(bytes).expect("records decode as generic MessagePack") +} + +/// The declared `key` indices are the wire layout, not documentation. A consumer +/// in another language reads position 2 for `frame_id` and position 13 for +/// `payload`; reordering the struct would silently break it. +#[test] +fn video_access_unit_encodes_as_a_positional_array() { + let encoded = rmp_serde::to_vec(&access_unit()).expect("encodes"); + let Value::Array(slots) = decode_as_value(&encoded) else { + panic!("access units must encode as a positional array"); + }; + + assert_eq!(slots.len(), 14, "field count is wire contract"); + assert_eq!(slots[0].as_str(), Some("raven-primary-av")); + assert_eq!(slots[2].as_u64(), Some(4242), "slot 2 is frame_id"); + assert_eq!(slots[3].as_str(), Some("h264")); + assert_eq!(slots[8].as_bool(), Some(true), "slot 8 is keyframe"); + assert!(slots[9].is_nil(), "an absent dependency stays nil, not 0"); + assert_eq!(slots[12].as_u64(), Some(5), "slot 12 is chunk_count"); + assert_eq!( + slots[13].as_slice(), + Some([0xDE, 0xAD, 0xBE, 0xEF].as_slice()), + "slot 13 is the payload, carried as bin" + ); +} + +/// Payloads must reach the wire as MessagePack `bin`, not as an array of +/// integers. The C#, TypeScript and Python runtimes all read byte fields as +/// `bin`, so a payload serialized as a sequence is unreadable by the reference +/// implementation, and costs 1.45x besides: 848 bytes encode to 1234 as an +/// integer array against 851 as `bin`. +/// +/// This is what the derive's `bytes` attribute buys. `#[serde(with = +/// "serde_bytes")]` will not do it — `DatabaseEntry` emits its own `Serialize` +/// and reads only its own `cultcache` attribute namespace. +#[test] +fn payload_should_encode_as_messagepack_bin() { + let encoded = rmp_serde::to_vec(&access_unit()).expect("encodes"); + let Value::Array(slots) = decode_as_value(&encoded) else { + panic!("access units must encode as a positional array"); + }; + + assert_eq!( + slots[13].as_slice(), + Some([0xDE, 0xAD, 0xBE, 0xEF].as_slice()), + "payloads belong on the wire as bin, not as an array of integers" + ); +} + +#[test] +fn audio_packets_are_whole_and_carry_their_own_timebase() { + let encoded = rmp_serde::to_vec(&audio_packet()).expect("encodes"); + let Value::Array(slots) = decode_as_value(&encoded) else { + panic!("audio packets must encode as a positional array"); + }; + + assert_eq!(slots.len(), 10, "field count is wire contract"); + assert_eq!(slots[2].as_u64(), Some(77), "slot 2 is packet_id"); + assert_eq!(slots[6].as_u64(), Some(1), "slot 6 is timebase_num"); + assert_eq!(slots[7].as_u64(), Some(48_000), "slot 7 is timebase_den"); + + let decoded: GameCultMediaAudioPacketRecord = + rmp_serde::from_slice(&encoded).expect("round-trips"); + assert_eq!(decoded, audio_packet()); +} + +#[test] +fn parity_shards_carry_the_geometry_recovery_needs() { + let shard = GameCultMediaVideoParityShardRecord { + stream_id: "raven-primary-av".to_string(), + session_id: "session-1".to_string(), + frame_id: 4242, + codec: "h264".to_string(), + pts_ticks: 90_000, + duration_ticks: 3_000, + timebase_num: 1, + timebase_den: 90_000, + keyframe: false, + dependency_frame_id: Some(4241), + deadline_ticks: 108_000, + chunk_count: 5, + parity_index: 1, + parity_count: 2, + chunk_payload_bytes: 1024, + last_chunk_payload_bytes: 512, + payload: vec![9, 9, 9], + }; + + let encoded = rmp_serde::to_vec(&shard).expect("encodes"); + let Value::Array(slots) = decode_as_value(&encoded) else { + panic!("parity shards must encode as a positional array"); + }; + + assert_eq!(slots.len(), 17, "field count is wire contract"); + assert_eq!(slots[12].as_u64(), Some(1), "slot 12 is parity_index"); + assert_eq!(slots[13].as_u64(), Some(2), "slot 13 is parity_count"); + assert_eq!( + slots[15].as_u64(), + Some(512), + "slot 15 is last_chunk_payload_bytes, which recovery needs to size the tail" + ); + + let decoded: GameCultMediaVideoParityShardRecord = + rmp_serde::from_slice(&encoded).expect("round-trips"); + assert_eq!(decoded, shard); +} + +/// `missing_video_chunk_keys` was appended after producers were already writing +/// feedback. A record written without it must still decode, or the append was a +/// breaking change wearing a `default` attribute. +#[test] +fn receiver_feedback_decodes_records_written_before_its_last_field() { + let without_appended_field = Value::Array(vec![ + Value::from("raven-primary-av"), + Value::from("session-1"), + Value::from("starfire-obs"), + Value::from(4200u64), + Value::Array(vec![Value::from(4198u64)]), + Value::Array(vec![Value::from(4199u64)]), + Value::from(true), + Value::from(1_500i64), + Value::from(9_000i64), + Value::from("2026-09-09T00:00:00Z"), + ]); + + let mut bytes = Vec::new(); + rmpv::encode::write_value(&mut bytes, &without_appended_field).expect("writes legacy record"); + + let decoded: GameCultMediaReceiverFeedbackRecord = + rmp_serde::from_slice(&bytes).expect("a pre-append record must still decode"); + + assert_eq!(decoded.receiver_id, "starfire-obs"); + assert_eq!(decoded.highest_decodable_frame_id, Some(4200)); + assert!(decoded.requested_keyframe); + assert!( + decoded.missing_video_chunk_keys.is_empty(), + "the appended field defaults rather than failing the decode" + ); +} + +/// The published constants and the derived type identity must not drift apart; +/// a consumer resolving by schema id and a producer registering by type would +/// then disagree without either failing to compile. +#[test] +fn published_schema_constants_match_the_derived_identity() { + assert_eq!( + GameCultMediaVideoAccessUnitRecord::SCHEMA_NAME, + GAMECULT_MEDIA_VIDEO_ACCESS_UNIT_SCHEMA + ); + assert_eq!( + GameCultMediaVideoParityShardRecord::SCHEMA_NAME, + GAMECULT_MEDIA_VIDEO_PARITY_SHARD_SCHEMA + ); + assert_eq!( + GameCultMediaAudioPacketRecord::SCHEMA_NAME, + GAMECULT_MEDIA_AUDIO_PACKET_SCHEMA + ); + assert_eq!( + GameCultMediaReceiverFeedbackRecord::SCHEMA_NAME, + GAMECULT_MEDIA_RECEIVER_FEEDBACK_SCHEMA + ); + + assert_eq!( + GameCultMediaAudioPacketRecord::TYPE, + "gamecult.media_audio_packet" + ); +} diff --git a/packages/cultnet-rs/tests/media_stream_wire.rs b/packages/cultnet-rs/tests/media_stream_wire.rs new file mode 100644 index 00000000..cee31a04 --- /dev/null +++ b/packages/cultnet-rs/tests/media_stream_wire.rs @@ -0,0 +1,217 @@ +//! The envelope, and one proof that removing Muninn's shadow structs changed +//! nothing on the wire. + +use cultnet_rs::{ + GAMECULT_MEDIA_CHANNEL, GameCultMediaAudioPacketRecord, GameCultMediaReceiverFeedbackRecord, + GameCultMediaVideoAccessUnitRecord, GameCultMediaVideoParityShardRecord, + GameCultMediaWireRecord, MediaWireProvenance, decode_media_wire_record, + encode_media_wire_record, +}; +use serde::Serialize; + +fn provenance() -> MediaWireProvenance<'static> { + MediaWireProvenance { + stored_at: "unix:1000", + runtime_id: "raven", + role: "muninn.media", + producer: "muninn", + } +} + +fn access_unit() -> GameCultMediaVideoAccessUnitRecord { + GameCultMediaVideoAccessUnitRecord { + stream_id: "raven-primary-av".to_string(), + session_id: "session-1".to_string(), + frame_id: 4242, + codec: "h264".to_string(), + pts_ticks: 90_000, + duration_ticks: 3_000, + timebase_num: 1, + timebase_den: 90_000, + keyframe: true, + dependency_frame_id: None, + deadline_ticks: 108_000, + chunk_index: 2, + chunk_count: 5, + payload: (0u8..=255).collect(), + } +} + +/// Muninn's `VideoAccessUnitWirePayload`, reproduced exactly as it was. +/// +/// It existed only so `#[serde(with = "serde_bytes")]` could reach the payload, +/// because the `DatabaseEntry` derive ignores serde attributes. Every other +/// field was restated by hand, in order, and had to stay in step with the record +/// by discipline alone. +#[derive(Serialize)] +struct LegacyVideoAccessUnitWirePayload<'a>( + &'a str, + &'a str, + u64, + &'a str, + i64, + u32, + u32, + u32, + bool, + Option, + i64, + u16, + u16, + #[serde(with = "serde_bytes")] &'a [u8], +); + +/// If these differ, deleting the shadow structs was a wire break rather than a +/// simplification, and every existing consumer would need to move in step. +#[test] +fn the_derive_reproduces_the_hand_written_wire_payload_byte_for_byte() { + let record = access_unit(); + let legacy = LegacyVideoAccessUnitWirePayload( + &record.stream_id, + &record.session_id, + record.frame_id, + &record.codec, + record.pts_ticks, + record.duration_ticks, + record.timebase_num, + record.timebase_den, + record.keyframe, + record.dependency_frame_id, + record.deadline_ticks, + record.chunk_index, + record.chunk_count, + &record.payload, + ); + + let from_derive = rmp_serde::to_vec(&record).expect("record encodes"); + let from_shadow = rmp_serde::to_vec(&legacy).expect("shadow struct encodes"); + + assert_eq!( + from_derive, from_shadow, + "the `bytes` attribute must reproduce what serde_bytes produced by hand" + ); +} + +#[test] +fn a_video_record_round_trips_through_the_envelope() { + let record = GameCultMediaWireRecord::Video(access_unit()); + let wire = encode_media_wire_record(&record, provenance()).expect("encodes"); + assert_eq!(decode_media_wire_record(&wire).expect("decodes"), record); +} + +#[test] +fn every_variant_round_trips() { + let variants = vec![ + GameCultMediaWireRecord::Video(access_unit()), + GameCultMediaWireRecord::VideoParity(GameCultMediaVideoParityShardRecord { + stream_id: "s".to_string(), + session_id: "x".to_string(), + frame_id: 1, + codec: "h264".to_string(), + pts_ticks: 0, + duration_ticks: 1, + timebase_num: 1, + timebase_den: 90_000, + keyframe: false, + dependency_frame_id: Some(0), + deadline_ticks: 1, + chunk_count: 4, + parity_index: 1, + parity_count: 2, + chunk_payload_bytes: 848, + last_chunk_payload_bytes: 400, + payload: vec![1, 2, 3], + }), + GameCultMediaWireRecord::Audio(GameCultMediaAudioPacketRecord { + stream_id: "s".to_string(), + session_id: "x".to_string(), + packet_id: 9, + codec: "opus".to_string(), + pts_ticks: 0, + duration_ticks: 960, + timebase_num: 1, + timebase_den: 48_000, + deadline_ticks: 1, + payload: vec![4, 5, 6], + }), + GameCultMediaWireRecord::Feedback(GameCultMediaReceiverFeedbackRecord { + stream_id: "s".to_string(), + session_id: "x".to_string(), + receiver_id: "starfire-obs".to_string(), + highest_decodable_frame_id: Some(41), + missing_frame_ids: vec![42], + late_frame_ids: vec![43], + requested_keyframe: true, + jitter_us: 500, + decode_queue_us: 2_000, + observed_at: "unix:1000".to_string(), + missing_video_chunk_keys: vec!["42:1".to_string()], + }), + ]; + + for record in variants { + let wire = encode_media_wire_record(&record, provenance()).expect("encodes"); + assert_eq!( + decode_media_wire_record(&wire).expect("decodes"), + record, + "variant {} did not survive the envelope", + record.schema_id() + ); + } +} + +/// Record keys address one piece of media within a session. A consumer parses +/// them, so their shape is contract. +#[test] +fn record_keys_distinguish_chunks_parity_and_senders() { + let mut second_chunk = access_unit(); + second_chunk.chunk_index = 3; + + let first = GameCultMediaWireRecord::Video(access_unit()).record_key(); + let second = GameCultMediaWireRecord::Video(second_chunk).record_key(); + assert_ne!(first, second, "chunks of one frame must not share a key"); + assert!(first.contains(":video:"), "got {first}"); +} + +/// A shared envelope must not label another producer's traffic as Muninn's. +#[test] +fn the_producer_namespace_is_the_callers_not_the_envelopes() { + let record = GameCultMediaWireRecord::Video(access_unit()); + let theirs = MediaWireProvenance { + stored_at: "unix:1000", + runtime_id: "nightwing", + role: "brokkr.media", + producer: "brokkr", + }; + + let wire = encode_media_wire_record(&record, theirs).expect("encodes"); + let text = String::from_utf8_lossy(&wire); + assert!(text.contains("brokkr-media:"), "message id names the producer"); + assert!(text.contains("brokkr.media"), "tag names the producer"); + assert!( + !text.contains("muninn"), + "the envelope must not name a producer the caller did not supply" + ); +} + +#[test] +fn provenance_must_be_complete() { + let record = GameCultMediaWireRecord::Video(access_unit()); + for blank in ["stored_at", "runtime_id", "role", "producer"] { + let mut incomplete = provenance(); + match blank { + "stored_at" => incomplete.stored_at = "", + "runtime_id" => incomplete.runtime_id = "", + "role" => incomplete.role = "", + _ => incomplete.producer = "", + } + let error = encode_media_wire_record(&record, incomplete) + .expect_err("empty provenance must be refused"); + assert!(error.to_string().contains(blank), "got {error}"); + } +} + +#[test] +fn the_media_channel_is_named_once() { + assert_eq!(GAMECULT_MEDIA_CHANNEL, "media"); +} diff --git a/packages/cultnet-rs/tests/rudp_server_hub.rs b/packages/cultnet-rs/tests/rudp_server_hub.rs index a8a2c114..201cf3b4 100644 --- a/packages/cultnet-rs/tests/rudp_server_hub.rs +++ b/packages/cultnet-rs/tests/rudp_server_hub.rs @@ -1,3 +1,4 @@ +use std::collections::VecDeque; use std::net::UdpSocket; use std::time::Duration; @@ -314,19 +315,36 @@ fn exact_reliable_acks_clear_more_than_one_ack_window_of_fragments() -> Result<( ordered: true, sequenced: false, now_ms: 1, + reliable_expire_after_ms: None, }, Some(1), )?; - assert_eq!(packets.len(), 40); + // Forty one-byte fragments cannot leave inside one reliable send window. The + // window's worth goes out now and the rest queue until acknowledgements free + // space, which is the crossing this test exists to prove. + assert_eq!(packets.len(), CULTNET_RUDP_RELIABLE_SEND_WINDOW_PACKETS); + assert_eq!( + sender.queued_reliable_packet_count(), + 40 - CULTNET_RUDP_RELIABLE_SEND_WINDOW_PACKETS + ); + + let mut wire = VecDeque::from(packets); let mut delivered = Vec::new(); - for packet in &packets { - let result = receiver.receive(packet, 2)?; - delivered.extend(result.delivered); - sender.receive(&receiver.create_ack_for(packet.sequence), 3)?; + while !wire.is_empty() { + let admitted = wire.len(); + for _ in 0..admitted { + let packet = wire.pop_front().expect("admitted packet remains queued"); + let received = receiver.receive(&packet, 2)?; + delivered.extend(received.delivered); + } + let acknowledged = sender.receive(&receiver.create_ack(), 3)?; + wire.extend(acknowledged.ready_to_send); } + assert_eq!(delivered.len(), 1); assert_eq!(delivered[0].payload, vec![0x5a; 40]); assert!(sender.pending_reliable_sequences().is_empty()); + assert_eq!(sender.outstanding_reliable_packet_count(), 0); Ok(()) } @@ -358,6 +376,7 @@ fn session_and_hub_reject_configured_memory_bounds() -> Result<()> { ordered: true, sequenced: false, now_ms: 1, + reliable_expire_after_ms: None, }, Some(2), )?; @@ -370,6 +389,7 @@ fn session_and_hub_reject_configured_memory_bounds() -> Result<()> { ordered: true, sequenced: false, now_ms: 2, + reliable_expire_after_ms: None, }; let first_set = sender.send_many( "schema", @@ -379,7 +399,12 @@ fn session_and_hub_reject_configured_memory_bounds() -> Result<()> { )?; let second_set = sender.send_many("schema", vec![5, 6, 7, 8], fragment_options, Some(2))?; receiver.receive(&first_set[0], 2)?; - assert!(receiver.receive(&second_set[0], 2).is_err()); + // The fragment-set bound evicts the stalest incomplete set rather than + // refusing the packet: a receiver that dies at its bound under loss is a + // fuse, not a limit (measured 2026-09-10, see tests/cultnet.rs). + assert_eq!(receiver.fragment_sets_evicted(), 0); + receiver.receive(&second_set[0], 2)?; + assert_eq!(receiver.fragment_sets_evicted(), 1, "the first set was evicted for the second"); let server_socket = socket()?; let server_addr = server_socket.local_addr()?; @@ -429,6 +454,7 @@ fn replay_history_is_bounded_and_idle_hub_sessions_expire() -> Result<()> { ordered: false, sequenced: false, now_ms: sequence, + reliable_expire_after_ms: None, }, )?; if oldest.is_none() { @@ -436,10 +462,43 @@ fn replay_history_is_bounded_and_idle_hub_sessions_expire() -> Result<()> { } assert_eq!(receiver.receive(&packet, sequence)?.delivered.len(), 1); } - assert!( + // Separating the sequence domains moved duplicate suppression onto the + // reliable domain, so a replayed lossy datagram is delivered again rather + // than silently dropped. Lossy traffic is remembered nowhere, which is what + // bounds its history. + assert_eq!( receiver .receive(&oldest.expect("oldest packet"), 4_101)? .delivered + .len(), + 1 + ); + + // The bounded replay window still guards the reliable domain, and nothing + // else asserts it once the lossy path stopped being remembered. + let mut oldest_reliable = None; + for step in 0..4_100u64 { + let packet = sender.send( + "schema", + vec![(step % 251) as u8], + CultNetRudpSendOptions { + reliable: true, + ordered: false, + sequenced: false, + now_ms: step, + reliable_expire_after_ms: None, + }, + )?; + if oldest_reliable.is_none() { + oldest_reliable = Some(packet.clone()); + } + assert_eq!(receiver.receive(&packet, step)?.delivered.len(), 1); + sender.receive(&receiver.create_ack_for(packet.sequence), step)?; + } + assert!( + receiver + .receive(&oldest_reliable.expect("oldest reliable packet"), 4_101)? + .delivered .is_empty() ); @@ -457,3 +516,59 @@ fn replay_history_is_bounded_and_idle_hub_sessions_expire() -> Result<()> { assert!(hub.sessions().is_empty()); Ok(()) } + +/// A packet that carries no event — an acknowledgement, a ping — must not +/// make the reader report "nothing waiting" while frames sit behind it. +/// A caller that polls once per send fell behind its peer's acknowledgements +/// this way and a reliable channel resent everything. +#[test] +fn a_reader_does_not_stop_at_a_packet_that_carries_no_event() -> Result<()> { + let mut hub = CultNetRudpServerHub::new(CultNetRudpServerHubOptions::new( + "hub", + socket()?, + CONNECTION_ID, + ))?; + let mut peer = client(hub.local_addr()?)?; + let session = connect(&mut hub, &mut peer, b"evidence")?; + + // Hub side: a ping (reply, no event) ahead of a reliable frame. + peer.ping(b"keepalive".to_vec())?; + peer.send("schema", b"after the ping".to_vec())?; + std::thread::sleep(Duration::from_millis(20)); + match hub.receive_event_once()? { + Some(CultNetRudpServerEvent::Frame { frame, .. }) => { + assert_eq!(frame.payload, b"after the ping"); + } + other => panic!("the first read must reach the frame behind the ping, got {other:?}"), + } + + // Client side: the hub's acknowledgement of that frame is queued ahead of + // two unreliable frames; the client must reach both without a "nothing" + // in between, and must not acknowledge unreliable frames back. + let mut unreliable_hub_options = + CultNetRudpServerHubOptions::new("hub-unreliable", socket()?, CONNECTION_ID); + unreliable_hub_options.media_delivery = Some(CultNetTransportDelivery::Unreliable); + let mut media_hub = CultNetRudpServerHub::new(unreliable_hub_options)?; + let mut media_client = client(media_hub.local_addr()?)?; + let media_session = connect(&mut media_hub, &mut media_client, b"media")?; + media_client.send("schema", b"reliable, will be acked".to_vec())?; + std::thread::sleep(Duration::from_millis(20)); + while media_hub.receive_event_once()?.is_some() {} + media_hub.send(&media_session, "media", b"one".to_vec())?; + media_hub.send(&media_session, "media", b"two".to_vec())?; + std::thread::sleep(Duration::from_millis(20)); + let before = media_hub.stats().bytes_received; + let first = media_client.receive_once()?.expect("the first read reaches the frame behind the ack"); + let second = media_client.receive_once()?.expect("the second frame follows"); + assert_eq!((first.payload, second.payload), (b"one".to_vec(), b"two".to_vec())); + assert!(media_client.receive_once()?.is_none()); + std::thread::sleep(Duration::from_millis(20)); + while media_hub.receive_event_once()?.is_some() {} + assert_eq!( + media_hub.stats().bytes_received, + before, + "unreliable frames are not acknowledged; the hub read nothing back" + ); + let _ = session; + Ok(()) +} diff --git a/packages/cultnet-ts/README.md b/packages/cultnet-ts/README.md index 5ff66e7e..75e77b88 100644 --- a/packages/cultnet-ts/README.md +++ b/packages/cultnet-ts/README.md @@ -195,7 +195,7 @@ the helper path. ## Quick Example ```ts -import { CultCache, defineDocumentType } from "cultcache-ts"; +import { CultCache, defineDocumentType } from "@gamecult/cultcache-ts"; import { CultNetDocumentRegistry, createTcpFramedCultNetPeer, diff --git a/packages/cultnet-ts/contracts/cultmesh.session-accepted.schema.json b/packages/cultnet-ts/contracts/cultmesh.session-accepted.schema.json new file mode 100644 index 00000000..aca1ff2b --- /dev/null +++ b/packages/cultnet-ts/contracts/cultmesh.session-accepted.schema.json @@ -0,0 +1,21 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/GameCult/cultnet-ts/contracts/cultmesh.session-accepted.schema.json", + "title": "CultMesh Session Accepted", + "type": "object", + "required": ["schemaVersion", "messageId", "accepted", "verseId", "authorityRuntimeId", "protocolId", "routeGeneration", "clientNonce", "providerKeyId", "providerSignature"], + "additionalProperties": false, + "properties": { + "schemaVersion": { "const": "cultmesh.session_accepted.v2" }, + "messageId": { "type": "string", "minLength": 1 }, + "accepted": { "type": "boolean" }, + "verseId": { "type": "string", "minLength": 1 }, + "authorityRuntimeId": { "type": "string", "minLength": 1 }, + "protocolId": { "type": "string", "minLength": 1 }, + "routeGeneration": { "type": "string" }, + "clientNonce": { "type": "string" }, + "providerKeyId": { "type": "string" }, + "providerSignature": { "type": "string" }, + "error": { "type": "string", "minLength": 1 } + } +} diff --git a/packages/cultnet-ts/contracts/cultmesh.session-open.schema.json b/packages/cultnet-ts/contracts/cultmesh.session-open.schema.json new file mode 100644 index 00000000..937f3e28 --- /dev/null +++ b/packages/cultnet-ts/contracts/cultmesh.session-open.schema.json @@ -0,0 +1,18 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/GameCult/cultnet-ts/contracts/cultmesh.session-open.schema.json", + "title": "CultMesh Session Open", + "type": "object", + "required": ["schemaVersion", "messageId", "sourceRuntimeId", "verseId", "authorityRuntimeId", "protocolId", "routeGeneration", "clientNonce"], + "additionalProperties": false, + "properties": { + "schemaVersion": { "const": "cultmesh.session_open.v2" }, + "messageId": { "type": "string", "minLength": 1 }, + "sourceRuntimeId": { "type": "string", "minLength": 1 }, + "verseId": { "type": "string", "minLength": 1 }, + "authorityRuntimeId": { "type": "string", "minLength": 1 }, + "protocolId": { "type": "string", "minLength": 1 }, + "routeGeneration": { "type": "string" }, + "clientNonce": { "type": "string", "minLength": 1 } + } +} diff --git a/packages/cultnet-ts/contracts/cultmesh.verse-catalog-request.schema.json b/packages/cultnet-ts/contracts/cultmesh.verse-catalog-request.schema.json new file mode 100644 index 00000000..e99b9dea --- /dev/null +++ b/packages/cultnet-ts/contracts/cultmesh.verse-catalog-request.schema.json @@ -0,0 +1,14 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/GameCult/cultnet-ts/contracts/cultmesh.verse-catalog-request.schema.json", + "title": "CultMesh Verse Catalog Request", + "type": "object", + "required": ["schemaVersion", "messageId"], + "additionalProperties": false, + "properties": { + "schemaVersion": { "const": "cultmesh.verse_catalog_request.v0" }, + "messageId": { "type": "string", "minLength": 1 }, + "verseIds": { "type": "array", "items": { "type": "string", "minLength": 1 }, "uniqueItems": true }, + "transportVersion": { "type": "string", "minLength": 1 } + } +} diff --git a/packages/cultnet-ts/contracts/cultmesh.verse-catalog-response.schema.json b/packages/cultnet-ts/contracts/cultmesh.verse-catalog-response.schema.json new file mode 100644 index 00000000..464908ad --- /dev/null +++ b/packages/cultnet-ts/contracts/cultmesh.verse-catalog-response.schema.json @@ -0,0 +1,72 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/GameCult/cultnet-ts/contracts/cultmesh.verse-catalog-response.schema.json", + "title": "CultMesh Verse Catalog Response", + "type": "object", + "required": ["schemaVersion", "messageId", "verses"], + "additionalProperties": false, + "$defs": { + "compatibility": { + "type": "object", + "required": ["transportVersion", "rulesHash", "compatibleVerseIds", "requiredPluginIds", "optionalPluginIds"], + "additionalProperties": false, + "properties": { + "transportVersion": { "type": "string", "minLength": 1 }, + "rulesHash": { "type": "string", "minLength": 1 }, + "compatibleVerseIds": { "type": "array", "items": { "type": "string", "minLength": 1 }, "uniqueItems": true }, + "requiredPluginIds": { "type": "array", "items": { "type": "string", "minLength": 1 }, "uniqueItems": true }, + "optionalPluginIds": { "type": "array", "items": { "type": "string", "minLength": 1 }, "uniqueItems": true } + } + }, + "authorityRoute": { + "type": "object", + "required": ["authorityRuntimeId", "endpoint", "protocolIds", "priority", "generation"], + "additionalProperties": false, + "properties": { + "authorityRuntimeId": { "type": "string", "minLength": 1 }, + "endpoint": { "type": "string", "minLength": 1 }, + "protocolIds": { "type": "array", "items": { "type": "string", "minLength": 1 }, "uniqueItems": true }, + "priority": { "type": "integer", "minimum": 0 }, + "generation": { "type": "string", "minLength": 1 }, + "certificate": { + "anyOf": [{ "$ref": "#/$defs/routeCertificate" }, { "type": "null" }] + } + } + }, + "routeCertificate": { + "type": "object", + "required": ["providerKeyId", "providerPublicKeyX", "providerPublicKeyY", "odinKeyId", "issuedAtUnixMilliseconds", "expiresAtUnixMilliseconds", "signature"], + "additionalProperties": false, + "properties": { + "providerKeyId": { "type": "string", "minLength": 1 }, + "providerPublicKeyX": { "type": "string", "minLength": 1 }, + "providerPublicKeyY": { "type": "string", "minLength": 1 }, + "odinKeyId": { "type": "string", "minLength": 1 }, + "issuedAtUnixMilliseconds": { "type": "integer", "minimum": 0 }, + "expiresAtUnixMilliseconds": { "type": "integer", "minimum": 1 }, + "signature": { "type": "string", "minLength": 1 } + } + }, + "verse": { + "type": "object", + "required": ["verseId", "displayName", "authorityModel", "compatibility", "discoveryEndpoints", "authorityRuntimeIds"], + "additionalProperties": false, + "properties": { + "verseId": { "type": "string", "minLength": 1 }, + "displayName": { "type": "string", "minLength": 1 }, + "authorityModel": { "type": "string", "minLength": 1 }, + "compatibility": { "$ref": "#/$defs/compatibility" }, + "discoveryEndpoints": { "type": "array", "items": { "type": "string", "minLength": 1 }, "uniqueItems": true }, + "authorityRuntimeIds": { "type": "array", "items": { "type": "string", "minLength": 1 }, "uniqueItems": true }, + "authorityRoutes": { "type": "array", "items": { "$ref": "#/$defs/authorityRoute" } }, + "parentVerseId": { "type": "string", "minLength": 1 }, + "description": { "type": "string" } + } + } + }, + "properties": { + "schemaVersion": { "const": "cultmesh.verse_catalog_response.v0" }, + "messageId": { "type": "string", "minLength": 1 }, + "verses": { "type": "array", "items": { "$ref": "#/$defs/verse" } } + } +} diff --git a/packages/cultnet-ts/contracts/cultnet.database-change-raw.schema.json b/packages/cultnet-ts/contracts/cultnet.database-change-raw.schema.json new file mode 100644 index 00000000..dd650fa0 --- /dev/null +++ b/packages/cultnet-ts/contracts/cultnet.database-change-raw.schema.json @@ -0,0 +1,24 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/GameCult/cultnet-ts/contracts/cultnet.database-change-raw.schema.json", + "title": "CultNet Raw Database Change Message", + "type": "object", + "required": ["schemaVersion", "messageId", "subscriptionId", "changeKind"], + "additionalProperties": false, + "properties": { + "schemaVersion": { "const": "cultnet.database_change_raw.v0" }, + "messageId": { "type": "string", "minLength": 1 }, + "subscriptionId": { "type": "string", "minLength": 1 }, + "changeKind": { "enum": ["added", "updated", "removed"] }, + "document": { "$ref": "./cultnet.raw-document-record.schema.json" }, + "recordKey": { "type": "string", "minLength": 1 }, + "schemaId": { "type": "string", "minLength": 1 } + }, + "allOf": [ + { + "if": { "properties": { "changeKind": { "const": "removed" } } }, + "then": { "required": ["recordKey", "schemaId"] }, + "else": { "required": ["document"] } + } + ] +} diff --git a/packages/cultnet-ts/contracts/cultnet.database-subscribe.schema.json b/packages/cultnet-ts/contracts/cultnet.database-subscribe.schema.json new file mode 100644 index 00000000..08304199 --- /dev/null +++ b/packages/cultnet-ts/contracts/cultnet.database-subscribe.schema.json @@ -0,0 +1,19 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/GameCult/cultnet-ts/contracts/cultnet.database-subscribe.schema.json", + "title": "CultNet Database Subscribe Message", + "type": "object", + "required": ["schemaVersion", "messageId", "subscriptionId", "includeSnapshot"], + "additionalProperties": false, + "properties": { + "schemaVersion": { "const": "cultnet.database_subscribe.v0" }, + "messageId": { "type": "string", "minLength": 1 }, + "subscriptionId": { "type": "string", "minLength": 1 }, + "schemaIds": { "type": "array", "items": { "type": "string", "minLength": 1 } }, + "recordKeys": { "type": "array", "items": { "type": "string", "minLength": 1 } }, + "includeSnapshot": { "type": "boolean" }, + "consumerRuntimeId": { "type": "string", "minLength": 1 }, + "bodyIds": { "type": "array", "items": { "type": "string", "minLength": 1 } }, + "supportedBodyTransports": { "type": "array", "items": { "type": "string", "minLength": 1 } } + } +} diff --git a/packages/cultnet-ts/contracts/cultnet.database-unsubscribe.schema.json b/packages/cultnet-ts/contracts/cultnet.database-unsubscribe.schema.json new file mode 100644 index 00000000..7d4b0855 --- /dev/null +++ b/packages/cultnet-ts/contracts/cultnet.database-unsubscribe.schema.json @@ -0,0 +1,13 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/GameCult/cultnet-ts/contracts/cultnet.database-unsubscribe.schema.json", + "title": "CultNet Database Unsubscribe Message", + "type": "object", + "required": ["schemaVersion", "messageId", "subscriptionId"], + "additionalProperties": false, + "properties": { + "schemaVersion": { "const": "cultnet.database_unsubscribe.v0" }, + "messageId": { "type": "string", "minLength": 1 }, + "subscriptionId": { "type": "string", "minLength": 1 } + } +} diff --git a/packages/cultnet-ts/package.json b/packages/cultnet-ts/package.json index f0a3e396..2f4b74f1 100644 --- a/packages/cultnet-ts/package.json +++ b/packages/cultnet-ts/package.json @@ -1,6 +1,6 @@ { "name": "cultnet-ts", - "version": "0.1.1", + "version": "0.2.0", "description": "CultNet-style typed direct-pipe messaging and state replication for TypeScript runtimes.", "main": "dist/index.js", "types": "dist/index.d.ts", @@ -9,6 +9,11 @@ "types": "./dist/index.d.ts", "require": "./dist/index.js", "default": "./dist/index.js" + }, + "./contracts": { + "types": "./dist/contracts.d.ts", + "require": "./dist/contracts.js", + "default": "./dist/contracts.js" } }, "files": [ @@ -21,7 +26,7 @@ "prepack": "node tools/generate-swarm-contracts.mjs && node ../../node_modules/typescript/bin/tsc -p tsconfig.json", "typecheck": "node tools/generate-swarm-contracts.mjs && node ../../node_modules/typescript/bin/tsc -p tsconfig.json --pretty false", "test": "node tools/generate-swarm-contracts.mjs && node ../../node_modules/typescript/bin/tsc -p tsconfig.json && node ../../node_modules/typescript/bin/tsc -p tsconfig.test.json && node --test dist-test/test/*.test.js", - "test:interop": "node tools/generate-swarm-contracts.mjs && node ../../node_modules/typescript/bin/tsc -p tsconfig.json && node ../../node_modules/typescript/bin/tsc -p tsconfig.test.json && node --test dist-test/test/interop/*.test.js" + "test:interop": "node tools/generate-swarm-contracts.mjs && node ../../node_modules/typescript/bin/tsc -p tsconfig.json && node ../../node_modules/typescript/bin/tsc -p tsconfig.test.json && node --test --test-force-exit dist-test/test/interop/*.test.js" }, "keywords": [ "messagepack", @@ -48,7 +53,7 @@ "dependencies": { "@msgpack/msgpack": "^3.1.3", "ajv": "^8.17.1", - "cultcache-ts": "^0.13.5" + "@gamecult/cultcache-ts": "^0.14.0" }, "devDependencies": { "@types/node": "^24.0.0", diff --git a/packages/cultnet-ts/src/contracts.ts b/packages/cultnet-ts/src/contracts.ts index 8795b06a..c71cdcc1 100644 --- a/packages/cultnet-ts/src/contracts.ts +++ b/packages/cultnet-ts/src/contracts.ts @@ -18,6 +18,9 @@ import transportProfileSchema from "../contracts/cultnet.transport-profile.schem import snapshotRequestSchema from "../contracts/cultnet.snapshot-request.schema.json"; import snapshotResponseSchema from "../contracts/cultnet.snapshot-response.schema.json"; import snapshotResponseRawSchema from "../contracts/cultnet.snapshot-response-raw.schema.json"; +import databaseSubscribeSchema from "../contracts/cultnet.database-subscribe.schema.json"; +import databaseUnsubscribeSchema from "../contracts/cultnet.database-unsubscribe.schema.json"; +import databaseChangeRawSchema from "../contracts/cultnet.database-change-raw.schema.json"; import schemaDescriptorSchema from "../contracts/cultnet.schema-descriptor.schema.json"; import schemaCatalogRequestSchema from "../contracts/cultnet.schema-catalog-request.schema.json"; import schemaCatalogResponseSchema from "../contracts/cultnet.schema-catalog-response.schema.json"; @@ -25,6 +28,10 @@ import shardCatalogRequestSchema from "../contracts/cultnet.shard-catalog-reques import shardCatalogResponseSchema from "../contracts/cultnet.shard-catalog-response.schema.json"; import operationRequestSchema from "../contracts/cultnet.operation-request.schema.json"; import operationResponseSchema from "../contracts/cultnet.operation-response.schema.json"; +import verseCatalogRequestSchema from "../contracts/cultmesh.verse-catalog-request.schema.json"; +import verseCatalogResponseSchema from "../contracts/cultmesh.verse-catalog-response.schema.json"; +import sessionOpenSchema from "../contracts/cultmesh.session-open.schema.json"; +import sessionAcceptedSchema from "../contracts/cultmesh.session-accepted.schema.json"; import ghostlightAgentStateSchema from "../contracts/ghostlight.agent-state.schema.json"; export type CultNetWireContract = "cultnet.schema.v0" | "gamecult.networking.v0"; @@ -44,12 +51,19 @@ export type CultNetSchemaVersion = | "cultnet.snapshot_request.v0" | "cultnet.snapshot_response.v0" | "cultnet.snapshot_response_raw.v0" + | "cultnet.database_subscribe.v0" + | "cultnet.database_unsubscribe.v0" + | "cultnet.database_change_raw.v0" | "cultnet.schema_catalog_request.v0" | "cultnet.schema_catalog_response.v0" | "cultnet.shard_catalog_request.v0" | "cultnet.shard_catalog_response.v0" | "cultnet.operation_request.v0" - | "cultnet.operation_response.v0"; + | "cultnet.operation_response.v0" + | "cultmesh.verse_catalog_request.v0" + | "cultmesh.verse_catalog_response.v0" + | "cultmesh.session_open.v2" + | "cultmesh.session_accepted.v2"; export type CultNetSchemaKind = "wire_message" | "document_payload" | "shared_contract"; export type CultNetRawPayloadEncoding = "messagepack"; @@ -180,6 +194,9 @@ export interface CultNetDocumentDeleteMessage { export interface CultNetRawDocumentRecord { schemaId: string; + schemaName?: string; + schemaVersion?: string; + schemaContentHash?: string; recordKey: string; storedAt: string; payloadEncoding: CultNetRawPayloadEncoding; @@ -217,6 +234,34 @@ export interface CultNetSnapshotResponseRawMessage { documents: CultNetRawDocumentRecord[]; } +export interface CultNetDatabaseSubscribeMessage { + schemaVersion: "cultnet.database_subscribe.v0"; + messageId: string; + subscriptionId: string; + schemaIds?: string[]; + recordKeys?: string[]; + includeSnapshot: boolean; + consumerRuntimeId?: string; + bodyIds?: string[]; + supportedBodyTransports?: string[]; +} + +export interface CultNetDatabaseUnsubscribeMessage { + schemaVersion: "cultnet.database_unsubscribe.v0"; + messageId: string; + subscriptionId: string; +} + +export interface CultNetDatabaseChangeRawMessage { + schemaVersion: "cultnet.database_change_raw.v0"; + messageId: string; + subscriptionId: string; + changeKind: "added" | "updated" | "removed"; + document?: CultNetRawDocumentRecord; + recordKey?: string; + schemaId?: string; +} + export interface CultNetSchemaDescriptor { schemaId: string; kind: CultNetSchemaKind; @@ -294,6 +339,83 @@ export interface CultNetOperationResponseMessage { sourceRuntimeId?: string; } +export interface CultMeshVerseCompatibilityMessage { + transportVersion: string; + rulesHash: string; + compatibleVerseIds: string[]; + requiredPluginIds: string[]; + optionalPluginIds: string[]; +} + +export interface CultMeshVerseDescriptorMessage { + verseId: string; + displayName: string; + authorityModel: string; + compatibility: CultMeshVerseCompatibilityMessage; + discoveryEndpoints: string[]; + authorityRuntimeIds: string[]; + authorityRoutes?: CultMeshAuthorityRouteMessage[]; + parentVerseId?: string; + description?: string; +} + +export interface CultMeshAuthorityRouteMessage { + authorityRuntimeId: string; + endpoint: string; + protocolIds: string[]; + priority: number; + generation: string; + certificate?: CultMeshRouteCertificateMessage; +} + +export interface CultMeshRouteCertificateMessage { + providerKeyId: string; + providerPublicKeyX: string; + providerPublicKeyY: string; + odinKeyId: string; + issuedAtUnixMilliseconds: number; + expiresAtUnixMilliseconds: number; + signature: string; +} + +export interface CultMeshSessionOpenMessage { + schemaVersion: "cultmesh.session_open.v2"; + messageId: string; + sourceRuntimeId: string; + verseId: string; + authorityRuntimeId: string; + protocolId: string; + routeGeneration: string; + clientNonce: string; +} + +export interface CultMeshSessionAcceptedMessage { + schemaVersion: "cultmesh.session_accepted.v2"; + messageId: string; + accepted: boolean; + verseId: string; + authorityRuntimeId: string; + protocolId: string; + routeGeneration: string; + clientNonce: string; + providerKeyId: string; + providerSignature: string; + error?: string; +} + +export interface CultMeshVerseCatalogRequestMessage { + schemaVersion: "cultmesh.verse_catalog_request.v0"; + messageId: string; + verseIds?: string[]; + transportVersion?: string; +} + +export interface CultMeshVerseCatalogResponseMessage { + schemaVersion: "cultmesh.verse_catalog_response.v0"; + messageId: string; + verses: CultMeshVerseDescriptorMessage[]; +} + export interface GhostlightPressure { pressure_id: string; label: string; @@ -535,12 +657,19 @@ export type CultNetMessage = | CultNetSnapshotRequestMessage | CultNetSnapshotResponseMessage | CultNetSnapshotResponseRawMessage + | CultNetDatabaseSubscribeMessage + | CultNetDatabaseUnsubscribeMessage + | CultNetDatabaseChangeRawMessage | CultNetSchemaCatalogRequestMessage | CultNetSchemaCatalogResponseMessage | CultNetShardCatalogRequestMessage | CultNetShardCatalogResponseMessage | CultNetOperationRequestMessage - | CultNetOperationResponseMessage; + | CultNetOperationResponseMessage + | CultMeshVerseCatalogRequestMessage + | CultMeshVerseCatalogResponseMessage + | CultMeshSessionOpenMessage + | CultMeshSessionAcceptedMessage; const ajv = new Ajv2020({ allErrors: true, @@ -562,12 +691,19 @@ const CULTNET_MESSAGE_SCHEMAS = [ snapshotRequestSchema, snapshotResponseSchema, snapshotResponseRawSchema, + databaseSubscribeSchema, + databaseUnsubscribeSchema, + databaseChangeRawSchema, schemaCatalogRequestSchema, schemaCatalogResponseSchema, shardCatalogRequestSchema, shardCatalogResponseSchema, operationRequestSchema, operationResponseSchema, + verseCatalogRequestSchema, + verseCatalogResponseSchema, + sessionOpenSchema, + sessionAcceptedSchema, ] as const; const cultNetValidators = new Map(); @@ -620,6 +756,11 @@ export function parseCultNetMessage(input: unknown, wireContract: CultNetWireCon return input as CultNetSnapshotResponseRawMessage; } + if (schemaVersion === "cultnet.database_change_raw.v0") { + validateDatabaseChangeRawMessage(input); + return input as CultNetDatabaseChangeRawMessage; + } + if (!validator(input)) { const keys = Object.keys(input as Record).join(","); throw new Error(`${renderValidationErrors(schemaVersion, validator)}; keys=${keys}`); @@ -906,8 +1047,11 @@ function encodeLegacySchemaDescriptor(schema: CultNetSchemaDescriptor): Record character.charCodeAt(0)); } function validateDocumentPutRawMessage(input: unknown): asserts input is CultNetDocumentPutRawMessage { @@ -951,9 +1101,39 @@ function validateSnapshotResponseRawMessage( }); } +function validateDatabaseChangeRawMessage( + input: unknown, +): asserts input is CultNetDatabaseChangeRawMessage { + const candidate = validateBinaryMessageRoot(input, "cultnet.database_change_raw.v0"); + requireNonEmptyString(candidate.subscriptionId, "/subscriptionId"); + if (candidate.changeKind !== "added" && + candidate.changeKind !== "updated" && + candidate.changeKind !== "removed") { + throw new Error( + "Validation failed for cultnet.database_change_raw.v0: /changeKind: must be added, updated, or removed", + ); + } + + if (candidate.changeKind === "removed") { + requireNonEmptyString(candidate.recordKey, "/recordKey"); + requireNonEmptyString(candidate.schemaId, "/schemaId"); + return; + } + + if (!("document" in candidate)) { + throw new Error( + "Validation failed for cultnet.database_change_raw.v0: /document: is required for added or updated changes", + ); + } + validateRawDocumentRecord(candidate.document, "/document"); +} + function validateBinaryMessageRoot( input: unknown, - schemaVersion: "cultnet.document_put_raw.v0" | "cultnet.snapshot_response_raw.v0", + schemaVersion: + | "cultnet.document_put_raw.v0" + | "cultnet.snapshot_response_raw.v0" + | "cultnet.database_change_raw.v0", ): Record { if (!input || typeof input !== "object") { throw new Error(`Validation failed for ${schemaVersion}: /: must be an object`); @@ -980,6 +1160,9 @@ function validateRawDocumentRecord( const candidate = input as Record; requireNonEmptyString(candidate.schemaId, `${instancePath}/schemaId`); + requireOptionalNonEmptyString(candidate.schemaName, `${instancePath}/schemaName`); + requireOptionalNonEmptyString(candidate.schemaVersion, `${instancePath}/schemaVersion`); + requireOptionalNonEmptyString(candidate.schemaContentHash, `${instancePath}/schemaContentHash`); requireNonEmptyString(candidate.recordKey, `${instancePath}/recordKey`); requireNonEmptyString(candidate.storedAt, `${instancePath}/storedAt`); if (candidate.payloadEncoding !== "messagepack") { @@ -1043,6 +1226,34 @@ function normalizeCultNetOptionalNulls( acceptLegacyPropertyName(candidate, "ShardEpoch", "shardEpoch"); stripNullProperties(candidate, ["schemaIds", "recordKeys", "shardId", "shardEpoch"]); return; + case "cultnet.database_subscribe.v0": + stripNullProperties(candidate, [ + "schemaIds", + "recordKeys", + "consumerRuntimeId", + "bodyIds", + "supportedBodyTransports", + ]); + return; + case "cultnet.operation_request.v0": + stripNullProperties(candidate, ["payloadEncoding", "sourceRuntimeId", "targetRuntimeId"]); + return; + case "cultnet.operation_response.v0": + stripNullProperties(candidate, ["payloadEncoding", "diagnostics", "sourceRuntimeId"]); + return; + case "cultmesh.verse_catalog_request.v0": + stripNullProperties(candidate, ["verseIds", "transportVersion"]); + return; + case "cultmesh.verse_catalog_response.v0": + for (const verse of Array.isArray(candidate.verses) ? candidate.verses : []) { + if (verse && typeof verse === "object") { + stripNullProperties(verse as Record, ["parentVerseId", "description"]); + } + } + return; + case "cultmesh.session_accepted.v2": + stripNullProperties(candidate, ["error"]); + return; case "cultnet.schema_catalog_request.v0": stripNullProperties(candidate, ["schemaIds", "kinds"]); return; @@ -1092,6 +1303,9 @@ export const cultNetSchemas = { snapshotRequestSchema, snapshotResponseSchema, snapshotResponseRawSchema, + databaseSubscribeSchema, + databaseUnsubscribeSchema, + databaseChangeRawSchema, schemaDescriptorSchema, schemaCatalogRequestSchema, schemaCatalogResponseSchema, diff --git a/packages/cultnet-ts/src/index.ts b/packages/cultnet-ts/src/index.ts index cd602952..140526c5 100644 --- a/packages/cultnet-ts/src/index.ts +++ b/packages/cultnet-ts/src/index.ts @@ -8,6 +8,9 @@ export { type CultNetDocumentPutMessage, type CultNetDocumentPutRawMessage, type CultNetDocumentRecord, + type CultNetDatabaseSubscribeMessage, + type CultNetDatabaseUnsubscribeMessage, + type CultNetDatabaseChangeRawMessage, type CultNetRawDocumentRecord, type CultNetRawPayloadEncoding, type CultNetErrorMessage, @@ -80,6 +83,7 @@ export { createRudpTransportProfile, decodeRudpPacket, encodeRudpPacket, + CULTNET_RUDP_RELIABLE_SEND_WINDOW_PACKETS, CultNetRudpReconnectLoop, CultNetRudpSession, CultNetRudpSocketTransportConnection, @@ -140,3 +144,33 @@ export { validateGhostlightAgentStateGenerated, type GhostlightAgentStateShape, } from "./generated/swarm-contracts.generated"; +export { + CULTNET_RUDP_PROTOCOL_ID, + createSignedDaemonHealthPublisher, + parseEndpoint as parseDaemonHealthEndpoint, + publishSignedDaemonHealth, + signedDaemonHealthPayload, + signedDaemonHealthSigningMessage, + type DaemonHealth, + type SignedDaemonHealthContract, + type SignedDaemonHealthPublisher, + type SignedDaemonHealthPublisherOptions, +} from "./signed-daemon-health"; + +export { + GAMECULT_RUNTIME_ACTIVATION_PROOF_SIGNING_PURPOSE, + GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA, + GAMECULT_RUNTIME_PRESENCE_HEALTH_SIGNING_PURPOSE, + GAMECULT_RUNTIME_PRESENCE_IDENTITY_NAME, + IDUNN_RUNTIME_ACTIVATION_CREDENTIAL_NAME, + RUNTIME_PRESENCE_FIELD_COUNT, + RUNTIME_PRESENCE_SLOT, + encodeRuntimePresenceHealth, + runtimePresenceActivationSigningMessage, + runtimePresenceProofPayload, + runtimePresenceProviderSigningMessage, + validateRuntimePresence, + type GameCultRuntimeCapability, + type RuntimePresenceHealth, + type RuntimePresenceState, +} from "./runtime-presence-health"; diff --git a/packages/cultnet-ts/src/json-schema-contract.ts b/packages/cultnet-ts/src/json-schema-contract.ts index b97a6762..4d8fb85d 100644 --- a/packages/cultnet-ts/src/json-schema-contract.ts +++ b/packages/cultnet-ts/src/json-schema-contract.ts @@ -1,5 +1,5 @@ import Ajv2020, { type ErrorObject, type ValidateFunction } from "ajv/dist/2020"; -import type { CultCacheSchema } from "cultcache-ts"; +import type { CultCacheSchema } from "@gamecult/cultcache-ts"; export interface JsonSchemaContract extends CultCacheSchema { readonly schemaId: string; diff --git a/packages/cultnet-ts/src/operation-service.ts b/packages/cultnet-ts/src/operation-service.ts index 362e4a06..ee13457f 100644 --- a/packages/cultnet-ts/src/operation-service.ts +++ b/packages/cultnet-ts/src/operation-service.ts @@ -145,6 +145,7 @@ async function handleServerDatagram( if (!peer) return; const result = peer.session.receive(packet, Date.now()); if (result.reply) sendPacket(remote, result.reply); + for (const ready of result.readyToSend ?? []) sendPacket(remote, ready); if (result.disconnected) { sessions.delete(key); return; @@ -162,7 +163,9 @@ async function handleServerDatagram( maxFragmentBytes: options.maxFragmentBytes ?? 2048, })) sendPacket(remote, responsePacket); } - if (packet.packetType === "data" || result.delivered.length > 0) sendPacket(remote, peer.session.createAck()); + if (packet.packetType === "data" || result.delivered.length > 0) { + sendPacket(remote, peer.session.createAckForReceived(packet.sequence)); + } } function parseRudpEndpoint(endpoint: string): { host: string; port: number } { diff --git a/packages/cultnet-ts/src/peer.ts b/packages/cultnet-ts/src/peer.ts index 816fcfd6..0e54ee58 100644 --- a/packages/cultnet-ts/src/peer.ts +++ b/packages/cultnet-ts/src/peer.ts @@ -106,6 +106,10 @@ export class CultNetPeer extends EventEmitter { this.#stream?.write(encodeFrame(payload)); } + async flush(timeoutMs?: number): Promise { + await this.#transport?.flush?.(timeoutMs); + } + sendHello(message: CultNetHelloMessage): void { this.send(message); } diff --git a/packages/cultnet-ts/src/replication.ts b/packages/cultnet-ts/src/replication.ts index d5706bea..77dab70f 100644 --- a/packages/cultnet-ts/src/replication.ts +++ b/packages/cultnet-ts/src/replication.ts @@ -6,7 +6,7 @@ import type { CultCacheDocumentFormatter, CultCacheDocumentValue, CultCacheEnvelope, -} from "cultcache-ts"; +} from "@gamecult/cultcache-ts"; import { type CultNetDocumentDeleteMessage, diff --git a/packages/cultnet-ts/src/rudp.ts b/packages/cultnet-ts/src/rudp.ts index 7a38cb1d..56dc67fa 100644 --- a/packages/cultnet-ts/src/rudp.ts +++ b/packages/cultnet-ts/src/rudp.ts @@ -16,6 +16,8 @@ const RUDP_MAGIC = [0x43, 0x4e, 0x52, 0x30] as const; // CNR0 const RUDP_VERSION = 0; const RUDP_FIXED_HEADER_BYTES = 36; const MAX_CHANNEL_ID_BYTES = 255; +export const CULTNET_RUDP_RELIABLE_SEND_WINDOW_PACKETS = 32; +const RUDP_RECEIVED_SEQUENCE_WINDOW = 4_096; export type CultNetRudpPacketType = | "connect" @@ -57,6 +59,7 @@ export interface CultNetRudpSessionOptions { export interface CultNetRudpReceiveResult { delivered: CultNetRudpDeliveredFrame[]; + readyToSend?: CultNetRudpPacket[]; reply?: CultNetRudpPacket; pong?: boolean; pongPayload?: Uint8Array; @@ -144,16 +147,25 @@ export class CultNetRudpSession { readonly connectionId: number; readonly resendDelayMs: number; #nextSequence: number; + readonly #nextSequencedByChannel = new Map(); #nextFragmentId = 1; #connected = false; readonly #maxPendingReliablePackets: number | undefined; #lastReceivedAtMs: number | undefined; #highestReceivedSequence: number | undefined; readonly #receivedSequences = new Set(); + readonly #latestSequencedByChannel = new Map(); readonly #pendingReliable = new Map(); + readonly #queuedReliable: CultNetRudpPacket[] = []; readonly #orderedNextSequenceByChannel = new Map(); readonly #orderedBuffers = new Map>(); readonly #fragmentBuffers = new Map(); + /// Matches cultnet-rs's `max_pending_fragment_sets`. A fragment set is only + /// removed on successful reassembly, so a set that loses one fragment is + /// stranded for the life of the session. Without a bound the map grows + /// forever under any loss on a fragmenting channel. + #maxPendingFragmentSets = 64; + #fragmentSetsEvicted = 0; constructor(options: CultNetRudpSessionOptions) { this.connectionId = toUint32(options.connectionId, "connectionId"); @@ -173,6 +185,25 @@ export class CultNetRudpSession { return [...this.#pendingReliable.keys()].sort((left, right) => left - right); } + get queuedReliablePacketCount(): number { + return this.#queuedReliable.length; + } + + get outstandingReliablePacketCount(): number { + return this.#pendingReliable.size + this.#queuedReliable.length; + } + + /// Incomplete fragment sets dropped to stay within the pending bound. + /// Non-zero means the peer is losing fragments: the receiver is still + /// serving, and something upstream is not. + get fragmentSetsEvicted(): number { + return this.#fragmentSetsEvicted; + } + + get pendingFragmentSetCount(): number { + return this.#fragmentBuffers.size; + } + get lastReceivedAtMs(): number | undefined { return this.#lastReceivedAtMs; } @@ -182,10 +213,14 @@ export class CultNetRudpSession { this.#lastReceivedAtMs = undefined; this.#highestReceivedSequence = undefined; this.#receivedSequences.clear(); + this.#nextSequencedByChannel.clear(); + this.#latestSequencedByChannel.clear(); this.#pendingReliable.clear(); + this.#queuedReliable.splice(0); this.#orderedNextSequenceByChannel.clear(); this.#orderedBuffers.clear(); this.#fragmentBuffers.clear(); + this.#fragmentSetsEvicted = 0; } createConnect(nowMs = 0, payload = new Uint8Array()): CultNetRudpPacket { @@ -238,6 +273,9 @@ export class CultNetRudpSession { payload: Uint8Array, options: { reliable?: boolean; ordered?: boolean; sequenced?: boolean; nowMs?: number } = {}, ): CultNetRudpPacket { + if (options.reliable && this.#pendingReliable.size >= CULTNET_RUDP_RELIABLE_SEND_WINDOW_PACKETS) { + throw new Error("RUDP reliable send window is full; receive acknowledgements before sending."); + } return this.sendMany(channelId, payload, options)[0]!; } @@ -255,6 +293,9 @@ export class CultNetRudpSession { if (!this.#connected) { throw new Error("Cannot send RUDP data before the session is connected."); } + if (options.ordered && !options.reliable) { + throw new Error("RUDP ordered delivery requires reliability."); + } const maxFragmentBytes = options.maxFragmentBytes; if (maxFragmentBytes !== undefined && maxFragmentBytes <= 0) { @@ -271,10 +312,9 @@ export class CultNetRudpSession { ordered: options.ordered, sequenced: options.sequenced, }); - if (packet.reliable) { - this.#trackReliable(packet, options.nowMs ?? 0); - } - return [packet]; + return packet.reliable + ? this.#admitReliablePackets([packet], options.nowMs ?? 0) + : [packet]; } const fragmentCount = Math.ceil(payload.byteLength / maxFragmentBytes); @@ -298,17 +338,17 @@ export class CultNetRudpSession { fragmentIndex: index, fragmentCount, }); - if (packet.reliable) { - this.#trackReliable(packet, options.nowMs ?? 0); - } packets.push(packet); } - return packets; + return options.reliable + ? this.#admitReliablePackets(packets, options.nowMs ?? 0) + : packets; } receive(packet: CultNetRudpPacket, nowMs = 0): CultNetRudpReceiveResult { this.#requireConnection(packet); this.#applyAcknowledgements(packet); + const readyToSend = this.#promoteQueuedReliable(nowMs); this.#lastReceivedAtMs = nowMs; const expectedSequenceIfUninitialized = this.#highestReceivedSequence === undefined ? packet.sequence @@ -317,13 +357,13 @@ export class CultNetRudpSession { if (packet.packetType === "accept") { this.#rememberReceived(packet.sequence); this.#connected = true; - return { delivered: [] }; + return { delivered: [], readyToSend }; } if (packet.packetType === "ping") { - this.#rememberReceived(packet.sequence); return { delivered: [], + readyToSend, reply: this.#createPacket({ packetType: "pong", channelId: "control", @@ -333,44 +373,61 @@ export class CultNetRudpSession { } if (packet.packetType === "ack" || packet.packetType === "pong") { - if (packet.packetType === "pong") this.#rememberReceived(packet.sequence); return { - delivered: this.#drainUnblockedOrdered(), + delivered: [], + readyToSend, pong: packet.packetType === "pong", pongPayload: packet.packetType === "pong" ? packet.payload ?? new Uint8Array() : undefined, }; } if (packet.packetType === "disconnect") { - this.#rememberReceived(packet.sequence); this.#connected = false; return { delivered: [], + readyToSend, disconnected: true, disconnectReason: packet.payload ?? new Uint8Array(), }; } if (packet.packetType !== "data") { - return { delivered: [] }; + return { delivered: [], readyToSend }; } - const isDuplicate = this.#receivedSequences.has(packet.sequence); - this.#rememberReceived(packet.sequence); + const isDuplicate = (packet.reliable ?? false) + && (this.#receivedSequences.has(packet.sequence) + || (this.#highestReceivedSequence !== undefined + && packet.sequence < this.#highestReceivedSequence + && this.#highestReceivedSequence - packet.sequence >= RUDP_RECEIVED_SEQUENCE_WINDOW)); + if (packet.reliable ?? false) this.#rememberReceived(packet.sequence); if (isDuplicate) { - return { delivered: [] }; + return { delivered: [], readyToSend }; } const reassembled = this.#reassemble(packet); if (!reassembled) { - return { delivered: [] }; + return { delivered: [], readyToSend }; + } + + if (!reassembled.ordered && (packet.sequenced ?? false)) { + const newestSequence = reassembled.nextSequence - 1; + const latestSequence = this.#latestSequencedByChannel.get(reassembled.frame.channelId); + if (latestSequence !== undefined && newestSequence <= latestSequence) { + return { delivered: [], readyToSend }; + } + this.#latestSequencedByChannel.set(reassembled.frame.channelId, newestSequence); + return { delivered: [reassembled.frame], readyToSend }; } if (!reassembled.ordered) { - return { delivered: [reassembled.frame] }; + return { delivered: [reassembled.frame], readyToSend }; } - return { delivered: this.#deliverOrdered(reassembled.frame, reassembled.nextSequence, expectedSequenceIfUninitialized) }; + return { + delivered: this.#deliverOrdered(reassembled.frame, reassembled.nextSequence, expectedSequenceIfUninitialized), + readyToSend, + }; } createAck(): CultNetRudpPacket { @@ -410,6 +467,14 @@ export class CultNetRudpSession { }; } + createAckForReceived(sequence: number): CultNetRudpPacket { + const receivedSequence = toUint32(sequence, "received sequence"); + const { ack } = this.#ackState(); + return ack >= receivedSequence && ack - receivedSequence <= 32 + ? this.createAck() + : this.createAckFor(receivedSequence); + } + createPing(payload = new Uint8Array()): CultNetRudpPacket { return this.#createPacket({ packetType: "ping", @@ -460,8 +525,17 @@ export class CultNetRudpSession { fragmentIndex?: number; fragmentCount?: number; }): CultNetRudpPacket { - const sequence = this.#nextSequence; - this.#nextSequence = toUint32(this.#nextSequence + 1, "sequence"); + let sequence = 0; + if (packet.reliable ?? false) { + sequence = this.#nextSequence; + this.#nextSequence = toUint32(this.#nextSequence + 1, "reliable sequence"); + } else if (packet.sequenced ?? false) { + sequence = this.#nextSequencedByChannel.get(packet.channelId) ?? 1; + this.#nextSequencedByChannel.set( + packet.channelId, + toUint32(sequence + 1, "sequenced channel sequence"), + ); + } const { ack, ackMask } = this.#ackState(); return { packetType: packet.packetType, @@ -487,11 +561,26 @@ export class CultNetRudpSession { }); } + #admitReliablePackets(packets: CultNetRudpPacket[], nowMs: number): CultNetRudpPacket[] { + const available = Math.max(0, CULTNET_RUDP_RELIABLE_SEND_WINDOW_PACKETS - this.#pendingReliable.size); + const ready = packets.slice(0, available); + for (const packet of ready) this.#trackReliable(packet, nowMs); + this.#queuedReliable.push(...packets.slice(available)); + return ready; + } + + #promoteQueuedReliable(nowMs: number): CultNetRudpPacket[] { + const available = Math.max(0, CULTNET_RUDP_RELIABLE_SEND_WINDOW_PACKETS - this.#pendingReliable.size); + const ready = this.#queuedReliable.splice(0, available); + for (const packet of ready) this.#trackReliable(packet, nowMs); + return ready; + } + #ensureReliableCapacity(packetCount: number): void { if (packetCount === 0 || this.#maxPendingReliablePackets === undefined) { return; } - if (this.#pendingReliable.size + packetCount > this.#maxPendingReliablePackets) { + if (this.outstandingReliablePacketCount + packetCount > this.#maxPendingReliablePackets) { throw new Error("RUDP reliable send queue is full."); } } @@ -510,6 +599,12 @@ export class CultNetRudpSession { if (this.#highestReceivedSequence === undefined || sequence > this.#highestReceivedSequence) { this.#highestReceivedSequence = sequence; } + if (this.#receivedSequences.size > RUDP_RECEIVED_SEQUENCE_WINDOW) { + const keepFrom = Math.max(0, (this.#highestReceivedSequence ?? sequence) - RUDP_RECEIVED_SEQUENCE_WINDOW + 1); + for (const received of this.#receivedSequences) { + if (received < keepFrom) this.#receivedSequences.delete(received); + } + } } #ackState(): { ack: number; ackMask: number } { @@ -559,6 +654,20 @@ export class CultNetRudpSession { payloads: new Map(), sequences: new Map(), }; + // Evict the oldest stranded set rather than refusing the payload. A + // receiver that stops accepting fragmented traffic after a bounded + // number of losses is a denial of service delivered by the network; + // dropping the least recent incomplete set costs one payload that was + // already incomplete. Map iteration is insertion-ordered, so the first + // key is the oldest. + while (this.#fragmentBuffers.size >= this.#maxPendingFragmentSets) { + const oldest = this.#fragmentBuffers.keys().next(); + if (oldest.done) { + break; + } + this.#fragmentBuffers.delete(oldest.value); + this.#fragmentSetsEvicted += 1; + } this.#fragmentBuffers.set(key, buffer); } if (buffer.fragmentCount !== fragmentCount || buffer.ordered !== (packet.ordered ?? false)) { @@ -659,15 +768,6 @@ export class CultNetRudpSession { return delivered; } - #drainUnblockedOrdered(): CultNetRudpDeliveredFrame[] { - const delivered: CultNetRudpDeliveredFrame[] = []; - for (const channelId of this.#orderedBuffers.keys()) { - this.#skipReceivedNonChannelSequences(channelId); - delivered.push(...this.#drainOrdered(channelId)); - } - return delivered; - } - #skipReceivedNonChannelSequences(channelId: string): void { let next = this.#orderedNextSequenceByChannel.get(channelId); while (next !== undefined && this.#receivedSequences.has(next) && !this.#orderedBuffers.get(channelId)?.has(next)) { @@ -754,6 +854,10 @@ export class CultNetRudpSocketTransportConnection extends EventEmitter implement return { ...this.#stats }; } + get outstandingReliablePacketCount(): number { + return this.#session.outstandingReliablePacketCount; + } + connect(payload = new Uint8Array()): void { if (this.#mode !== "client") { throw new Error("Only a client RUDP socket transport can initiate connect."); @@ -773,6 +877,21 @@ export class CultNetRudpSocketTransportConnection extends EventEmitter implement this.#stats.framesSent += 1; } + async flush(timeoutMs = 30_000): Promise { + const deadline = Date.now() + Math.max(0, timeoutMs); + while (this.#session.outstandingReliablePacketCount > 0) { + if (this.#closed) { + throw new Error("RUDP transport closed before reliable packets were acknowledged."); + } + if (Date.now() >= deadline) { + throw new Error( + `RUDP reliable flush timed out with ${this.#session.outstandingReliablePacketCount} packets outstanding.`, + ); + } + await new Promise(resolve => setTimeout(resolve, 5)); + } + } + ping(payload = new Uint8Array()): void { this.#sendPacket(this.#session.createPing(payload)); } @@ -834,6 +953,9 @@ export class CultNetRudpSocketTransportConnection extends EventEmitter implement if (result.reply) { this.#sendPacket(result.reply); } + for (const ready of result.readyToSend ?? []) { + this.#sendPacket(ready); + } if (result.pong) { this.emit("pong", { payload: result.pongPayload ?? new Uint8Array() }); } @@ -850,7 +972,7 @@ export class CultNetRudpSocketTransportConnection extends EventEmitter implement return; } if (packet.packetType === "accept" || packet.packetType === "data" || result.delivered.length > 0) { - this.#sendPacket(this.#session.createAck()); + this.#sendPacket(this.#session.createAckForReceived(packet.sequence)); } } catch (error) { this.emit("error", error instanceof Error ? error : new Error(String(error))); diff --git a/packages/cultnet-ts/src/runtime-presence-health.ts b/packages/cultnet-ts/src/runtime-presence-health.ts new file mode 100644 index 00000000..8d70da8e --- /dev/null +++ b/packages/cultnet-ts/src/runtime-presence-health.ts @@ -0,0 +1,256 @@ +// Runtime presence health: the dual-proved statement one launched incarnation +// makes about itself. +// +// This is the TypeScript half of `gamecult.runtime_presence_health.v2`, the +// contract `cultnet-rs` defines in `runtime_authority_contracts.rs`. Odin admits +// a document as runtime presence only when it carries this schema id and its +// record key equals the signed target; anything else it files away as an +// ordinary document, and no topology correlation is ever produced for it. +// +// Two proofs cover one payload. The stable provider key says "this service made +// this statement"; the activation key Idunn hands the process at launch says +// "and it is the incarnation Idunn started". Neither establishes admission +// alone, which is why both signature fields are empty in the signed bytes: it +// avoids a circular signature while binding every authority-bearing field and +// both signer identities. +// +// The wire shape is a positional tuple indexed by the same field slots the Rust +// derive emits. Field N here is field N there; that correspondence is the whole +// contract, and the constants below name the slots so it stays checkable. + +import { encode } from "@msgpack/msgpack"; + +export const GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA = "gamecult.runtime_presence_health.v2"; + +/** Purpose for the stable provider-health signature over the proof payload. */ +export const GAMECULT_RUNTIME_PRESENCE_HEALTH_SIGNING_PURPOSE = + "gamecult.runtime_presence_health.v2"; + +/** Purpose for the activation-scoped proof over the same payload. */ +export const GAMECULT_RUNTIME_ACTIVATION_PROOF_SIGNING_PURPOSE = + "gamecult.runtime_presence.activation-proof.v1"; + +/** systemd file-descriptor name Idunn uses for the activation signing key. */ +export const IDUNN_RUNTIME_ACTIVATION_CREDENTIAL_NAME = "gamecult-idunn-runtime-activation-key"; + +/** systemd file-descriptor name Idunn uses for the provider-health identity. */ +export const GAMECULT_RUNTIME_PRESENCE_IDENTITY_NAME = "gamecult-runtime-presence-identity"; + +const PROVIDER_HEALTH_SIGNATURE_DOMAIN = Buffer.from( + "gamecult.provider-health.signature.v1\0", + "utf8", +); + +const RUNTIME_ACTIVATION_SIGNATURE_DOMAIN = Buffer.from( + "idunn.runtime-activation.signature.v1\0", + "utf8", +); + +/** + * Positional slots, mirroring the `#[cultcache(key = N)]` attributes on + * `GameCultRuntimePresenceHealthRecord`. The tuple is always this long: the + * Rust derive serialises `max_slot + 1` elements regardless of which are set. + */ +export const RUNTIME_PRESENCE_SLOT = { + schemaVersion: 0, + target: 1, + expectedProjectionSha256: 2, + planId: 3, + incarnationId: 4, + sealedReleaseId: 5, + activationWitnessSha256: 6, + stateSchemaGeneration: 7, + stateContractSha256: 8, + runtimeId: 9, + runtimeInstanceId: 10, + boundEndpoint: 11, + capabilities: 12, + healthContract: 13, + state: 14, + detail: 15, + writeLeaseSha256: 16, + signerIdentityId: 17, + publisherSequence: 18, + observedAtUnixMillis: 19, + signatureAlgorithm: 20, + signature: 21, + activationSignerIdentityId: 22, + activationSignature: 23, +} as const; + +export const RUNTIME_PRESENCE_FIELD_COUNT = 24; + +/** A capability claim. Nested as its own positional tuple, not a map. */ +export interface GameCultRuntimeCapability { + capability: string; + schema: string; + compatibility: string; + capacity: number; +} + +/** The states the contract admits. */ +export type RuntimePresenceState = "warming" | "active" | "degraded" | "failed"; + +export interface RuntimePresenceHealth { + target: string; + expectedProjectionSha256: string; + planId: string; + incarnationId: string; + sealedReleaseId: string; + activationWitnessSha256: string; + stateSchemaGeneration: string | null; + stateContractSha256: string | null; + runtimeId: string; + runtimeInstanceId: string; + boundEndpoint: string | null; + capabilities: GameCultRuntimeCapability[]; + healthContract: string; + state: RuntimePresenceState; + detail: string; + writeLeaseSha256: string | null; + signerIdentityId: string; + publisherSequence: number; + observedAtUnixMillis: number; + activationSignerIdentityId: string; +} + +function encodeCapability(capability: GameCultRuntimeCapability): unknown[] { + // Plain serde struct on the Rust side, and the record is encoded with + // rmp_serde's compact form, so a capability is a positional tuple too. + return [capability.capability, capability.schema, capability.compatibility, capability.capacity]; +} + +/** + * Capabilities are canonical only when strictly increasing by + * `(capability, schema, compatibility)`. Sorting here rather than trusting the + * caller keeps a correct statement from being rejected over field order. + */ +function canonicalCapabilities(capabilities: GameCultRuntimeCapability[]): unknown[][] { + const sorted = [...capabilities].sort((left, right) => { + const leftKey = [left.capability, left.schema, left.compatibility]; + const rightKey = [right.capability, right.schema, right.compatibility]; + for (let index = 0; index < leftKey.length; index += 1) { + if (leftKey[index]! < rightKey[index]!) return -1; + if (leftKey[index]! > rightKey[index]!) return 1; + } + return 0; + }); + return sorted.map(encodeCapability); +} + +function presenceTuple( + presence: RuntimePresenceHealth, + signature: Uint8Array, + activationSignature: Uint8Array, +): unknown[] { + const fields = new Array(RUNTIME_PRESENCE_FIELD_COUNT); + fields[RUNTIME_PRESENCE_SLOT.schemaVersion] = GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA; + fields[RUNTIME_PRESENCE_SLOT.target] = presence.target; + fields[RUNTIME_PRESENCE_SLOT.expectedProjectionSha256] = presence.expectedProjectionSha256; + fields[RUNTIME_PRESENCE_SLOT.planId] = presence.planId; + fields[RUNTIME_PRESENCE_SLOT.incarnationId] = presence.incarnationId; + fields[RUNTIME_PRESENCE_SLOT.sealedReleaseId] = presence.sealedReleaseId; + fields[RUNTIME_PRESENCE_SLOT.activationWitnessSha256] = presence.activationWitnessSha256; + fields[RUNTIME_PRESENCE_SLOT.stateSchemaGeneration] = presence.stateSchemaGeneration; + fields[RUNTIME_PRESENCE_SLOT.stateContractSha256] = presence.stateContractSha256; + fields[RUNTIME_PRESENCE_SLOT.runtimeId] = presence.runtimeId; + fields[RUNTIME_PRESENCE_SLOT.runtimeInstanceId] = presence.runtimeInstanceId; + fields[RUNTIME_PRESENCE_SLOT.boundEndpoint] = presence.boundEndpoint; + fields[RUNTIME_PRESENCE_SLOT.capabilities] = canonicalCapabilities(presence.capabilities); + fields[RUNTIME_PRESENCE_SLOT.healthContract] = presence.healthContract; + fields[RUNTIME_PRESENCE_SLOT.state] = presence.state; + fields[RUNTIME_PRESENCE_SLOT.detail] = presence.detail; + fields[RUNTIME_PRESENCE_SLOT.writeLeaseSha256] = presence.writeLeaseSha256; + fields[RUNTIME_PRESENCE_SLOT.signerIdentityId] = presence.signerIdentityId; + fields[RUNTIME_PRESENCE_SLOT.publisherSequence] = presence.publisherSequence; + fields[RUNTIME_PRESENCE_SLOT.observedAtUnixMillis] = presence.observedAtUnixMillis; + fields[RUNTIME_PRESENCE_SLOT.signatureAlgorithm] = "ed25519"; + fields[RUNTIME_PRESENCE_SLOT.signature] = signature; + fields[RUNTIME_PRESENCE_SLOT.activationSignerIdentityId] = presence.activationSignerIdentityId; + fields[RUNTIME_PRESENCE_SLOT.activationSignature] = activationSignature; + return fields; +} + +/** + * Reject statements the contract forbids, before they are signed. + * + * These are the same conditions `validate_shape` enforces on the Rust side. A + * statement that fails here would be refused after a round trip and a signature; + * failing early names the actual problem instead. + */ +export function validateRuntimePresence(presence: RuntimePresenceHealth): void { + if (presence.publisherSequence <= 0 || !Number.isSafeInteger(presence.publisherSequence)) { + throw new Error("runtime presence publisher sequence must be a positive integer"); + } + if (presence.observedAtUnixMillis <= 0 || !Number.isSafeInteger(presence.observedAtUnixMillis)) { + throw new Error("runtime presence observation time must be a positive integer"); + } + if (presence.detail.length > 512 || /\p{Cc}/u.test(presence.detail)) { + throw new Error("runtime presence detail is too long or carries control characters"); + } + if (presence.state === "warming" && presence.writeLeaseSha256 !== null) { + throw new Error("warming runtime presence cannot claim a process write lease"); + } + if (presence.stateSchemaGeneration === null && presence.writeLeaseSha256 !== null) { + throw new Error("stateless runtime presence cannot claim a process write lease"); + } + const generationPresent = presence.stateSchemaGeneration !== null; + const contractPresent = presence.stateContractSha256 !== null; + if (generationPresent !== contractPresent) { + throw new Error("runtime presence state lineage is partial"); + } +} + +/** + * The one payload both proofs cover: the complete record with both signature + * fields empty. + */ +export function runtimePresenceProofPayload(presence: RuntimePresenceHealth): Uint8Array { + validateRuntimePresence(presence); + return encode(presenceTuple(presence, new Uint8Array(), new Uint8Array())); +} + +/** The complete signed record, as put on the wire. */ +export function encodeRuntimePresenceHealth( + presence: RuntimePresenceHealth, + signature: Uint8Array, + activationSignature: Uint8Array, +): Uint8Array { + if (signature.length !== 64 || activationSignature.length !== 64) { + throw new Error("runtime presence proofs must both be 64-byte ed25519 signatures"); + } + validateRuntimePresence(presence); + return encode(presenceTuple(presence, signature, activationSignature)); +} + +function domainSeparated(domain: Buffer, purpose: string, payload: Uint8Array): Buffer { + const purposeBytes = Buffer.from(purpose, "utf8"); + const lengths = Buffer.alloc(16); + lengths.writeBigUInt64BE(BigInt(purposeBytes.length), 0); + lengths.writeBigUInt64BE(BigInt(payload.length), 8); + return Buffer.concat([ + domain, + lengths.subarray(0, 8), + purposeBytes, + lengths.subarray(8, 16), + Buffer.from(payload), + ]); +} + +/** Message the stable provider-health key signs. */ +export function runtimePresenceProviderSigningMessage(payload: Uint8Array): Buffer { + return domainSeparated( + PROVIDER_HEALTH_SIGNATURE_DOMAIN, + GAMECULT_RUNTIME_PRESENCE_HEALTH_SIGNING_PURPOSE, + payload, + ); +} + +/** Message the activation key Idunn passed at launch signs. */ +export function runtimePresenceActivationSigningMessage(payload: Uint8Array): Buffer { + return domainSeparated( + RUNTIME_ACTIVATION_SIGNATURE_DOMAIN, + GAMECULT_RUNTIME_ACTIVATION_PROOF_SIGNING_PURPOSE, + payload, + ); +} diff --git a/packages/cultnet-ts/src/signed-daemon-health.ts b/packages/cultnet-ts/src/signed-daemon-health.ts new file mode 100644 index 00000000..d4f0f0ce --- /dev/null +++ b/packages/cultnet-ts/src/signed-daemon-health.ts @@ -0,0 +1,377 @@ +// Signed daemon-health publication over CultNet RUDP. +// +// This is the transport half of daemon health publication: bind a socket, shake +// hands, sign the statement, put the document, wait for the ack. It is generic +// across services. The receiving service owns the connection id and the schema +// names, and supplies them as a contract; nothing service-specific is hardcoded +// here. +// +// The signature and identity domains are deliberately not parameterised. They +// are the GameCult provider-health domains, shared by every publisher and +// verifier, and a caller that could vary them could silently produce statements +// nobody verifies. + +import dgram from "node:dgram"; +import crypto from "node:crypto"; +import fs from "node:fs"; +import { encode } from "@msgpack/msgpack"; + +import { CultNetRudpSession, decodeRudpPacket, encodeRudpPacket } from "./rudp"; +import type { CultNetRudpPacket } from "./rudp"; +import { encodeCultNetMessageForWire } from "./contracts"; + +export const CULTNET_RUDP_PROTOCOL_ID = "cultnet.transport.rudp.v0"; + +const SIGNATURE_DOMAIN = Buffer.from("gamecult.provider-health.signature.v1\0", "utf8"); +const ID_DOMAIN = Buffer.from("gamecult.provider-health.identity.v1\0", "utf8"); + +/** What the receiving service owns: where to connect and what to call the documents. */ +export interface SignedDaemonHealthContract { + /** RUDP connection id the receiving service listens on. */ + connectionId: number; + /** Schema id for a signed statement. Also the signing purpose. */ + signedSchemaId: string; + /** Schema id used when no signing key is configured. */ + unsignedSchemaId: string; + /** Prefix for generated message ids. Defaults to "daemon-health". */ + messageIdPrefix?: string; +} + +export interface SignedDaemonHealthPublisherOptions { + /** host:port, or [ipv6]:port. */ + endpoint: string; + daemonId: string; + healthContract: string; + sourceRuntimeId?: string; + /** Omit to publish unsigned. */ + privateKeyPath?: string; + contract: SignedDaemonHealthContract; +} + +export interface DaemonHealth { + state: string; + detail?: string; + observedAt?: string; +} + +export interface SignedDaemonHealthPublisher { + daemonId: string; + endpoint: { host: string; port: number }; + healthContract: string; + publisherIncarnationId: string; + publisherSequence: number; + sourceRuntimeId: string; + contract: SignedDaemonHealthContract; + privateKey?: crypto.KeyObject; + publicKey?: Buffer; + signerIdentityId?: string; +} + +export function createSignedDaemonHealthPublisher( + options: SignedDaemonHealthPublisherOptions | null | undefined, +): SignedDaemonHealthPublisher | null { + if (!options) return null; + if (!options.contract) { + throw new Error("Signed daemon health publisher requires a contract naming the connection id and schema ids."); + } + + const publisher: SignedDaemonHealthPublisher = { + daemonId: options.daemonId, + endpoint: parseEndpoint(options.endpoint), + healthContract: options.healthContract, + publisherIncarnationId: crypto.randomUUID(), + publisherSequence: 0, + sourceRuntimeId: options.sourceRuntimeId || "daemon-health-publisher", + contract: options.contract, + }; + + if (options.privateKeyPath) { + publisher.privateKey = crypto.createPrivateKey(fs.readFileSync(options.privateKeyPath)); + publisher.publicKey = rawEd25519PublicKey(publisher.privateKey); + publisher.signerIdentityId = crypto + .createHash("sha256") + .update(ID_DOMAIN) + .update(publisher.publicKey) + .digest("hex"); + } + return publisher; +} + +export async function publishSignedDaemonHealth( + publisher: SignedDaemonHealthPublisher | null, + health: DaemonHealth, +): Promise { + if (!publisher) return; + const { contract } = publisher; + + const socket = dgram.createSocket(endpointFamily(publisher.endpoint.host)); + await bindSocket(socket, publisher.endpoint); + const receiver = createPacketReceiver(socket); + const session = new CultNetRudpSession({ + connectionId: contract.connectionId, + initialSequence: 1, + resendDelayMs: 100, + }); + + try { + const connect = session.createConnect(Date.now(), new Uint8Array()); + await sendPacket(socket, publisher.endpoint, connect); + await receiveUntil(receiver, session, publisher.endpoint, (packet) => packet.packetType === "accept", 5000, "accept"); + + const observedAt = health.observedAt || new Date().toISOString(); + const signed = publisher.privateKey ? signedDaemonHealthPayload(publisher, health, observedAt) : null; + const payload = signed?.payload || encode([ + publisher.daemonId, + health.state, + String(health.detail || "").slice(0, 512), + observedAt, + publisher.healthContract, + "daemon-published", + CULTNET_RUDP_PROTOCOL_ID, + ]); + + const message = { + schemaVersion: "cultnet.document_put_raw.v0", + messageId: `${contract.messageIdPrefix || "daemon-health"}:${publisher.daemonId}:${observedAt.replace(/[:.]/g, "-")}`, + document: { + schemaId: signed ? contract.signedSchemaId : contract.unsignedSchemaId, + recordKey: publisher.daemonId, + storedAt: observedAt, + payloadEncoding: "messagepack", + payload, + sourceRuntimeId: publisher.sourceRuntimeId, + sourceRole: "daemon-health-publisher", + tags: [CULTNET_RUDP_PROTOCOL_ID], + }, + }; + + const wirePayload = encode(encodeCultNetMessageForWire(message as never, "cultnet.schema.v0")); + const dataPackets = session.sendMany("schema", wirePayload, { reliable: true, ordered: true, nowMs: Date.now() }); + const ack = receiveUntil( + receiver, + session, + publisher.endpoint, + (packet) => packet.packetType === "ack", + 500, + "ack", + ).catch(() => undefined); + for (const packet of dataPackets) { + await sendPacket(socket, publisher.endpoint, packet); + } + await ack; + } finally { + receiver.close(); + socket.close(); + } +} + +export function signedDaemonHealthPayload( + publisher: SignedDaemonHealthPublisher, + health: DaemonHealth, + observedAt: string, +): { payload: Uint8Array; statement: unknown[]; unsignedPayload: Uint8Array } { + if (!publisher.privateKey) { + throw new Error("Signed daemon health requires a configured private key."); + } + publisher.publisherSequence += 1; + const observedAtUnixMillis = Date.parse(observedAt); + if (!Number.isSafeInteger(observedAtUnixMillis) || observedAtUnixMillis <= 0) { + throw new Error("Signed daemon health observation time is invalid."); + } + + const unsigned: unknown[] = [ + publisher.contract.signedSchemaId, + publisher.daemonId, + publisher.healthContract, + publisher.sourceRuntimeId, + health.state, + String(health.detail || "").slice(0, 512), + publisher.signerIdentityId, + publisher.publisherIncarnationId, + publisher.publisherSequence, + observedAtUnixMillis, + null, + null, + null, + null, + "ed25519", + new Uint8Array(), + false, + ]; + + const unsignedPayload = encode(unsigned); + const signature = crypto.sign( + null, + signedDaemonHealthSigningMessage(publisher.contract.signedSchemaId, unsignedPayload), + publisher.privateKey, + ); + const statement = unsigned.slice(); + statement[15] = new Uint8Array(signature); + return { payload: encode(statement), statement, unsignedPayload }; +} + +/** + * Domain-separated signing preimage. The signing purpose is the signed schema + * id, so a statement signed for one service's schema cannot verify as another's. + */ +export function signedDaemonHealthSigningMessage(signedSchemaId: string, payload: Uint8Array): Buffer { + const purpose = Buffer.from(signedSchemaId, "utf8"); + const purposeLength = Buffer.alloc(8); + purposeLength.writeBigUInt64BE(BigInt(purpose.length)); + const payloadLength = Buffer.alloc(8); + payloadLength.writeBigUInt64BE(BigInt(payload.length)); + return Buffer.concat([SIGNATURE_DOMAIN, purposeLength, purpose, payloadLength, Buffer.from(payload)]); +} + +function rawEd25519PublicKey(privateKey: crypto.KeyObject): Buffer { + const der = crypto.createPublicKey(privateKey).export({ type: "spki", format: "der" }) as Buffer; + if (der.length < 32) throw new Error("Ed25519 public key export is too short."); + return Buffer.from(der.subarray(der.length - 32)); +} + +async function bindSocket(socket: dgram.Socket, endpoint: { host: string }): Promise { + await new Promise((resolve, reject) => { + socket.once("error", reject); + socket.bind(0, endpoint.host.includes(":") ? "::" : "0.0.0.0", () => { + socket.off("error", reject); + resolve(); + }); + }); +} + +export function parseEndpoint(value: string): { host: string; port: number } { + const text = String(value || "").trim(); + const ipv6 = text.match(/^\[([^\]]+)\]:(\d+)$/); + if (ipv6) return { host: ipv6[1], port: parsePort(ipv6[2]) }; + const index = text.lastIndexOf(":"); + if (index <= 0) { + throw new Error(`Daemon health RUDP endpoint must be host:port, got "${value}".`); + } + return { host: text.slice(0, index), port: parsePort(text.slice(index + 1)) }; +} + +function parsePort(value: string): number { + const port = Number(value); + if (!Number.isInteger(port) || port <= 0 || port > 65535) { + throw new Error(`Daemon health RUDP endpoint port is invalid: ${value}`); + } + return port; +} + +function endpointFamily(host: string): "udp4" | "udp6" { + return host.includes(":") ? "udp6" : "udp4"; +} + +interface PacketReceiver { + socket: dgram.Socket; + next(timeoutMs: number, label?: string): Promise; + close(): void; +} + +async function receiveUntil( + receiver: PacketReceiver, + session: CultNetRudpSession, + endpoint: { host: string; port: number }, + predicate: (packet: CultNetRudpPacket) => boolean, + timeoutMs: number, + label: string, +): Promise { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + try { + const packet = await receiver.next(Math.min(100, deadline - Date.now()), label); + const result = session.receive(packet, Date.now()); + if (result.reply) { + throw new Error("Daemon health publisher received an unexpected reply-required packet."); + } + if (predicate(packet)) return; + } catch (error) { + if ((error as NodeJS.ErrnoException)?.code !== "ETIMEDOUT") throw error; + } + for (const packet of session.dueResends(Date.now())) { + await sendPacket(receiver.socket, endpoint, packet); + } + } + throw new Error(`timed out waiting for daemon health RUDP ${label} response after ${timeoutMs}ms`); +} + +function createPacketReceiver(socket: dgram.Socket): PacketReceiver { + const packets: CultNetRudpPacket[] = []; + const waiters: Array<{ + resolve: (packet: CultNetRudpPacket) => void; + reject: (error: Error) => void; + timer: NodeJS.Timeout; + }> = []; + const errors: Error[] = []; + + const resolveNext = () => { + while (waiters.length > 0 && (packets.length > 0 || errors.length > 0)) { + const waiter = waiters.shift()!; + clearTimeout(waiter.timer); + if (errors.length > 0) waiter.reject(errors.shift()!); + else waiter.resolve(packets.shift()!); + } + }; + const onMessage = (wire: Buffer) => { + try { + packets.push(decodeRudpPacket(wire)); + } catch (error) { + errors.push(error as Error); + } + resolveNext(); + }; + const onError = (error: Error) => { + errors.push(error); + resolveNext(); + }; + + socket.on("message", onMessage); + socket.on("error", onError); + + return { + socket, + next(timeoutMs: number, label = "packet") { + if (packets.length > 0) return Promise.resolve(packets.shift()!); + if (errors.length > 0) return Promise.reject(errors.shift()!); + return new Promise((resolve, reject) => { + const waiter = { + resolve, + reject, + timer: setTimeout(() => { + const index = waiters.indexOf(waiter); + if (index >= 0) waiters.splice(index, 1); + const error = new Error(`timed out waiting for daemon health RUDP ${label}`) as NodeJS.ErrnoException; + error.code = "ETIMEDOUT"; + reject(error); + }, Math.max(1, timeoutMs)), + }; + waiters.push(waiter); + }); + }, + close() { + socket.off("message", onMessage); + socket.off("error", onError); + while (waiters.length > 0) { + const waiter = waiters.shift()!; + clearTimeout(waiter.timer); + const error = new Error("Daemon health publisher closed.") as NodeJS.ErrnoException; + error.code = "ECLOSED"; + waiter.reject(error); + } + }, + }; +} + +async function sendPacket( + socket: dgram.Socket, + endpoint: { host: string; port: number }, + packet: CultNetRudpPacket, +): Promise { + const wire = encodeRudpPacket(packet); + await new Promise((resolve, reject) => { + socket.send(wire, endpoint.port, endpoint.host, (error) => { + if (error) reject(error); + else resolve(); + }); + }); +} diff --git a/packages/cultnet-ts/src/transport.ts b/packages/cultnet-ts/src/transport.ts index 53f64127..a1b9b3ce 100644 --- a/packages/cultnet-ts/src/transport.ts +++ b/packages/cultnet-ts/src/transport.ts @@ -131,6 +131,7 @@ export interface CultNetTransportConnection { readonly profile: CultNetTransportProfile; readonly stats: CultNetTransportStats; send(channelId: string, payload: Uint8Array): void; + flush?(timeoutMs?: number): Promise; close(): void; on( eventName: EventName, diff --git a/packages/cultnet-ts/test/cultnet.test.ts b/packages/cultnet-ts/test/cultnet.test.ts index 46e7cb6f..2f540a86 100644 --- a/packages/cultnet-ts/test/cultnet.test.ts +++ b/packages/cultnet-ts/test/cultnet.test.ts @@ -12,7 +12,7 @@ import { CultCache, SingleFileMessagePackBackingStore, defineDocumentType, -} from "cultcache-ts"; +} from "@gamecult/cultcache-ts"; import { CultNetClientSecurityOptions, @@ -21,6 +21,7 @@ import { CultNetRudpReconnectLoop, CultNetRudpSession, CultNetRudpSocketTransportConnection, + CULTNET_RUDP_RELIABLE_SEND_WINDOW_PACKETS, CultNetSchemaCatalog, CultNetSchemaRegistry, CultNetSecret, @@ -58,6 +59,64 @@ import { type InteropNote, } from "./interop/cultnet-interop-shared"; +test("CultNet validates the canonical CultMesh Verse catalog messages", () => { + const request = parseCultNetMessage({ + schemaVersion: "cultmesh.verse_catalog_request.v0", + messageId: "catalog-1", + verseIds: ["sample.counter"], + transportVersion: "cultmesh.v1", + }); + assert.equal(request.schemaVersion, "cultmesh.verse_catalog_request.v0"); + + const response = parseCultNetMessage({ + schemaVersion: "cultmesh.verse_catalog_response.v0", + messageId: "catalog-1", + verses: [{ + verseId: "sample.counter", + displayName: "Counter", + authorityModel: "OperatorCluster", + compatibility: { + transportVersion: "cultmesh.v1", + rulesHash: "counter-v1", + compatibleVerseIds: [], + requiredPluginIds: [], + optionalPluginIds: [], + }, + discoveryEndpoints: ["wss://provider.example/cultmesh"], + authorityRuntimeIds: ["sample.counter-provider"], + authorityRoutes: [{ + authorityRuntimeId: "sample.counter-provider", + endpoint: "wss://provider.example/cultmesh", + protocolIds: ["cultmesh.documents.v1"], + priority: 0, + generation: "provider-route-1", + }], + }], + }); + assert.equal(response.schemaVersion, "cultmesh.verse_catalog_response.v0"); + assert.equal(response.verses[0].authorityRuntimeIds[0], "sample.counter-provider"); + assert.equal(response.verses[0].authorityRoutes![0].generation, "provider-route-1"); + + const session = parseCultNetMessage({ + schemaVersion: "cultmesh.session_accepted.v2", + messageId: "session-1", + accepted: true, + verseId: "sample.counter", + authorityRuntimeId: "sample.counter-provider", + protocolId: "cultmesh.documents.v1", + routeGeneration: "provider-route-1", + clientNonce: "bm9uY2U=", + providerKeyId: "provider-key-1", + providerSignature: "c2lnbmF0dXJl", + }); + assert.equal(session.schemaVersion, "cultmesh.session_accepted.v2"); + + assert.throws(() => parseCultNetMessage({ + ...response, + verses: [{ ...response.verses[0], compatibility: { ...response.verses[0].compatibility, rulesHash: "" } }], + }), /rulesHash/); +}); + class FakeRudpReconnectTransport extends EventEmitter { public connectCalls = 0; public closeCalls = 0; @@ -494,6 +553,53 @@ test("rudp session pings and detects receive timeout", () => { assert.equal(client.connected, false); }); +test("rudp lossy packets cannot create reliable ordered gaps", () => { + const client = new CultNetRudpSession({ connectionId: 198, initialSequence: 1 }); + const server = new CultNetRudpSession({ connectionId: 198, initialSequence: 100 }); + const connect = client.createConnect(0); + const accept = server.acceptConnect(connect, 1); + client.receive(accept, 2); + + const ping = client.createPing(Buffer.from("pulse")); + const pong = server.receive(ping, 3).reply!; + const realtime = server.send("realtime", Buffer.from("discarded realtime")); + const latest = server.send("latest", Buffer.from("discarded latest state"), { sequenced: true }); + const schema = server.send("schema", Buffer.from("committed response"), { reliable: true, ordered: true }); + + assert.equal(ping.sequence, 0); + assert.equal(pong.sequence, 0); + assert.equal(realtime.sequence, 0); + assert.equal(latest.sequence, 1); + assert.equal(schema.sequence, accept.sequence + 1); + assert.deepEqual(client.receive(schema, 4).delivered.map(frame => Buffer.from(frame.payload).toString("utf8")), [ + "committed response", + ]); +}); + +test("rudp unreliable sequenced delivery is scoped to its channel", () => { + const sender = new CultNetRudpSession({ connectionId: 197, initialSequence: 50 }); + const receiver = new CultNetRudpSession({ connectionId: 197, initialSequence: 100 }); + const connect = sender.createConnect(0); + const accept = receiver.acceptConnect(connect, 1); + sender.receive(accept, 2); + const older = sender.send("latest", Buffer.from("older"), { sequenced: true }); + const newer = sender.send("latest", Buffer.from("newer"), { sequenced: true }); + + assert.equal(older.sequence, 1); + assert.equal(newer.sequence, 2); + assert.deepEqual(receiver.receive(newer, 3).delivered.map(frame => Buffer.from(frame.payload).toString("utf8")), ["newer"]); + assert.deepEqual(receiver.receive(older, 4).delivered, []); +}); + +test("rudp rejects unreliable ordered delivery", () => { + const session = new CultNetRudpSession({ connectionId: 196 }); + session.receive({ packetType: "accept", connectionId: 196, sequence: 50, ack: 0, ackMask: 0, channelId: "control" }); + assert.throws( + () => session.send("schema", Buffer.from("cannot order what will not retransmit"), { ordered: true }), + /ordered delivery requires reliability/, + ); +}); + test("rudp session bounds pending reliable packets before enqueue", () => { const session = new CultNetRudpSession({ connectionId: 102, initialSequence: 1, maxPendingReliablePackets: 2 }); session.receive({ packetType: "accept", connectionId: 102, sequence: 50, ack: 0, ackMask: 0, channelId: "control" }); @@ -821,9 +927,9 @@ test("CultNet can round-trip gamecult.networking.v0 auth messages through the ex assert.deepEqual(wireValue, [ 0, [ - Buffer.from("nonce", "utf8"), - Buffer.from("auth", "utf8"), - Buffer.from("password", "utf8"), + new TextEncoder().encode("nonce"), + new TextEncoder().encode("auth"), + new TextEncoder().encode("password"), ], ]); @@ -1261,6 +1367,124 @@ test("operation envelopes preserve typed service routing and payload correlation assert.equal(response.messageId, request.messageId); }); +test("rudp session advances large fragment sets through a bounded reliable window", () => { + const connectionId = 457; + const sender = new CultNetRudpSession({ connectionId, initialSequence: 1, resendDelayMs: 25 }); + const receiver = new CultNetRudpSession({ connectionId, initialSequence: 100, resendDelayMs: 25 }); + sender.receive({ packetType: "accept", connectionId, sequence: 0, ack: 0, ackMask: 0, channelId: "control" }); + receiver.receive({ packetType: "accept", connectionId, sequence: 0, ack: 0, ackMask: 0, channelId: "control" }); + + const fragmentCount = CULTNET_RUDP_RELIABLE_SEND_WINDOW_PACKETS + 17; + const payload = Uint8Array.from( + { length: fragmentCount * 8 }, + (_, index) => index % 251, + ); + const wire = sender.sendMany("schema", payload, { + reliable: true, + ordered: true, + nowMs: 1, + maxFragmentBytes: 8, + }); + assert.equal(wire.length, CULTNET_RUDP_RELIABLE_SEND_WINDOW_PACKETS); + const oldestSequence = wire[0]!.sequence; + assert.equal(sender.pendingReliableSequences.length, wire.length); + assert.equal(sender.queuedReliablePacketCount, 17); + + const delivered = []; + while (wire.length > 0) { + const admitted = wire.length; + for (let index = 0; index < admitted; index += 1) { + const packet = wire.shift()!; + delivered.push(...receiver.receive(packet, 2).delivered); + } + const acknowledged = sender.receive(receiver.createAck(), 3); + wire.push(...(acknowledged.readyToSend ?? [])); + } + + assert.equal(sender.outstandingReliablePacketCount, 0); + assert.equal(delivered.length, 1); + assert.deepEqual(delivered[0]?.payload, payload); + const oldAck = receiver.createAckForReceived(oldestSequence); + assert.equal(oldAck.ack, oldestSequence); + assert.equal(oldAck.ackMask, 0); +}); + +test("CultNet contracts encode legacy bytes without Node Buffer authority", () => { + const originalBuffer = globalThis.Buffer; + try { + Object.defineProperty(globalThis, "Buffer", { configurable: true, value: undefined }); + const message = parseCultNetMessage({ + schemaVersion: "cultnet.login.v0", + nonce: "AQID", + auth: "BAUG", + password: "BwgJ", + }); + const wire = encodeCultNetMessageForWire(message, "gamecult.networking.v0") as [number, Uint8Array[]]; + assert.equal(wire[0], 0); + assert.deepEqual([...wire[1][0]], [1, 2, 3]); + } finally { + Object.defineProperty(globalThis, "Buffer", { configurable: true, value: originalBuffer }); + } +}); + +test("database subscription contracts match the C# live document lane", () => { + const subscribe = parseCultNetMessage({ + schemaVersion: "cultnet.database_subscribe.v0", + messageId: "subscribe-counter", + subscriptionId: "counter:browser-1", + schemaIds: ["sample.counter_state.v1"], + recordKeys: ["counter:main"], + includeSnapshot: true, + consumerRuntimeId: "browser-1", + }); + assert.equal(subscribe.schemaVersion, "cultnet.database_subscribe.v0"); + assert.equal(subscribe.subscriptionId, "counter:browser-1"); + + const added = parseCultNetMessage({ + schemaVersion: "cultnet.database_change_raw.v0", + messageId: "change-counter-1", + subscriptionId: "counter:browser-1", + changeKind: "added", + document: { + schemaId: "sample.counter_state.v1", + recordKey: "counter:main", + storedAt: "2026-08-17T00:00:00Z", + payloadEncoding: "messagepack", + payload: new Uint8Array([0x81, 0xa5, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x01]), + }, + }); + assert.equal(added.schemaVersion, "cultnet.database_change_raw.v0"); + assert.equal(added.document?.recordKey, "counter:main"); + + const removed = parseCultNetMessage({ + schemaVersion: "cultnet.database_change_raw.v0", + messageId: "change-counter-2", + subscriptionId: "counter:browser-1", + changeKind: "removed", + recordKey: "counter:main", + schemaId: "sample.counter_state.v1", + }); + assert.equal(removed.schemaVersion, "cultnet.database_change_raw.v0"); + assert.equal(removed.recordKey, "counter:main"); + + const unsubscribe = parseCultNetMessage({ + schemaVersion: "cultnet.database_unsubscribe.v0", + messageId: "unsubscribe-counter", + subscriptionId: "counter:browser-1", + }); + assert.equal(unsubscribe.schemaVersion, "cultnet.database_unsubscribe.v0"); + + assert.throws( + () => parseCultNetMessage({ + schemaVersion: "cultnet.database_change_raw.v0", + messageId: "invalid-change", + subscriptionId: "counter:browser-1", + changeKind: "updated", + }), + /document: is required/, + ); +}); + test("RUDP operation service correlates typed payload envelopes", async () => { const server = await startCultNetOperationServer({ runtimeId: "sai-sidecar", @@ -1597,3 +1821,49 @@ test("Generated Ghostlight contracts can feed CultCacheTS directly without a Zod rmSync(tempDir, { recursive: true, force: true }); } }); + +test("rudp receiver evicts stranded fragment sets instead of refusing new ones", () => { + // A fragment set is removed only on successful reassembly, so every set that + // loses a fragment is stranded for the life of the session. Ghostlight sends + // Heimdall fragmented envelopes (max_fragment_bytes = 2048), so on the live + // auth path this map grew for every lost fragment and was never swept. + const receiver = new CultNetRudpSession({ connectionId: 9, initialSequence: 1, resendDelayMs: 50 }); + receiver.receive({ packetType: "accept", connectionId: 9, sequence: 1, ack: 0, ackMask: 0, channelId: "control" }); + + // 200 fragment sets that each lose their second fragment. + for (let id = 1; id <= 200; id += 1) { + const stranded = receiver.receive({ + packetType: "data", + connectionId: 9, + sequence: 100 + id, + ack: 0, + ackMask: 0, + channelId: "schema", + fragmentId: id, + fragmentIndex: 0, + fragmentCount: 2, + payload: Buffer.from([id & 0xff]), + }); + assert.equal(stranded.delivered.length, 0, "an incomplete set must not reassemble"); + } + + // Bounded rather than unbounded, and still serving. + assert.ok(receiver.pendingFragmentSetCount <= 64, `pending ${receiver.pendingFragmentSetCount}`); + assert.ok(receiver.fragmentSetsEvicted > 0, "nothing was evicted"); + + // The receiver still completes a fresh fragmented payload, which is what + // erroring at the cap took away: any nonzero loss killed the channel. + const first = receiver.receive({ + packetType: "data", connectionId: 9, sequence: 900, ack: 0, ackMask: 0, + channelId: "schema", fragmentId: 1000, fragmentIndex: 0, fragmentCount: 2, + payload: Buffer.from("he"), + }); + assert.equal(first.delivered.length, 0); + const second = receiver.receive({ + packetType: "data", connectionId: 9, sequence: 901, ack: 0, ackMask: 0, + channelId: "schema", fragmentId: 1000, fragmentIndex: 1, fragmentCount: 2, + payload: Buffer.from("llo"), + }); + assert.equal(second.delivered.length, 1); + assert.equal(Buffer.from(second.delivered[0]!.payload).toString(), "hello"); +}); diff --git a/packages/cultnet-ts/test/fixtures/runtime-presence-vectors.json b/packages/cultnet-ts/test/fixtures/runtime-presence-vectors.json new file mode 100644 index 00000000..043689f4 --- /dev/null +++ b/packages/cultnet-ts/test/fixtures/runtime-presence-vectors.json @@ -0,0 +1,4 @@ +{ + "proof": "dc0018d92367616d6563756c742e72756e74696d655f70726573656e63655f6865616c74682e7632a86865696d64616c6cd9477368613235362d61616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161d9477368613235362d62626262626262626262626262626262626262626262626262626262626262626262626262626262626262626262626262626262626262626262626262626262b5736572766963652d696e6361726e6174696f6e2d31d9477368613235362d63636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363d9477368613235362d64646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464a27631d9477368613235362d65656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565b26865696d64616c6c2d79676764726173696cd9477368613235362d66666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666b6687474703a2f2f3132372e302e302e313a31343130319294af6865696d64616c6c2e616363657373b26865696d64616c6c2e6163636573732e7631a276310294ac7a6574612e72756e74696d65af7a6574612e72756e74696d652e7631a2763101d9256865696d64616c6c2e63756c746e65742d727564702d70726f76696465722d6865616c7468a77761726d696e67ba4865696d64616c6c2063616e646964617465207761726d696e67c0d9403131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313107cf00000199155c6200a765643235353139c400d94032323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232c400", + "signed": "dc0018d92367616d6563756c742e72756e74696d655f70726573656e63655f6865616c74682e7632a86865696d64616c6cd9477368613235362d61616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161d9477368613235362d62626262626262626262626262626262626262626262626262626262626262626262626262626262626262626262626262626262626262626262626262626262b5736572766963652d696e6361726e6174696f6e2d31d9477368613235362d63636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363d9477368613235362d64646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464a27631d9477368613235362d65656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565b26865696d64616c6c2d79676764726173696cd9477368613235362d66666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666b6687474703a2f2f3132372e302e302e313a31343130319294af6865696d64616c6c2e616363657373b26865696d64616c6c2e6163636573732e7631a276310294ac7a6574612e72756e74696d65af7a6574612e72756e74696d652e7631a2763101d9256865696d64616c6c2e63756c746e65742d727564702d70726f76696465722d6865616c7468a77761726d696e67ba4865696d64616c6c2063616e646964617465207761726d696e67c0d9403131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313107cf00000199155c6200a765643235353139c44011111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111d94032323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232c44022222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222" +} diff --git a/packages/cultnet-ts/test/interop/cultnet-interop-peer.ts b/packages/cultnet-ts/test/interop/cultnet-interop-peer.ts index 0af43301..57b7fa37 100644 --- a/packages/cultnet-ts/test/interop/cultnet-interop-peer.ts +++ b/packages/cultnet-ts/test/interop/cultnet-interop-peer.ts @@ -6,7 +6,7 @@ import process from "node:process"; import { setTimeout as delay } from "node:timers/promises"; import { decode, encode } from "@msgpack/msgpack"; -import { CultCache, SingleFileMessagePackBackingStore, defineDocumentType, type AnyCultCacheDocumentDefinition } from "cultcache-ts"; +import { CultCache, SingleFileMessagePackBackingStore, defineDocumentType, type AnyCultCacheDocumentDefinition } from "@gamecult/cultcache-ts"; import { CultNetDocumentRegistry, CultNetPeer, diff --git a/packages/cultnet-ts/test/interop/cultnet-interop.test.ts b/packages/cultnet-ts/test/interop/cultnet-interop.test.ts index bec86942..00fff83d 100644 --- a/packages/cultnet-ts/test/interop/cultnet-interop.test.ts +++ b/packages/cultnet-ts/test/interop/cultnet-interop.test.ts @@ -24,7 +24,9 @@ const pythonCommand = process.env.PYTHON ?? resolvePythonCommand(); const cultNetTsRoot = resolve(__dirname, "../../.."); const cultLibRoot = findAncestor(cultNetTsRoot, "CultLib.sln") ?? resolve(cultNetTsRoot, "..", "CultLib"); const cultcachePyRoot = resolve(cultLibRoot, "packages", "cultcache-py"); -const cultcachePySrc = resolve(cultcachePyRoot, "src"); +const cultcachePySrc = ["cultcache-py", "cultnet-py", "cultmesh-py"] + .map((name) => resolve(cultLibRoot, "packages", name, "src")) + .join(delimiter); const cultmeshKotlinRoot = resolve(cultLibRoot, "packages", "cultmesh-kotlin"); const kotlinHome = process.env.KOTLIN_HOME ?? "C:\\Program Files\\Android\\Android Studio\\plugins\\Kotlin\\kotlinc"; const kotlinJavaHome = process.env.KOTLIN_JAVA_HOME ?? "C:\\Program Files\\Android\\Android Studio\\jbr"; @@ -414,6 +416,12 @@ test("CultNet TS/Rust/C#/Python peers discover each other and exchange raw state const advertiseHost = findAdvertiseHost(); const servers: RunningServeProcess[] = []; + // Registered before anything is spawned. A peer whose readiness rejects + // would otherwise skip this and strand every peer already running. + t.after(async () => { + await Promise.all(servers.map(stopProcess)); + }); + logInteropPhase("schema-v0", "start rust peer"); servers.push(await spawnServeProcess("rust", { command: rustBinaryPath, @@ -498,10 +506,6 @@ test("CultNet TS/Rust/C#/Python peers discover each other and exchange raw state })); await servers[servers.length - 1].ready; - t.after(async () => { - await Promise.all(servers.map(stopProcess)); - }); - logInteropPhase("schema-v0", "typescript dials rust"); const tsDial = await runJsonCommand("ts-dial", process.execPath, [ tsPeerScript, @@ -989,6 +993,12 @@ test("CultNet TypeScript and Rust full interop peers exchange schema-v0 over RUD const discoveryPort = await getFreePort(); const advertiseHost = findAdvertiseHost(); const servers: RunningServeProcess[] = []; + // Registered before anything is spawned. A peer whose readiness rejects + // would otherwise skip this and strand every peer already running. + t.after(async () => { + await Promise.all(servers.map(stopProcess)); + }); + servers.push(await spawnServeProcess("rust-rudp", { command: rustBinaryPath, args: [ @@ -1030,10 +1040,6 @@ test("CultNet TypeScript and Rust full interop peers exchange schema-v0 over RUD })); await servers[servers.length - 1].ready; - t.after(async () => { - await Promise.all(servers.map(stopProcess)); - }); - logInteropPhase("ts-rust-full-rudp", "typescript dials rust over rudp"); const tsRudpDial = await runJsonCommand("ts-rust-rudp-dial", process.execPath, [ tsPeerScript, @@ -1139,6 +1145,7 @@ test("CultNet Python and TypeScript exchange schema frames when Python dials a T assert.equal(exitCode, 0, pythonClient.stderr.join("")); } finally { server.close(); + closeUdpSocket(serverSocket); if (pythonClient.child.exitCode === null && !pythonClient.child.killed) { pythonClient.child.kill("SIGTERM"); await once(pythonClient.child, "exit").catch(() => undefined); @@ -1246,6 +1253,7 @@ test("CultNet Python and TypeScript exchange schema-v0 MessagePack messages over assert.equal(exitCode, 0, pythonClient.stderr.join("")); } finally { server.close(); + closeUdpSocket(serverSocket); if (pythonClient.child.exitCode === null && !pythonClient.child.killed) { pythonClient.child.kill("SIGTERM"); await once(pythonClient.child, "exit").catch(() => undefined); @@ -1322,6 +1330,7 @@ test("CultNet Rust and TypeScript exchange schema frames when Rust dials a TypeS assert.equal(exitCode, 0, rustClient.stderr.join("")); } finally { server.close(); + closeUdpSocket(serverSocket); if (rustClient.child.exitCode === null && !rustClient.child.killed) { rustClient.child.kill("SIGTERM"); await once(rustClient.child, "exit").catch(() => undefined); @@ -1431,6 +1440,7 @@ test("CultNet Rust and TypeScript exchange schema-v0 MessagePack messages over R assert.equal(exitCode, 0, rustClient.stderr.join("")); } finally { server.close(); + closeUdpSocket(serverSocket); if (rustClient.child.exitCode === null && !rustClient.child.killed) { rustClient.child.kill("SIGTERM"); await once(rustClient.child, "exit").catch(() => undefined); @@ -1507,6 +1517,7 @@ test("CultNet C# and TypeScript exchange schema frames when C# dials a TypeScrip assert.equal(exitCode, 0, csharpClient.stderr.join("")); } finally { server.close(); + closeUdpSocket(serverSocket); if (csharpClient.child.exitCode === null && !csharpClient.child.killed) { csharpClient.child.kill("SIGTERM"); await once(csharpClient.child, "exit").catch(() => undefined); @@ -1616,6 +1627,7 @@ test("CultNet C# and TypeScript exchange schema-v0 MessagePack messages over RUD assert.equal(exitCode, 0, csharpClient.stderr.join("")); } finally { server.close(); + closeUdpSocket(serverSocket); if (csharpClient.child.exitCode === null && !csharpClient.child.killed) { csharpClient.child.kill("SIGTERM"); await once(csharpClient.child, "exit").catch(() => undefined); @@ -1692,6 +1704,7 @@ test("CultNet Kotlin and TypeScript exchange schema frames when Kotlin dials a T assert.equal(exitCode, 0, kotlinClient.stderr.join("")); } finally { server.close(); + closeUdpSocket(serverSocket); if (kotlinClient.child.exitCode === null && !kotlinClient.child.killed) { kotlinClient.child.kill("SIGTERM"); await once(kotlinClient.child, "exit").catch(() => undefined); @@ -1801,6 +1814,7 @@ test("CultNet Kotlin and TypeScript exchange schema-v0 MessagePack messages over assert.equal(exitCode, 0, kotlinClient.stderr.join("")); } finally { server.close(); + closeUdpSocket(serverSocket); if (kotlinClient.child.exitCode === null && !kotlinClient.child.killed) { kotlinClient.child.kill("SIGTERM"); await once(kotlinClient.child, "exit").catch(() => undefined); @@ -1823,6 +1837,12 @@ test("CultNet TypeScript and C# full interop peers exchange schema-v0 over RUDP" const discoveryPort = await getFreePort(); const advertiseHost = findAdvertiseHost(); const servers: RunningServeProcess[] = []; + // Registered before anything is spawned. A peer whose readiness rejects + // would otherwise skip this and strand every peer already running. + t.after(async () => { + await Promise.all(servers.map(stopProcess)); + }); + servers.push(await spawnServeProcess("csharp-rudp", { command: dotnetCommand, args: [ @@ -1865,10 +1885,6 @@ test("CultNet TypeScript and C# full interop peers exchange schema-v0 over RUDP" })); await servers[servers.length - 1].ready; - t.after(async () => { - await Promise.all(servers.map(stopProcess)); - }); - logInteropPhase("ts-csharp-full-rudp", "typescript dials csharp over rudp"); const tsRudpDial = await runJsonCommand("ts-csharp-rudp-dial", process.execPath, [ tsPeerScript, @@ -1923,6 +1939,12 @@ test("CultNet TypeScript and Kotlin full interop peers exchange schema-v0 over R const discoveryPort = await getFreePort(); const advertiseHost = findAdvertiseHost(); const servers: RunningServeProcess[] = []; + // Registered before anything is spawned. A peer whose readiness rejects + // would otherwise skip this and strand every peer already running. + t.after(async () => { + await Promise.all(servers.map(stopProcess)); + }); + servers.push(await spawnServeProcess("kotlin-rudp", { command: kotlinJavaCommand, args: [ @@ -1966,10 +1988,6 @@ test("CultNet TypeScript and Kotlin full interop peers exchange schema-v0 over R })); await servers[servers.length - 1].ready; - t.after(async () => { - await Promise.all(servers.map(stopProcess)); - }); - logInteropPhase("ts-kotlin-full-rudp", "typescript dials kotlin over rudp"); const tsRudpDial = await runJsonCommand("ts-kotlin-rudp-dial", process.execPath, [ tsPeerScript, @@ -3160,6 +3178,20 @@ function spawnRustRudpMessageClient(remotePort: number): Omit { const socket = dgram.createSocket("udp4"); await new Promise((resolveBind, rejectBind) => { diff --git a/packages/cultnet-ts/test/runtime-presence-health.test.ts b/packages/cultnet-ts/test/runtime-presence-health.test.ts new file mode 100644 index 00000000..fa21aeaa --- /dev/null +++ b/packages/cultnet-ts/test/runtime-presence-health.test.ts @@ -0,0 +1,123 @@ +// Cross-fork conformance for `gamecult.runtime_presence_health.v2`. +// +// The vectors below were produced by `cultnet-rs`, not written by hand: +// +// cargo run --example runtime_presence_vector -p cultnet-rs +// +// That matters. A fixture authored for both sides at once agrees with itself +// and proves nothing; the contract is only real if one fork emits the bytes and +// the other reproduces them. Regenerate these vectors from the Rust example if +// the record ever changes, and never by editing them to match this code. + +import test from "node:test"; +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { join } from "node:path"; + +import { + RUNTIME_PRESENCE_FIELD_COUNT, + encodeRuntimePresenceHealth, + runtimePresenceProofPayload, + validateRuntimePresence, + type RuntimePresenceHealth, +} from "../src/runtime-presence-health"; + +// Read from the source tree, not a copy beside the compiled test: the vectors +// are generated output and must have exactly one home. +const VECTORS = JSON.parse( + readFileSync( + join(__dirname, "..", "..", "test", "fixtures", "runtime-presence-vectors.json"), + "utf8", + ), +) as { proof: string; signed: string }; + +/** The same statement the Rust example builds. */ +function referencePresence(): RuntimePresenceHealth { + return { + target: "heimdall", + expectedProjectionSha256: `sha256-${"a".repeat(64)}`, + planId: `sha256-${"b".repeat(64)}`, + incarnationId: "service-incarnation-1", + sealedReleaseId: `sha256-${"c".repeat(64)}`, + activationWitnessSha256: `sha256-${"d".repeat(64)}`, + stateSchemaGeneration: "v1", + stateContractSha256: `sha256-${"e".repeat(64)}`, + runtimeId: "heimdall-yggdrasil", + runtimeInstanceId: `sha256-${"f".repeat(64)}`, + boundEndpoint: "http://127.0.0.1:14101", + capabilities: [ + { + capability: "heimdall.access", + schema: "heimdall.access.v1", + compatibility: "v1", + capacity: 2, + }, + { capability: "zeta.runtime", schema: "zeta.runtime.v1", compatibility: "v1", capacity: 1 }, + ], + healthContract: "heimdall.cultnet-rudp-provider-health", + state: "warming", + detail: "Heimdall candidate warming", + writeLeaseSha256: null, + signerIdentityId: "1".repeat(64), + publisherSequence: 7, + observedAtUnixMillis: 1_757_000_000_000, + activationSignerIdentityId: "2".repeat(64), + }; +} + +function hex(bytes: Uint8Array): string { + return Buffer.from(bytes).toString("hex"); +} + +test("the proof payload is byte-identical to the one cultnet-rs signs", () => { + assert.equal(hex(runtimePresenceProofPayload(referencePresence())), VECTORS.proof); +}); + +test("the signed record carries both proofs in their own slots", () => { + const signed = encodeRuntimePresenceHealth( + referencePresence(), + new Uint8Array(64).fill(0x11), + new Uint8Array(64).fill(0x22), + ); + // Byte-identical to the fully signed record cultnet-rs emits, so the proofs + // land in their own slots rather than being appended. + assert.equal(hex(signed), VECTORS.signed); +}); + +test("capabilities are ordered canonically regardless of caller order", () => { + const presence = referencePresence(); + const reversed = { ...presence, capabilities: [...presence.capabilities].reverse() }; + assert.equal( + hex(runtimePresenceProofPayload(reversed)), + hex(runtimePresenceProofPayload(presence)), + ); +}); + +test("the tuple is always the full contract width", () => { + assert.equal(RUNTIME_PRESENCE_FIELD_COUNT, 24); +}); + +test("statements the contract forbids are refused before they are signed", () => { + const warmingWithLease = { + ...referencePresence(), + writeLeaseSha256: `sha256-${"0".repeat(64)}`, + }; + assert.throws(() => validateRuntimePresence(warmingWithLease), /cannot claim a process write/); + + const statelessWithLease = { + ...referencePresence(), + state: "active" as const, + stateSchemaGeneration: null, + stateContractSha256: null, + writeLeaseSha256: `sha256-${"0".repeat(64)}`, + }; + assert.throws(() => validateRuntimePresence(statelessWithLease), /stateless runtime presence/); + + assert.throws( + () => validateRuntimePresence({ ...referencePresence(), publisherSequence: 0 }), + /publisher sequence/, + ); + + const partialLineage = { ...referencePresence(), stateContractSha256: null }; + assert.throws(() => validateRuntimePresence(partialLineage), /state lineage is partial/); +}); diff --git a/packages/gamecult-geometry-rs/Cargo.toml b/packages/gamecult-geometry-rs/Cargo.toml deleted file mode 100644 index b05c0246..00000000 --- a/packages/gamecult-geometry-rs/Cargo.toml +++ /dev/null @@ -1,17 +0,0 @@ -[package] -name = "gamecult-geometry" -version = "0.1.0" -edition = "2024" -description = "GameCult.Geometry typed contracts and geometry runtime for Rust" -license = "MIT" -repository = "https://github.com/GameCult/CultLib" -readme = "README.md" - -[dependencies] -bevy_math = "0.17" -rmp-serde = "1" -serde = { version = "1", features = ["derive"] } -sha2 = "0.10" - -[dev-dependencies] -approx = "0.5" diff --git a/packages/gamecult-geometry-rs/LICENSE b/packages/gamecult-geometry-rs/LICENSE deleted file mode 100644 index a431c3ab..00000000 --- a/packages/gamecult-geometry-rs/LICENSE +++ /dev/null @@ -1,22 +0,0 @@ -MIT License - -Copyright (c) 2026 GameCult - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - diff --git a/packages/gamecult-geometry-rs/README.md b/packages/gamecult-geometry-rs/README.md deleted file mode 100644 index 0b3ec053..00000000 --- a/packages/gamecult-geometry-rs/README.md +++ /dev/null @@ -1,11 +0,0 @@ -# GameCult.Geometry for Rust - -Rust projection of the GameCult.Geometry v2 document contract. It owns the -typed MessagePack layouts and stable geometry record identities used at the -Rust boundary; it does not depend on VibeGeometry or `vg-csg`. - -The initial contract and stable-hash implementation were mined from the -MIT-licensed GameCult `vg-csg` crate at commit `8f070f4`. The implementation -was rewritten here under GameCult.Geometry ownership. The original copyright -and MIT terms are preserved in [LICENSE](LICENSE). - diff --git a/packages/gamecult-geometry-rs/src/assembler.rs b/packages/gamecult-geometry-rs/src/assembler.rs deleted file mode 100644 index 078a07b5..00000000 --- a/packages/gamecult-geometry-rs/src/assembler.rs +++ /dev/null @@ -1,1509 +0,0 @@ -#![allow(dead_code)] -use std::cell::RefCell; - -use bevy_math::Vec3; - -use crate::{ - Aabb, Brush, BrushId, BrushOp, ConvexPolygon, ConvexSolid, DirtyDemandFrontier, MaterialId, - PolygonRouteScratch, Primitive, TriangleMesh, append_cylinder_z, append_dome_cap_z, - append_floret_arm, - primitives::{DomeCapZSpec, FloretArmSpec}, -}; - -#[derive(Clone, Debug, PartialEq, Eq)] -pub enum BuildWarning { - SubtractIgnoredForNonBox { brush: String }, - IntersectIgnoredForNonBox { brush: String }, -} - -#[derive(Clone, Debug, Default, PartialEq, Eq)] -pub struct BuildReport { - pub input_brushes: usize, - pub emitted_convex_fragments: usize, - pub operator_brushes: usize, - pub candidate_pairs: usize, - pub rejected_pairs: usize, - pub reused_mesh: bool, - pub used_checkpoint: bool, - pub checkpoint_rebuild_from: Option, - pub direct_dirty_rebuild: bool, - pub warnings: Vec, -} - -#[derive(Clone, Debug)] -pub struct BuildOutput { - pub mesh: TriangleMesh, - pub report: BuildReport, - pub generation: u64, -} - -#[derive(Clone, Debug, Default)] -pub struct Assembler { - brushes: Vec, - compiled: Vec, - cache: RefCell>, - box_cache: RefCell>, - general_cache: RefCell>, - first_dirty_index: RefCell>, - next_id: u32, - generation: u64, -} - -#[derive(Clone, Debug)] -struct CompiledBrush { - op: BrushOp, - material: MaterialId, - bounds: Aabb, - geometry: CompiledGeometry, - name: String, -} - -#[derive(Clone, Debug)] -enum CompiledGeometry { - Box(Aabb), - Convex(ConvexSolid), - CylinderZ { - center: Vec3, - radius: f32, - depth: f32, - segments: usize, - }, - DomeCapZ { - center: Vec3, - radius: f32, - height: f32, - rings: usize, - segments: usize, - }, - FloretArm { - anchor: Vec3, - direction: Vec3, - length: f32, - root_width: f32, - tip_width: f32, - thickness: f32, - tip_lift: f32, - }, -} - -#[derive(Clone, Debug, Default)] -struct BoxBuildCache { - generation: u64, - valid_checkpoint_index: usize, - checkpoints: Vec, -} - -#[derive(Clone, Debug, Default, PartialEq)] -struct BoxBuildState { - boxes: Vec<(Aabb, MaterialId)>, - report: BuildReport, -} - -#[derive(Clone, Debug, Default)] -struct GeneralBuildCache { - generation: u64, - valid_checkpoint_index: usize, - checkpoints: Vec, -} - -#[derive(Clone, Debug, Default, PartialEq)] -struct GeneralBuildState { - mesh: TriangleMesh, - solids: Vec, - report: BuildReport, -} - -impl Assembler { - pub fn new() -> Self { - Self::default() - } - - pub fn brushes(&self) -> &[Brush] { - &self.brushes - } - - pub fn generation(&self) -> u64 { - self.generation - } - - pub fn brush_index(&self, id: BrushId) -> Option { - self.brushes.iter().position(|brush| brush.id == id) - } - - pub fn dirty_frontier_for_indices( - &self, - dirty_indices: impl IntoIterator, - ) -> DirtyDemandFrontier { - DirtyDemandFrontier::from_dirty_indices(&self.brushes, dirty_indices) - } - - pub fn dirty_frontier_for_brushes( - &self, - dirty_brushes: impl IntoIterator, - ) -> DirtyDemandFrontier { - let indices = dirty_brushes - .into_iter() - .filter_map(|id| self.brush_index(id)) - .collect::>(); - self.dirty_frontier_for_indices(indices) - } - - pub fn add_brush( - &mut self, - name: impl Into, - op: BrushOp, - primitive: Primitive, - material: MaterialId, - ) -> BrushId { - let id = BrushId(self.next_id); - self.next_id += 1; - let index = self.brushes.len(); - let brush = Brush::new(id, name, op, primitive, material); - self.compiled.push(CompiledBrush::from_brush(&brush)); - self.brushes.push(brush); - self.generation = self.generation.wrapping_add(1); - self.mark_dirty_index(index); - id - } - - pub fn set_brush_primitive(&mut self, id: BrushId, primitive: Primitive) -> bool { - let Some(index) = self.brush_index(id) else { - return false; - }; - self.brushes[index].primitive = primitive; - self.brushes[index].set_dirty(); - self.compiled[index] = CompiledBrush::from_brush(&self.brushes[index]); - self.generation = self.generation.wrapping_add(1); - self.mark_dirty_index(index); - true - } - - pub fn set_brush_op(&mut self, id: BrushId, op: BrushOp) -> bool { - let Some(index) = self.brush_index(id) else { - return false; - }; - self.brushes[index].op = op; - self.brushes[index].set_dirty(); - self.compiled[index] = CompiledBrush::from_brush(&self.brushes[index]); - self.generation = self.generation.wrapping_add(1); - self.mark_dirty_index(index); - true - } - - pub fn solid_box( - &mut self, - name: impl Into, - bounds: Aabb, - material: MaterialId, - ) -> BrushId { - self.add_brush(name, BrushOp::Add, Primitive::Box { bounds }, material) - } - - pub fn cut_box(&mut self, name: impl Into, bounds: Aabb) -> BrushId { - self.add_brush( - name, - BrushOp::Subtract, - Primitive::Box { bounds }, - MaterialId::default(), - ) - } - - pub fn solid_oriented_box( - &mut self, - name: impl Into, - center: Vec3, - size: Vec3, - rotation: bevy_math::Quat, - material: MaterialId, - ) -> BrushId { - self.add_brush( - name, - BrushOp::Add, - Primitive::OrientedBox { - center, - size, - rotation, - }, - material, - ) - } - - pub fn cut_oriented_box( - &mut self, - name: impl Into, - center: Vec3, - size: Vec3, - rotation: bevy_math::Quat, - ) -> BrushId { - self.add_brush( - name, - BrushOp::Subtract, - Primitive::OrientedBox { - center, - size, - rotation, - }, - MaterialId::default(), - ) - } - - pub fn build(&self) -> BuildOutput { - if let Some(output) = self - .cache - .borrow() - .as_ref() - .filter(|output| output.generation == self.generation) - { - return output.clone(); - } - - let output = self.build_with_checkpoints(); - *self.cache.borrow_mut() = Some(output.clone()); - *self.first_dirty_index.borrow_mut() = None; - output - } - - pub fn build_incremental(&self) -> BuildOutput { - if let Some(output) = self - .cache - .borrow() - .as_ref() - .filter(|output| output.generation == self.generation) - { - return output.clone(); - } - - let first_dirty_index = *self.first_dirty_index.borrow(); - let output = self - .try_build_axis_aligned_boxes_from_cache(first_dirty_index) - .unwrap_or_else(|| self.build_general_from_cache(first_dirty_index)); - *self.cache.borrow_mut() = Some(output.clone()); - *self.first_dirty_index.borrow_mut() = None; - output - } - - pub fn rebuild_incremental_for_indices( - &self, - dirty_indices: impl IntoIterator, - ) -> BuildOutput { - let first_dirty_index = self - .dirty_frontier_for_indices(dirty_indices) - .first_dirty_index; - self.try_build_axis_aligned_boxes_from_cache(first_dirty_index) - .unwrap_or_else(|| self.build_general_from_cache(first_dirty_index)) - } - - pub fn rebuild_incremental_for_brushes( - &self, - dirty_brushes: impl IntoIterator, - ) -> BuildOutput { - let indices = dirty_brushes - .into_iter() - .filter_map(|id| self.brush_index(id)) - .collect::>(); - self.rebuild_incremental_for_indices(indices) - } - - pub fn rebuild(&self) -> BuildOutput { - self.build_uncached() - } - - pub fn rebuild_routed_surfaces(&self) -> BuildOutput { - self.try_build_routed_surfaces() - .unwrap_or_else(|| self.build_uncached()) - } - - pub fn supports_routed_surfaces(&self) -> bool { - self.supports_routed_surface_subset() - } - - fn mark_dirty_index(&self, index: usize) { - let mut first_dirty_index = self.first_dirty_index.borrow_mut(); - *first_dirty_index = Some(first_dirty_index.map_or(index, |dirty| dirty.min(index))); - } - - fn try_build_routed_surfaces(&self) -> Option { - if !self.supports_routed_surface_subset() { - return None; - } - - let mut report = BuildReport { - input_brushes: self.brushes.len(), - ..BuildReport::default() - }; - let mut source = None::; - let mut surfaces = Vec::::new(); - let mut previous_cutters = Vec::::new(); - let mut scratch = PolygonRouteScratch::default(); - - for brush in &self.compiled { - match brush.op { - BrushOp::Add => { - if source.is_some() { - return None; - } - let solid = brush.convex_cutter()?; - surfaces = solid.polygons.clone(); - source = Some(solid); - } - BrushOp::Subtract => { - report.operator_brushes += 1; - let source_solid = source.as_ref()?; - if !source_solid.bounds.intersects(brush.bounds) { - report.rejected_pairs += 1; - continue; - } - let cutter = brush.convex_cutter()?; - report.candidate_pairs += 1; - surfaces = scratch.route_outside_of(surfaces, &cutter); - - let mut caps = scratch.route_inside_of(cutter.polygons.clone(), source_solid); - for previous in &previous_cutters { - caps = scratch.route_outside_of(caps, previous); - } - for cap in &mut caps { - cap.material = source_solid.material; - cap.reversed = !cap.reversed; - } - surfaces.extend(caps); - previous_cutters.push(cutter); - } - BrushOp::Intersect => return None, - } - } - - let mut mesh = TriangleMesh::new(); - ConvexSolid::append_polygons_to_mesh(&surfaces, &mut mesh); - report.emitted_convex_fragments = surfaces.len(); - - Some(BuildOutput { - mesh, - report, - generation: self.generation, - }) - } - - fn supports_routed_surface_subset(&self) -> bool { - let mut source = None::; - let mut candidate_subtracts = 0usize; - - for brush in &self.compiled { - match brush.op { - BrushOp::Add => { - if source.is_some() || brush.convex_cutter().is_none() { - return false; - } - source = Some(brush.bounds); - } - BrushOp::Subtract => { - let Some(source_bounds) = source else { - return false; - }; - if brush.convex_cutter().is_none() { - return false; - } - if source_bounds.intersects(brush.bounds) { - candidate_subtracts += 1; - if candidate_subtracts > 1 { - return false; - } - } - } - BrushOp::Intersect => return false, - } - } - - source.is_some() && candidate_subtracts == 1 - } - - fn build_uncached(&self) -> BuildOutput { - if self - .compiled - .iter() - .all(|brush| matches!(brush.geometry, CompiledGeometry::Box(_))) - { - return self.build_axis_aligned_boxes_direct(); - } - - self.build_general_direct() - } - - fn build_with_checkpoints(&self) -> BuildOutput { - self.try_build_axis_aligned_boxes_from_cache(Some(0)) - .unwrap_or_else(|| self.build_general_from_cache(Some(0))) - } - - fn build_general_direct(&self) -> BuildOutput { - let mut state = GeneralBuildState::default(); - for brush in &self.compiled { - state.apply_compiled_brush(brush); - } - state.into_output(self.brushes.len(), self.generation) - } - - fn build_axis_aligned_boxes_direct(&self) -> BuildOutput { - let mut state = BoxBuildState::default(); - for brush in &self.compiled { - state.apply_compiled_box(brush); - } - state.report.direct_dirty_rebuild = true; - state.into_output_with_mesh(self.brushes.len(), self.generation, None) - } - - fn try_build_axis_aligned_boxes_from_cache( - &self, - first_dirty_index: Option, - ) -> Option { - if !self - .compiled - .iter() - .all(|brush| matches!(brush.geometry, CompiledGeometry::Box(_))) - { - return None; - } - - let brush_count = self.compiled.len(); - let previous_output = self.cache.borrow().as_ref().cloned(); - if let Some(first_dirty_index) = first_dirty_index - && first_dirty_index > 0 - { - return Some(self.build_axis_aligned_boxes_direct()); - } - let rebuild_from = match first_dirty_index { - Some(index) => index.min(brush_count), - None => self - .box_cache - .borrow() - .as_ref() - .filter(|cache| { - cache.generation == self.generation - && cache.valid_checkpoint_index == brush_count - && cache.checkpoints.len() == brush_count + 1 - }) - .map_or(0, |_| brush_count), - }; - let cached_state = self - .box_cache - .borrow() - .as_ref() - .filter(|cache| rebuild_from <= cache.valid_checkpoint_index) - .and_then(|cache| cache.checkpoints.get(rebuild_from).cloned()); - let (mut state, start_index) = if let Some(state) = cached_state { - (state, rebuild_from) - } else { - (BoxBuildState::default(), 0) - }; - if start_index > 0 { - state.report.used_checkpoint = true; - state.report.checkpoint_rebuild_from = Some(start_index); - } - - let mut checkpoints = (start_index == 0).then(|| vec![BoxBuildState::default()]); - - let mut changed_geometry = start_index == 0; - for brush in &self.compiled[start_index..] { - changed_geometry |= state.apply_compiled_box(brush); - if let Some(checkpoints) = &mut checkpoints { - checkpoints.push(state.clone()); - } - } - - if let Some(checkpoints) = checkpoints { - *self.box_cache.borrow_mut() = Some(BoxBuildCache { - generation: self.generation, - valid_checkpoint_index: brush_count, - checkpoints, - }); - } else if let Some(cache) = self.box_cache.borrow_mut().as_mut() { - cache.generation = self.generation; - cache.valid_checkpoint_index = cache.valid_checkpoint_index.min(start_index); - } - - let reused_mesh = (!changed_geometry) - .then_some(previous_output) - .flatten() - .map(|output| output.mesh); - Some(state.into_output_with_mesh(self.brushes.len(), self.generation, reused_mesh)) - } - - fn build_general_from_cache(&self, first_dirty_index: Option) -> BuildOutput { - let brush_count = self.compiled.len(); - let previous_output = self.cache.borrow().as_ref().cloned(); - let rebuild_from = match first_dirty_index { - Some(index) => index.min(brush_count), - None => self - .general_cache - .borrow() - .as_ref() - .filter(|cache| { - cache.generation == self.generation - && cache.valid_checkpoint_index == brush_count - && cache.checkpoints.len() == brush_count + 1 - }) - .map_or(0, |_| brush_count), - }; - let cached_state = self - .general_cache - .borrow() - .as_ref() - .filter(|cache| rebuild_from <= cache.valid_checkpoint_index) - .and_then(|cache| cache.checkpoints.get(rebuild_from).cloned()); - let (mut state, start_index) = if let Some(state) = cached_state { - (state, rebuild_from) - } else { - (GeneralBuildState::default(), 0) - }; - if start_index > 0 { - state.report.used_checkpoint = true; - state.report.checkpoint_rebuild_from = Some(start_index); - } - - let mut checkpoints = (start_index == 0).then(|| vec![GeneralBuildState::default()]); - - let mut changed_geometry = start_index == 0; - for brush in &self.compiled[start_index..] { - changed_geometry |= state.apply_compiled_brush(brush); - if let Some(checkpoints) = &mut checkpoints { - checkpoints.push(state.clone()); - } - } - - if let Some(checkpoints) = checkpoints { - *self.general_cache.borrow_mut() = Some(GeneralBuildCache { - generation: self.generation, - valid_checkpoint_index: brush_count, - checkpoints, - }); - } else if let Some(cache) = self.general_cache.borrow_mut().as_mut() { - cache.generation = self.generation; - cache.valid_checkpoint_index = cache.valid_checkpoint_index.min(start_index); - } - - let reused_mesh = (!changed_geometry) - .then_some(previous_output) - .flatten() - .map(|output| output.mesh); - state.into_output_with_mesh(self.brushes.len(), self.generation, reused_mesh) - } - - pub fn sample_room_with_door() -> Self { - let wall = MaterialId(1); - let floor = MaterialId(2); - let mut asm = Self::new(); - asm.solid_box( - "floor", - Aabb::from_center_size(Vec3::new(0.0, 0.0, -0.1), Vec3::new(8.0, 8.0, 0.2)), - floor, - ); - asm.solid_box( - "north wall", - Aabb::from_center_size(Vec3::new(0.0, 4.0, 1.5), Vec3::new(8.0, 0.25, 3.0)), - wall, - ); - asm.cut_box( - "door void", - Aabb::from_center_size(Vec3::new(0.0, 4.0, 1.0), Vec3::new(1.2, 0.5, 2.0)), - ); - asm - } -} - -impl CompiledBrush { - fn from_brush(brush: &Brush) -> Self { - let bounds = brush.bounds(); - let geometry = match &brush.primitive { - Primitive::Box { bounds } => CompiledGeometry::Box(*bounds), - Primitive::OrientedBox { - center, - size, - rotation, - } => CompiledGeometry::Convex(ConvexSolid::box_from_center_size( - *center, - *size, - *rotation, - brush.material, - )), - Primitive::CylinderZ { - center, - radius, - depth, - segments, - } => CompiledGeometry::CylinderZ { - center: *center, - radius: *radius, - depth: *depth, - segments: *segments, - }, - Primitive::DomeCapZ { - center, - radius, - height, - rings, - segments, - } => CompiledGeometry::DomeCapZ { - center: *center, - radius: *radius, - height: *height, - rings: *rings, - segments: *segments, - }, - Primitive::FloretArm { - anchor, - direction, - length, - root_width, - tip_width, - thickness, - tip_lift, - } => CompiledGeometry::FloretArm { - anchor: *anchor, - direction: *direction, - length: *length, - root_width: *root_width, - tip_width: *tip_width, - thickness: *thickness, - tip_lift: *tip_lift, - }, - }; - - Self { - op: brush.op, - material: brush.material, - bounds, - geometry, - name: brush.name.clone(), - } - } - - fn convex_cutter(&self) -> Option { - match &self.geometry { - CompiledGeometry::Box(bounds) => Some(ConvexSolid::from_aabb(*bounds, self.material)), - CompiledGeometry::Convex(solid) => Some(solid.clone()), - _ => None, - } - } -} - -impl BoxBuildState { - fn apply_compiled_box(&mut self, brush: &CompiledBrush) -> bool { - let CompiledGeometry::Box(bounds) = brush.geometry else { - unreachable!("box build state received a non-box brush"); - }; - - match brush.op { - BrushOp::Add => { - self.boxes.push((bounds, brush.material)); - true - } - BrushOp::Subtract => { - self.report.operator_brushes += 1; - if !bounds.intersects_any(self.boxes.iter().map(|(bounds, _)| *bounds)) { - self.report.rejected_pairs += self.boxes.len(); - return false; - } - - let mut out = Vec::with_capacity(self.boxes.len() + 4); - for (solid, material) in self.boxes.drain(..) { - if solid.intersects(bounds) { - self.report.candidate_pairs += 1; - out.extend( - solid - .subtract_box(bounds) - .into_iter() - .map(|piece| (piece, material)), - ); - } else { - self.report.rejected_pairs += 1; - out.push((solid, material)); - } - } - self.boxes = out; - true - } - BrushOp::Intersect => { - self.report.operator_brushes += 1; - if !bounds.intersects_any(self.boxes.iter().map(|(bounds, _)| *bounds)) { - let changed = !self.boxes.is_empty(); - self.report.rejected_pairs += self.boxes.len(); - self.boxes.clear(); - return changed; - } - - self.boxes = self - .boxes - .drain(..) - .filter_map(|(solid, material)| { - if let Some(hit) = solid.intersection(bounds) { - self.report.candidate_pairs += 1; - Some((hit, material)) - } else { - self.report.rejected_pairs += 1; - None - } - }) - .collect(); - true - } - } - } - - fn into_output_with_mesh( - mut self, - input_brushes: usize, - generation: u64, - reused_mesh: Option, - ) -> BuildOutput { - let mut mesh = TriangleMesh::new(); - self.report.input_brushes = input_brushes; - self.report.emitted_convex_fragments = self.boxes.len(); - if let Some(reused_mesh) = reused_mesh { - self.report.reused_mesh = true; - mesh = reused_mesh; - } else { - for (bounds, material) in self.boxes { - mesh.append_box(bounds, material); - } - } - - BuildOutput { - mesh, - report: self.report, - generation, - } - } -} - -impl GeneralBuildState { - fn apply_compiled_brush(&mut self, brush: &CompiledBrush) -> bool { - match brush.op { - BrushOp::Add => { - self.add_brush_geometry(brush); - true - } - BrushOp::Subtract => { - self.report.operator_brushes += 1; - if !brush - .bounds - .intersects_any(self.solids.iter().map(|solid| solid.bounds)) - { - self.report.rejected_pairs += self.solids.len(); - return false; - } - if let Some(cutter) = brush.convex_cutter() { - self.solids = subtract_from_solids( - std::mem::take(&mut self.solids), - &cutter, - &mut self.report, - ); - true - } else { - let has_candidate = - record_solid_pairs(&self.solids, brush.bounds, &mut self.report); - if has_candidate { - self.report - .warnings - .push(BuildWarning::SubtractIgnoredForNonBox { - brush: brush.name.clone(), - }); - } - false - } - } - BrushOp::Intersect => { - self.report.operator_brushes += 1; - if !brush - .bounds - .intersects_any(self.solids.iter().map(|solid| solid.bounds)) - { - let changed = !self.solids.is_empty(); - self.report.rejected_pairs += self.solids.len(); - self.solids.clear(); - return changed; - } - if let Some(cutter) = brush.convex_cutter() { - self.solids = intersect_solids( - std::mem::take(&mut self.solids), - &cutter, - &mut self.report, - ); - true - } else { - let has_candidate = - record_solid_pairs(&self.solids, brush.bounds, &mut self.report); - let changed = !self.solids.is_empty(); - self.solids.clear(); - if has_candidate { - self.report - .warnings - .push(BuildWarning::IntersectIgnoredForNonBox { - brush: brush.name.clone(), - }); - } - changed - } - } - } - } - - fn add_brush_geometry(&mut self, brush: &CompiledBrush) { - match &brush.geometry { - CompiledGeometry::Box(bounds) => { - self.solids - .push(ConvexSolid::from_aabb(*bounds, brush.material)); - } - CompiledGeometry::Convex(solid) => self.solids.push(solid.clone()), - CompiledGeometry::CylinderZ { - center, - radius, - depth, - segments, - } => append_cylinder_z( - &mut self.mesh, - *center, - *radius, - *depth, - *segments, - brush.material, - ), - CompiledGeometry::DomeCapZ { - center, - radius, - height, - rings, - segments, - } => append_dome_cap_z( - &mut self.mesh, - DomeCapZSpec { - center: *center, - radius: *radius, - height: *height, - rings: *rings, - segments: *segments, - material: brush.material, - }, - ), - CompiledGeometry::FloretArm { - anchor, - direction, - length, - root_width, - tip_width, - thickness, - tip_lift, - } => append_floret_arm( - &mut self.mesh, - FloretArmSpec { - anchor: *anchor, - direction: *direction, - length: *length, - root_width: *root_width, - tip_width: *tip_width, - thickness: *thickness, - tip_lift: *tip_lift, - material: brush.material, - }, - ), - } - } - - fn into_output(self, input_brushes: usize, generation: u64) -> BuildOutput { - self.into_output_with_mesh(input_brushes, generation, None) - } - - fn into_output_with_mesh( - mut self, - input_brushes: usize, - generation: u64, - reused_mesh: Option, - ) -> BuildOutput { - self.report.input_brushes = input_brushes; - self.report.emitted_convex_fragments = self.solids.len(); - if let Some(reused_mesh) = reused_mesh { - self.report.reused_mesh = true; - self.mesh = reused_mesh; - } else { - for solid in self.solids { - solid.append_to_mesh(&mut self.mesh); - } - } - - BuildOutput { - mesh: self.mesh, - report: self.report, - generation, - } - } -} - -fn subtract_from_solids( - solids: Vec, - cutter: &ConvexSolid, - report: &mut BuildReport, -) -> Vec { - let mut out = Vec::with_capacity(solids.len() + 4); - for solid in solids { - if solid.bounds.intersects(cutter.bounds) { - report.candidate_pairs += 1; - out.extend(solid.subtract_convex_owned(cutter)); - } else { - report.rejected_pairs += 1; - out.push(solid); - } - } - out -} - -fn intersect_solids( - solids: Vec, - cutter: &ConvexSolid, - report: &mut BuildReport, -) -> Vec { - let mut out = Vec::with_capacity(solids.len()); - for solid in solids { - if !solid.bounds.intersects(cutter.bounds) { - report.rejected_pairs += 1; - continue; - } - report.candidate_pairs += 1; - if let Some(fragment) = solid.intersect_convex_owned(cutter) { - out.push(fragment); - } - } - out -} - -fn record_solid_pairs(solids: &[ConvexSolid], bounds: Aabb, report: &mut BuildReport) -> bool { - let mut has_candidate = false; - for solid in solids { - if solid.bounds.intersects(bounds) { - report.candidate_pairs += 1; - has_candidate = true; - } else { - report.rejected_pairs += 1; - } - } - has_candidate -} - -#[cfg(test)] -mod tests { - use bevy_math::Vec3; - - use super::*; - - #[test] - fn centered_subtract_splits_box_into_six_fragments() { - let source = Aabb::from_center_size(Vec3::ZERO, Vec3::splat(4.0)); - let cutter = Aabb::from_center_size(Vec3::ZERO, Vec3::splat(2.0)); - let pieces = source.subtract_box(cutter); - - assert_eq!(pieces.len(), 6); - assert!(pieces.iter().all(|piece| piece.is_valid())); - assert!(pieces.iter().all(|piece| !piece.intersects(cutter))); - } - - #[test] - fn door_cut_leaves_no_box_fragment_inside_void() { - let output = Assembler::sample_room_with_door().build(); - let cutter = Aabb::from_center_size(Vec3::new(0.0, 4.0, 1.0), Vec3::new(1.2, 0.5, 2.0)); - - assert!(output.report.emitted_convex_fragments > 2); - for tri in output.mesh.indices.chunks_exact(3) { - let center = (output.mesh.positions[tri[0] as usize] - + output.mesh.positions[tri[1] as usize] - + output.mesh.positions[tri[2] as usize]) - / 3.0; - assert!(!cutter.contains_point_strict(center, 1.0e-4)); - } - } - - #[test] - fn rotated_door_cut_produces_diagonal_portal_faces() { - let mut asm = Assembler::new(); - asm.solid_box( - "wall", - Aabb::from_center_size(Vec3::new(0.0, 0.0, 1.5), Vec3::new(5.0, 0.5, 3.0)), - MaterialId(1), - ); - asm.cut_oriented_box( - "angled void", - Vec3::new(0.0, 0.0, 1.5), - Vec3::new(1.5, 2.0, 2.0), - bevy_math::Quat::from_rotation_z(std::f32::consts::FRAC_PI_4), - ); - - let output = asm.build(); - assert!(output.report.emitted_convex_fragments > 2); - assert!(output.mesh.normals.iter().any(|normal| { - normal.x.abs() > 0.2 && normal.y.abs() > 0.2 && normal.z.abs() < 0.1 - })); - } - - #[test] - fn mesh_box_emits_twelve_triangles() { - let mut mesh = TriangleMesh::new(); - mesh.append_box(Aabb::from_center_size(Vec3::ZERO, Vec3::ONE), MaterialId(7)); - - assert_eq!(mesh.triangle_count(), 12); - assert_eq!(mesh.triangle_materials.len(), 12); - assert_eq!(mesh.triangle_materials[0], MaterialId(7)); - } - - #[test] - fn distant_subtract_skips_convex_split_and_keeps_source_solid() { - let mut asm = Assembler::new(); - asm.solid_box( - "source", - Aabb::from_center_size(Vec3::ZERO, Vec3::splat(2.0)), - MaterialId(1), - ); - asm.cut_box( - "far void", - Aabb::from_center_size(Vec3::splat(10.0), Vec3::splat(1.0)), - ); - - let output = asm.build(); - assert_eq!(output.report.emitted_convex_fragments, 1); - assert_eq!(output.report.operator_brushes, 1); - assert_eq!(output.report.candidate_pairs, 0); - assert_eq!(output.report.rejected_pairs, 1); - assert_eq!(output.mesh.triangle_count(), 12); - } - - #[test] - fn intersecting_box_keeps_common_convex_region() { - let mut asm = Assembler::new(); - asm.solid_box( - "a", - Aabb::from_center_size(Vec3::ZERO, Vec3::splat(4.0)), - MaterialId(1), - ); - asm.add_brush( - "b", - BrushOp::Intersect, - Primitive::Box { - bounds: Aabb::from_center_size(Vec3::X, Vec3::splat(4.0)), - }, - MaterialId(1), - ); - - let output = asm.build(); - assert_eq!(output.report.emitted_convex_fragments, 1); - assert_eq!(output.report.operator_brushes, 1); - assert_eq!(output.report.candidate_pairs, 1); - assert_eq!(output.report.rejected_pairs, 0); - assert_eq!(output.report.warnings.len(), 0); - assert_eq!(output.mesh.triangle_count(), 12); - } - - #[test] - fn cached_build_and_dirty_rebuild_emit_same_output() { - let mut asm = Assembler::new(); - asm.solid_box( - "slab", - Aabb::from_center_size(Vec3::ZERO, Vec3::new(8.0, 8.0, 2.0)), - MaterialId(1), - ); - asm.cut_box( - "center void", - Aabb::from_center_size(Vec3::ZERO, Vec3::splat(2.0)), - ); - asm.cut_oriented_box( - "angled void", - Vec3::new(1.0, 1.0, 0.0), - Vec3::new(1.25, 4.0, 2.5), - bevy_math::Quat::from_rotation_z(0.35), - ); - - let stable = asm.build(); - let cached = asm.build(); - let dirty = asm.rebuild(); - - assert_eq!( - stable.report.emitted_convex_fragments, - dirty.report.emitted_convex_fragments - ); - assert_eq!( - stable.report.operator_brushes, - dirty.report.operator_brushes - ); - assert_eq!(stable.report.candidate_pairs, dirty.report.candidate_pairs); - assert_eq!(stable.report.rejected_pairs, dirty.report.rejected_pairs); - assert_eq!(stable.report.warnings, dirty.report.warnings); - assert_eq!(stable.mesh.triangle_count(), dirty.mesh.triangle_count()); - assert_eq!(stable.mesh.vertex_count(), dirty.mesh.vertex_count()); - assert_eq!(stable.mesh.positions, dirty.mesh.positions); - assert_eq!(stable.mesh.indices, dirty.mesh.indices); - assert_eq!(cached.mesh.positions, stable.mesh.positions); - assert_eq!(cached.mesh.indices, stable.mesh.indices); - } - - #[test] - fn routed_surface_rebuild_emits_clean_center_cut_boundary() { - let mut asm = Assembler::new(); - let cutter = Aabb::from_center_size(Vec3::ZERO, Vec3::splat(2.0)); - asm.solid_box( - "source", - Aabb::from_center_size(Vec3::ZERO, Vec3::splat(4.0)), - MaterialId(1), - ); - asm.cut_box("void", cutter); - - let routed = asm.rebuild_routed_surfaces(); - - assert_eq!(routed.report.operator_brushes, 1); - assert_eq!(routed.report.candidate_pairs, 1); - assert_eq!(routed.report.rejected_pairs, 0); - assert_eq!(routed.report.emitted_convex_fragments, 24); - assert!(routed.mesh.triangle_count() < asm.rebuild().mesh.triangle_count()); - for tri in routed.mesh.indices.chunks_exact(3) { - let center = (routed.mesh.positions[tri[0] as usize] - + routed.mesh.positions[tri[1] as usize] - + routed.mesh.positions[tri[2] as usize]) - / 3.0; - assert!(!cutter.contains_point_strict(center, 1.0e-4)); - } - } - - #[test] - fn routed_surface_rebuild_falls_back_for_intersections() { - let mut asm = Assembler::new(); - asm.solid_box( - "source", - Aabb::from_center_size(Vec3::ZERO, Vec3::splat(4.0)), - MaterialId(1), - ); - asm.add_brush( - "common", - BrushOp::Intersect, - Primitive::Box { - bounds: Aabb::from_center_size(Vec3::X, Vec3::splat(4.0)), - }, - MaterialId(1), - ); - - let stable = asm.rebuild(); - let routed = asm.rebuild_routed_surfaces(); - - assert_eq!( - routed.report.emitted_convex_fragments, - stable.report.emitted_convex_fragments - ); - assert_eq!(routed.mesh.triangle_count(), stable.mesh.triangle_count()); - assert_eq!(routed.mesh.positions, stable.mesh.positions); - assert_eq!(routed.mesh.indices, stable.mesh.indices); - } - - #[test] - fn routed_surface_rebuild_falls_back_for_multiple_candidate_cutters() { - let mut asm = Assembler::new(); - asm.solid_box( - "source", - Aabb::from_center_size(Vec3::ZERO, Vec3::splat(4.0)), - MaterialId(1), - ); - asm.cut_oriented_box( - "a", - Vec3::new(-0.5, 0.0, 0.0), - Vec3::new(1.0, 5.0, 5.0), - bevy_math::Quat::from_rotation_z(0.25), - ); - asm.cut_oriented_box( - "b", - Vec3::new(0.5, 0.0, 0.0), - Vec3::new(1.0, 5.0, 5.0), - bevy_math::Quat::from_rotation_z(-0.25), - ); - - let stable = asm.rebuild(); - let routed = asm.rebuild_routed_surfaces(); - - assert_eq!( - routed.report.emitted_convex_fragments, - stable.report.emitted_convex_fragments - ); - assert_eq!(routed.mesh.triangle_count(), stable.mesh.triangle_count()); - assert_eq!(routed.mesh.positions, stable.mesh.positions); - assert_eq!(routed.mesh.indices, stable.mesh.indices); - } - - #[test] - fn routed_surface_rebuild_falls_back_when_no_cutter_touches_source() { - let mut asm = Assembler::new(); - asm.solid_box( - "source", - Aabb::from_center_size(Vec3::ZERO, Vec3::splat(4.0)), - MaterialId(1), - ); - asm.cut_box( - "far", - Aabb::from_center_size(Vec3::splat(20.0), Vec3::splat(2.0)), - ); - - let stable = asm.rebuild(); - let routed = asm.rebuild_routed_surfaces(); - - assert_eq!( - routed.report.emitted_convex_fragments, - stable.report.emitted_convex_fragments - ); - assert_eq!(routed.mesh.triangle_count(), stable.mesh.triangle_count()); - assert_eq!(routed.mesh.positions, stable.mesh.positions); - assert_eq!(routed.mesh.indices, stable.mesh.indices); - } - - #[test] - fn mutating_brush_primitive_invalidates_cache_and_exposes_dirty_frontier() { - let mut asm = Assembler::new(); - let source = asm.solid_box( - "source", - Aabb::from_center_size(Vec3::ZERO, Vec3::splat(4.0)), - MaterialId(1), - ); - let cutter = asm.cut_box("void", Aabb::from_center_size(Vec3::ZERO, Vec3::splat(1.0))); - let before = asm.build(); - - assert!(asm.set_brush_primitive( - cutter, - Primitive::Box { - bounds: Aabb::from_center_size(Vec3::ZERO, Vec3::splat(2.0)), - }, - )); - let frontier = asm.dirty_frontier_for_brushes([cutter]); - let after = asm.build(); - - assert_eq!(frontier.first_dirty_index, Some(1)); - assert_eq!(frontier.operator_brushes, 1); - assert_eq!(frontier.candidate_pairs(), 1); - assert_eq!(frontier.rejected_pairs, 0); - assert!(after.generation > before.generation); - assert_ne!(after.mesh.positions, before.mesh.positions); - assert_eq!(asm.brush_index(source), Some(0)); - assert_eq!(asm.brush_index(cutter), Some(1)); - } - - #[test] - fn incremental_box_rebuild_reuses_prefix_and_matches_full_rebuild() { - let mut asm = Assembler::new(); - asm.solid_box( - "source", - Aabb::from_center_size(Vec3::ZERO, Vec3::splat(8.0)), - MaterialId(1), - ); - asm.cut_box( - "early", - Aabb::from_center_size(Vec3::new(-2.0, 0.0, 0.0), Vec3::splat(2.0)), - ); - let dirty = asm.cut_box( - "dirty", - Aabb::from_center_size(Vec3::ZERO, Vec3::splat(2.0)), - ); - asm.cut_box( - "late", - Aabb::from_center_size(Vec3::new(2.0, 0.0, 0.0), Vec3::splat(2.0)), - ); - let before = asm.build(); - - assert!(asm.set_brush_primitive( - dirty, - Primitive::Box { - bounds: Aabb::from_center_size(Vec3::ZERO, Vec3::splat(3.0)), - }, - )); - let incremental = asm.build_incremental(); - let full = asm.rebuild(); - - assert!(!incremental.report.used_checkpoint); - assert!(incremental.report.direct_dirty_rebuild); - assert_eq!(incremental.mesh.positions, full.mesh.positions); - assert_eq!(incremental.mesh.indices, full.mesh.indices); - assert_eq!( - incremental.report.candidate_pairs, - full.report.candidate_pairs - ); - assert_eq!( - incremental.report.rejected_pairs, - full.report.rejected_pairs - ); - assert_eq!(incremental.generation, full.generation); - assert_ne!(incremental.mesh.positions, before.mesh.positions); - } - - #[test] - fn explicit_incremental_box_rebuild_matches_full_rebuild() { - let mut asm = Assembler::new(); - asm.solid_box( - "source", - Aabb::from_center_size(Vec3::ZERO, Vec3::splat(8.0)), - MaterialId(1), - ); - asm.cut_box( - "early", - Aabb::from_center_size(Vec3::new(-2.0, 0.0, 0.0), Vec3::splat(2.0)), - ); - let dirty = asm.cut_box( - "dirty", - Aabb::from_center_size(Vec3::ZERO, Vec3::splat(2.0)), - ); - asm.cut_box( - "late", - Aabb::from_center_size(Vec3::new(2.0, 0.0, 0.0), Vec3::splat(2.0)), - ); - asm.build_incremental(); - - assert!(asm.set_brush_primitive( - dirty, - Primitive::Box { - bounds: Aabb::from_center_size(Vec3::ZERO, Vec3::splat(3.0)), - }, - )); - let incremental = asm.rebuild_incremental_for_brushes([dirty]); - let full = asm.rebuild(); - - assert_eq!(incremental.mesh.positions, full.mesh.positions); - assert_eq!(incremental.mesh.indices, full.mesh.indices); - assert_eq!( - incremental.report.candidate_pairs, - full.report.candidate_pairs - ); - assert_eq!( - incremental.report.rejected_pairs, - full.report.rejected_pairs - ); - } - - #[test] - fn incremental_distant_tail_edit_reuses_previous_mesh() { - let mut asm = Assembler::new(); - asm.solid_box( - "source", - Aabb::from_center_size(Vec3::ZERO, Vec3::splat(8.0)), - MaterialId(1), - ); - let far = asm.cut_oriented_box( - "far", - Vec3::splat(100.0), - Vec3::splat(1.0), - bevy_math::Quat::from_rotation_z(0.2), - ); - let before = asm.build_incremental(); - - assert!(asm.set_brush_primitive( - far, - Primitive::OrientedBox { - center: Vec3::splat(120.0), - size: Vec3::splat(1.0), - rotation: bevy_math::Quat::from_rotation_z(0.2), - }, - )); - let after = asm.build_incremental(); - - assert_eq!(after.mesh, before.mesh); - assert!(after.report.reused_mesh); - assert!(after.report.used_checkpoint); - assert_eq!(after.report.checkpoint_rebuild_from, Some(1)); - assert!(!after.report.direct_dirty_rebuild); - assert_eq!(after.report.candidate_pairs, 0); - assert_eq!(after.report.rejected_pairs, 1); - } - - #[test] - fn incremental_convex_rebuild_matches_full_rebuild() { - let mut asm = Assembler::new(); - asm.solid_box( - "slab", - Aabb::from_center_size(Vec3::ZERO, Vec3::new(16.0, 16.0, 4.0)), - MaterialId(1), - ); - - let mut tail = BrushId(0); - for index in 0..12 { - let angle = index as f32 * 0.217; - let center = Vec3::new(angle.cos() * 4.0, angle.sin() * 4.0, 0.0); - tail = asm.cut_oriented_box( - format!("rotated_void_{index}"), - center, - Vec3::new(1.0, 6.0, 5.0), - bevy_math::Quat::from_rotation_z(angle), - ); - } - asm.build_incremental(); - - assert!(asm.set_brush_primitive( - tail, - Primitive::OrientedBox { - center: Vec3::new(3.0, 0.0, 0.0), - size: Vec3::new(1.25, 6.0, 5.0), - rotation: bevy_math::Quat::from_rotation_z(0.5), - }, - )); - let incremental = asm.build_incremental(); - let full = asm.rebuild(); - - assert!(incremental.report.used_checkpoint); - assert_eq!(incremental.report.checkpoint_rebuild_from, Some(12)); - assert!(!incremental.report.direct_dirty_rebuild); - assert_eq!(incremental.mesh.positions, full.mesh.positions); - assert_eq!(incremental.mesh.indices, full.mesh.indices); - assert_eq!( - incremental.report.candidate_pairs, - full.report.candidate_pairs - ); - assert_eq!( - incremental.report.rejected_pairs, - full.report.rejected_pairs - ); - } - - #[test] - fn mutating_brush_operation_changes_ordered_output() { - let mut asm = Assembler::new(); - let _source = asm.solid_box( - "source", - Aabb::from_center_size(Vec3::ZERO, Vec3::splat(4.0)), - MaterialId(1), - ); - let other = asm.solid_box( - "other", - Aabb::from_center_size(Vec3::new(0.5, 0.0, 0.0), Vec3::splat(3.0)), - MaterialId(1), - ); - let before = asm.rebuild(); - - assert!(asm.set_brush_op(other, BrushOp::Intersect)); - let after = asm.rebuild(); - - assert_eq!(after.report.operator_brushes, 1); - assert_eq!(after.report.candidate_pairs, 1); - assert!(after.mesh.triangle_count() < before.mesh.triangle_count()); - } - - #[test] - fn rotated_cut_stack_regression_preserves_ordered_output_shape() { - let mut asm = Assembler::new(); - asm.solid_box( - "slab", - Aabb::from_center_size(Vec3::ZERO, Vec3::new(32.0, 32.0, 4.0)), - MaterialId(1), - ); - - for index in 0..64 { - let angle = index as f32 * 0.173; - let radius = 11.0 + (index % 7) as f32 * 0.35; - let center = Vec3::new(angle.cos() * radius, angle.sin() * radius, 0.0); - asm.cut_oriented_box( - format!("rotated_void_{index}"), - center, - Vec3::new(1.0 + (index % 3) as f32 * 0.3, 8.0, 5.0), - bevy_math::Quat::from_rotation_z(angle), - ); - } - - let output = asm.rebuild(); - - assert_eq!(output.report.emitted_convex_fragments, 280); - assert_eq!(output.mesh.triangle_count(), 3404); - assert_eq!(output.report.candidate_pairs, 804); - assert_eq!(output.report.rejected_pairs, 9417); - } -} diff --git a/packages/gamecult-geometry-rs/src/brush.rs b/packages/gamecult-geometry-rs/src/brush.rs deleted file mode 100644 index 314c84d0..00000000 --- a/packages/gamecult-geometry-rs/src/brush.rs +++ /dev/null @@ -1,370 +0,0 @@ -#![allow(dead_code)] -use bevy_math::{Quat, Vec3}; - -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] -pub struct BrushId(pub u32); - -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Default)] -pub struct MaterialId(pub u32); - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum BrushOp { - Add, - Subtract, - Intersect, -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] -pub enum PolygonCategory { - Inside, - Outside, - Aligned, - ReverseAligned, -} - -#[derive(Clone, Copy, Debug, PartialEq)] -pub struct Aabb { - pub min: Vec3, - pub max: Vec3, -} - -impl Aabb { - pub fn empty() -> Self { - Self { - min: Vec3::splat(f32::INFINITY), - max: Vec3::splat(f32::NEG_INFINITY), - } - } - - pub fn from_center_size(center: Vec3, size: Vec3) -> Self { - let half = size * 0.5; - Self { - min: center - half, - max: center + half, - } - } - - pub fn new(min: Vec3, max: Vec3) -> Self { - Self { min, max } - } - - pub fn from_points(points: &[Vec3]) -> Self { - let mut bounds = Self::empty(); - for point in points { - bounds.include_point(*point); - } - bounds - } - - pub fn include_point(&mut self, point: Vec3) { - self.min = self.min.min(point); - self.max = self.max.max(point); - } - - pub fn union(self, other: Self) -> Self { - if !self.has_finite_extents() { - return other; - } - if !other.has_finite_extents() { - return self; - } - Self { - min: self.min.min(other.min), - max: self.max.max(other.max), - } - } - - pub fn size(self) -> Vec3 { - self.max - self.min - } - - pub fn center(self) -> Vec3 { - (self.min + self.max) * 0.5 - } - - pub fn is_valid(self) -> bool { - self.max.x > self.min.x && self.max.y > self.min.y && self.max.z > self.min.z - } - - pub fn has_finite_extents(self) -> bool { - self.min.is_finite() && self.max.is_finite() - } - - pub fn intersects(self, other: Self) -> bool { - self.min.x < other.max.x - && self.max.x > other.min.x - && self.min.y < other.max.y - && self.max.y > other.min.y - && self.min.z < other.max.z - && self.max.z > other.min.z - } - - pub fn intersects_any(self, others: impl IntoIterator) -> bool { - others.into_iter().any(|other| self.intersects(other)) - } - - pub fn intersection(self, other: Self) -> Option { - if !self.intersects(other) { - return None; - } - - let min = self.min.max(other.min); - let max = self.max.min(other.max); - let hit = Self { min, max }; - hit.is_valid().then_some(hit) - } - - pub fn contains_point(self, point: Vec3) -> bool { - point.x >= self.min.x - && point.x <= self.max.x - && point.y >= self.min.y - && point.y <= self.max.y - && point.z >= self.min.z - && point.z <= self.max.z - } - - pub fn contains_point_strict(self, point: Vec3, epsilon: f32) -> bool { - point.x > self.min.x + epsilon - && point.x < self.max.x - epsilon - && point.y > self.min.y + epsilon - && point.y < self.max.y - epsilon - && point.z > self.min.z + epsilon - && point.z < self.max.z - epsilon - } - - pub fn subtract_box(self, cutter: Self) -> Vec { - let Some(hit) = self.intersection(cutter) else { - return vec![self]; - }; - - let mut pieces = Vec::with_capacity(6); - - push_valid( - &mut pieces, - Self::new(self.min, Vec3::new(hit.min.x, self.max.y, self.max.z)), - ); - push_valid( - &mut pieces, - Self::new(Vec3::new(hit.max.x, self.min.y, self.min.z), self.max), - ); - - let x_min = self.min.x.max(hit.min.x); - let x_max = self.max.x.min(hit.max.x); - - push_valid( - &mut pieces, - Self::new( - Vec3::new(x_min, self.min.y, self.min.z), - Vec3::new(x_max, hit.min.y, self.max.z), - ), - ); - push_valid( - &mut pieces, - Self::new( - Vec3::new(x_min, hit.max.y, self.min.z), - Vec3::new(x_max, self.max.y, self.max.z), - ), - ); - - let y_min = self.min.y.max(hit.min.y); - let y_max = self.max.y.min(hit.max.y); - - push_valid( - &mut pieces, - Self::new( - Vec3::new(x_min, y_min, self.min.z), - Vec3::new(x_max, y_max, hit.min.z), - ), - ); - push_valid( - &mut pieces, - Self::new( - Vec3::new(x_min, y_min, hit.max.z), - Vec3::new(x_max, y_max, self.max.z), - ), - ); - - pieces - } -} - -fn push_valid(pieces: &mut Vec, bounds: Aabb) { - if bounds.is_valid() { - pieces.push(bounds); - } -} - -#[derive(Clone, Debug)] -pub enum Primitive { - Box { - bounds: Aabb, - }, - OrientedBox { - center: Vec3, - size: Vec3, - rotation: Quat, - }, - CylinderZ { - center: Vec3, - radius: f32, - depth: f32, - segments: usize, - }, - DomeCapZ { - center: Vec3, - radius: f32, - height: f32, - rings: usize, - segments: usize, - }, - FloretArm { - anchor: Vec3, - direction: Vec3, - length: f32, - root_width: f32, - tip_width: f32, - thickness: f32, - tip_lift: f32, - }, -} - -#[derive(Clone, Debug)] -pub struct Brush { - pub id: BrushId, - pub name: String, - pub op: BrushOp, - pub primitive: Primitive, - pub material: MaterialId, - pub rotation: Quat, - pub generation: u64, -} - -impl Brush { - pub fn new( - id: BrushId, - name: impl Into, - op: BrushOp, - primitive: Primitive, - material: MaterialId, - ) -> Self { - Self { - id, - name: name.into(), - op, - primitive, - material, - rotation: Quat::IDENTITY, - generation: 0, - } - } - - pub fn set_dirty(&mut self) { - self.generation = self.generation.wrapping_add(1); - } - - pub fn as_box(&self) -> Option { - match self.primitive { - Primitive::Box { bounds } => Some(bounds), - _ => None, - } - } - - pub fn bounds(&self) -> Aabb { - self.primitive.bounds() - } -} - -impl Primitive { - pub fn bounds(&self) -> Aabb { - match *self { - Self::Box { bounds } => bounds, - Self::OrientedBox { - center, - size, - rotation, - } => oriented_box_bounds(center, size, rotation), - Self::CylinderZ { - center, - radius, - depth, - .. - } => Aabb::new( - center - Vec3::new(radius, radius, depth * 0.5), - center + Vec3::new(radius, radius, depth * 0.5), - ), - Self::DomeCapZ { - center, - radius, - height, - .. - } => Aabb::new( - center - Vec3::new(radius, radius, 0.0), - center + Vec3::new(radius, radius, height), - ), - Self::FloretArm { - anchor, - direction, - length, - root_width, - tip_width, - thickness, - tip_lift, - } => floret_bounds( - anchor, direction, length, root_width, tip_width, thickness, tip_lift, - ), - } - } -} - -fn oriented_box_bounds(center: Vec3, size: Vec3, rotation: Quat) -> Aabb { - let half = size * 0.5; - let local_corners = [ - Vec3::new(-half.x, -half.y, -half.z), - Vec3::new(half.x, -half.y, -half.z), - Vec3::new(half.x, half.y, -half.z), - Vec3::new(-half.x, half.y, -half.z), - Vec3::new(-half.x, -half.y, half.z), - Vec3::new(half.x, -half.y, half.z), - Vec3::new(half.x, half.y, half.z), - Vec3::new(-half.x, half.y, half.z), - ]; - let corners = local_corners.map(|point| center + rotation * point); - Aabb::from_points(&corners) -} - -fn floret_bounds( - anchor: Vec3, - direction: Vec3, - length: f32, - root_width: f32, - tip_width: f32, - thickness: f32, - tip_lift: f32, -) -> Aabb { - let forward = direction.normalize_or_zero(); - let forward = if forward.length_squared() == 0.0 { - Vec3::X - } else { - forward - }; - let mut side = forward.cross(Vec3::Z).normalize_or_zero(); - if side.length_squared() == 0.0 { - side = Vec3::Y; - } - - let root_center = anchor; - let tip_center = anchor + forward * length + Vec3::Z * tip_lift; - let z = Vec3::Z * (thickness * 0.5); - let r = side * (root_width * 0.5); - let t = side * (tip_width * 0.5); - Aabb::from_points(&[ - root_center - r - z, - root_center + r - z, - root_center - r + z, - root_center + r + z, - tip_center - t - z, - tip_center + t - z, - tip_center - t + z, - tip_center + t + z, - ]) -} diff --git a/packages/gamecult-geometry-rs/src/convex.rs b/packages/gamecult-geometry-rs/src/convex.rs deleted file mode 100644 index 7c8d78b6..00000000 --- a/packages/gamecult-geometry-rs/src/convex.rs +++ /dev/null @@ -1,905 +0,0 @@ -#![allow(dead_code)] -use bevy_math::{Quat, Vec2, Vec3}; - -use crate::{Aabb, MaterialId, PolygonCategory, TriangleMesh}; - -const EPSILON: f32 = 1.0e-5; - -#[derive(Clone, Copy, Debug, PartialEq)] -pub struct Plane { - pub normal: Vec3, - pub distance: f32, -} - -impl Plane { - pub fn from_point_normal(point: Vec3, normal: Vec3) -> Self { - let normal = normal.normalize_or_zero(); - Self { - normal, - distance: -normal.dot(point), - } - } - - pub fn signed_distance(self, point: Vec3) -> f32 { - self.normal.dot(point) + self.distance - } - - pub fn is_coplanar_with(self, other: Self) -> bool { - self.normal.dot(other.normal).abs() > 1.0 - EPSILON - && (self.distance - other.distance).abs() <= EPSILON - } -} - -#[derive(Clone, Debug, PartialEq)] -pub struct ConvexPolygon { - pub vertices: Vec, - pub normal: Vec3, - pub material: MaterialId, - pub category: PolygonCategory, - pub visible: bool, - pub reversed: bool, - pub bounds: Aabb, -} - -impl ConvexPolygon { - pub fn new(vertices: Vec, normal: Vec3, material: MaterialId) -> Self { - Self { - bounds: Aabb::from_points(&vertices), - vertices, - normal: normal.normalize_or_zero(), - material, - category: PolygonCategory::Aligned, - visible: true, - reversed: false, - } - } - - pub fn centroid(&self) -> Vec3 { - if self.vertices.is_empty() { - return Vec3::ZERO; - } - self.vertices.iter().copied().sum::() / self.vertices.len() as f32 - } - - fn with_vertices(&self, vertices: Vec) -> Self { - Self { - bounds: Aabb::from_points(&vertices), - vertices, - normal: self.normal, - material: self.material, - category: self.category, - visible: self.visible, - reversed: self.reversed, - } - } -} - -#[derive(Clone, Debug, PartialEq)] -pub struct ConvexSolid { - pub polygons: Vec, - pub clip_planes: Vec, - pub material: MaterialId, - pub bounds: Aabb, -} - -#[derive(Clone, Debug, Default)] -pub struct CategorizedPolygons { - pub inside: Vec, - pub outside: Vec, - pub aligned: Vec, - pub reverse_aligned: Vec, -} - -#[derive(Default)] -pub struct PolygonRouteScratch { - pieces: Vec, - inside_pieces: Vec, - cap_points: Vec, -} - -impl ConvexSolid { - pub fn from_aabb(bounds: Aabb, material: MaterialId) -> Self { - Self::box_from_center_size(bounds.center(), bounds.size(), Quat::IDENTITY, material) - } - - pub fn box_from_center_size( - center: Vec3, - size: Vec3, - rotation: Quat, - material: MaterialId, - ) -> Self { - let half = size * 0.5; - let local_corners = [ - Vec3::new(-half.x, -half.y, -half.z), - Vec3::new(half.x, -half.y, -half.z), - Vec3::new(half.x, half.y, -half.z), - Vec3::new(-half.x, half.y, -half.z), - Vec3::new(-half.x, -half.y, half.z), - Vec3::new(half.x, -half.y, half.z), - Vec3::new(half.x, half.y, half.z), - Vec3::new(-half.x, half.y, half.z), - ]; - let corners = local_corners.map(|point| center + rotation * point); - let faces = [ - ([0, 3, 2, 1], Vec3::NEG_Z), - ([4, 5, 6, 7], Vec3::Z), - ([0, 1, 5, 4], Vec3::NEG_Y), - ([3, 7, 6, 2], Vec3::Y), - ([0, 4, 7, 3], Vec3::NEG_X), - ([1, 2, 6, 5], Vec3::X), - ]; - - let mut polygons = Vec::with_capacity(6); - let mut clip_planes = Vec::with_capacity(6); - for (indices, local_normal) in faces { - let normal = rotation * local_normal; - let vertices = indices.map(|index| corners[index]).to_vec(); - clip_planes.push(Plane::from_point_normal(vertices[0], normal)); - polygons.push(ConvexPolygon::new(vertices, normal, material)); - } - - Self { - bounds: Aabb::from_points(&corners), - polygons, - clip_planes, - material, - } - } - - pub fn subtract_convex(&self, cutter: &Self) -> Vec { - self.clone().subtract_convex_owned(cutter) - } - - pub fn subtract_convex_owned(self, cutter: &Self) -> Vec { - let mut fragments = Vec::new(); - let mut remainder = Some(self); - - for plane in &cutter.clip_planes { - let Some(current) = remainder.take() else { - break; - }; - - match current.split_owned(*plane) { - SplitResult::Front(front) => { - fragments.push(front); - break; - } - SplitResult::Back(back) => { - remainder = Some(back); - } - SplitResult::Both { front, back } => { - fragments.push(front); - remainder = Some(back); - } - SplitResult::Coplanar(solid) => { - remainder = Some(solid); - } - } - } - - fragments - } - - pub fn intersect_convex(&self, cutter: &Self) -> Option { - self.clone().intersect_convex_owned(cutter) - } - - pub fn intersect_convex_owned(self, cutter: &Self) -> Option { - if cutter.contains_solid(&self) { - return Some(self); - } - if self.contains_solid(cutter) { - return Some(cutter.clone_with_material(self.material)); - } - - let mut remainder = self; - - for plane in &cutter.clip_planes { - remainder = match remainder.split_owned(*plane) { - SplitResult::Front(_) => return None, - SplitResult::Back(back) | SplitResult::Coplanar(back) => back, - SplitResult::Both { back, .. } => back, - }; - } - - Some(remainder) - } - - pub fn contains_point(&self, point: Vec3) -> bool { - self.clip_planes - .iter() - .all(|plane| plane.signed_distance(point) <= EPSILON) - } - - pub fn contains_solid(&self, other: &Self) -> bool { - other - .polygons - .iter() - .flat_map(|polygon| polygon.vertices.iter()) - .all(|point| self.contains_point(*point)) - } - - pub fn clone_with_material(&self, material: MaterialId) -> Self { - let mut solid = self.clone(); - solid.material = material; - for polygon in &mut solid.polygons { - polygon.material = material; - } - solid - } - - pub fn append_to_mesh(&self, mesh: &mut TriangleMesh) { - for polygon in &self.polygons { - if !polygon.visible || polygon.vertices.len() < 3 { - continue; - } - let base = polygon.vertices[0]; - for i in 1..polygon.vertices.len() - 1 { - let triangle = if polygon.reversed { - [base, polygon.vertices[i + 1], polygon.vertices[i]] - } else { - [base, polygon.vertices[i], polygon.vertices[i + 1]] - }; - let normal = if polygon.reversed { - -polygon.normal - } else { - polygon.normal - }; - mesh.append_triangle( - triangle, - normal, - [Vec2::ZERO, Vec2::X, Vec2::Y], - polygon.material, - ); - } - } - } - - pub fn categorize_whole_polygons_against(&mut self, cutter: &Self) { - for polygon in &mut self.polygons { - if let Some(category) = whole_polygon_category_against(polygon, cutter) { - polygon.category = category; - } - } - } - - pub fn categorize_polygons_against(&self, cutter: &Self) -> CategorizedPolygons { - PolygonRouteScratch::default().categorize_against(&self.polygons, cutter) - } - - pub fn route_polygons_outside_of( - polygons: Vec, - cutter: &Self, - ) -> Vec { - let categorized = PolygonRouteScratch::default().categorize_against(&polygons, cutter); - categorized.outside - } - - pub fn route_polygons_inside_of( - polygons: Vec, - cutter: &Self, - ) -> Vec { - let categorized = PolygonRouteScratch::default().categorize_against(&polygons, cutter); - let mut inside = categorized.inside; - inside.extend(categorized.aligned); - inside.extend(categorized.reverse_aligned); - inside - } - - pub fn subtract_convex_routed_polygons(&self, cutter: &Self) -> Vec { - let mut polygons = Vec::new(); - let source = self.categorize_polygons_against(cutter); - polygons.extend(source.outside); - - let cutter_inside_source = cutter.categorize_polygons_against(self); - for mut polygon in cutter_inside_source.inside { - polygon.reversed = !polygon.reversed; - polygons.push(polygon); - } - for mut polygon in cutter_inside_source.aligned { - polygon.reversed = !polygon.reversed; - polygons.push(polygon); - } - for mut polygon in cutter_inside_source.reverse_aligned { - polygon.reversed = !polygon.reversed; - polygons.push(polygon); - } - - polygons - } - - pub fn append_polygons_to_mesh(polygons: &[ConvexPolygon], mesh: &mut TriangleMesh) { - for polygon in polygons { - if !polygon.visible || polygon.vertices.len() < 3 { - continue; - } - let base = polygon.vertices[0]; - for i in 1..polygon.vertices.len() - 1 { - let triangle = if polygon.reversed { - [base, polygon.vertices[i + 1], polygon.vertices[i]] - } else { - [base, polygon.vertices[i], polygon.vertices[i + 1]] - }; - let normal = if polygon.reversed { - -polygon.normal - } else { - polygon.normal - }; - mesh.append_triangle( - triangle, - normal, - [Vec2::ZERO, Vec2::X, Vec2::Y], - polygon.material, - ); - } - } - } - - fn split_owned(self, plane: Plane) -> SplitResult { - let mut saw_front = false; - let mut saw_back = false; - for polygon in &self.polygons { - for vertex in &polygon.vertices { - let distance = plane.signed_distance(*vertex); - saw_front |= distance > EPSILON; - saw_back |= distance < -EPSILON; - } - } - - match (saw_front, saw_back) { - (true, false) => return SplitResult::Front(self), - (false, true) => return SplitResult::Back(self), - (false, false) => return SplitResult::Coplanar(self), - (true, true) => {} - } - - let mut front_polygons = Vec::with_capacity(self.polygons.len() + 1); - let mut back_polygons = Vec::with_capacity(self.polygons.len() + 1); - let mut cap_points = Vec::with_capacity(self.polygons.len() * 2); - - for polygon in &self.polygons { - let front = clip_polygon(&polygon.vertices, plane, KeepSide::Front, &mut cap_points); - if front.len() >= 3 { - front_polygons.push(polygon.with_vertices(front)); - } - - let back = clip_polygon(&polygon.vertices, plane, KeepSide::Back, &mut cap_points); - if back.len() >= 3 { - back_polygons.push(polygon.with_vertices(back)); - } - } - - let cap_points = unique_points_on_plane(cap_points); - if cap_points.len() >= 3 { - front_polygons.push(make_cap_polygon(&cap_points, -plane.normal, self.material)); - back_polygons.push(make_cap_polygon(&cap_points, plane.normal, self.material)); - } - - SplitResult::Both { - front: Self { - bounds: bounds_from_polygons(&front_polygons), - clip_planes: planes_from_polygons(&front_polygons), - polygons: front_polygons, - material: self.material, - }, - back: Self { - bounds: bounds_from_polygons(&back_polygons), - clip_planes: planes_from_polygons(&back_polygons), - polygons: back_polygons, - material: self.material, - }, - } - } -} - -impl PolygonRouteScratch { - pub fn categorize_against( - &mut self, - polygons: &[ConvexPolygon], - cutter: &ConvexSolid, - ) -> CategorizedPolygons { - let mut categorized = CategorizedPolygons::default(); - - for polygon in polygons { - self.pieces.clear(); - self.pieces.push(CategorizationPiece { - polygon: polygon.clone(), - aligned_category: None, - }); - - for plane in &cutter.clip_planes { - self.inside_pieces.clear(); - for piece in self.pieces.drain(..) { - classify_piece_against_plane( - piece, - *plane, - &mut self.inside_pieces, - &mut categorized, - &mut self.cap_points, - ); - } - std::mem::swap(&mut self.pieces, &mut self.inside_pieces); - } - - for piece in self.pieces.drain(..) { - match piece.aligned_category { - Some(PolygonCategory::Aligned) => categorized.aligned.push(piece.polygon), - Some(PolygonCategory::ReverseAligned) => { - categorized.reverse_aligned.push(piece.polygon); - } - _ => categorized.inside.push(piece.polygon), - } - } - } - - categorized - } - - pub fn route_outside_of( - &mut self, - polygons: Vec, - cutter: &ConvexSolid, - ) -> Vec { - self.categorize_against(&polygons, cutter).outside - } - - pub fn route_inside_of( - &mut self, - polygons: Vec, - cutter: &ConvexSolid, - ) -> Vec { - let categorized = self.categorize_against(&polygons, cutter); - let mut inside = categorized.inside; - inside.extend(categorized.aligned); - inside.extend(categorized.reverse_aligned); - inside - } -} - -enum SplitResult { - Front(ConvexSolid), - Back(ConvexSolid), - Both { - front: ConvexSolid, - back: ConvexSolid, - }, - Coplanar(ConvexSolid), -} - -struct CategorizationPiece { - polygon: ConvexPolygon, - aligned_category: Option, -} - -#[derive(Clone, Copy)] -enum KeepSide { - Front, - Back, -} - -fn classify_piece_against_plane( - mut piece: CategorizationPiece, - plane: Plane, - inside_pieces: &mut Vec, - categorized: &mut CategorizedPolygons, - cap_points: &mut Vec, -) { - let piece_plane = piece - .polygon - .vertices - .first() - .map(|point| Plane::from_point_normal(*point, piece.polygon.normal)); - - if piece_plane.is_some_and(|piece_plane| piece_plane.is_coplanar_with(plane)) { - piece.aligned_category = Some(if piece.polygon.normal.dot(plane.normal) >= 0.0 { - PolygonCategory::Aligned - } else { - PolygonCategory::ReverseAligned - }); - inside_pieces.push(piece); - return; - } - - let mut saw_front = false; - let mut saw_back = false; - for vertex in &piece.polygon.vertices { - let distance = plane.signed_distance(*vertex); - saw_front |= distance > EPSILON; - saw_back |= distance < -EPSILON; - } - - match (saw_front, saw_back) { - (true, false) => { - let mut polygon = piece.polygon; - polygon.category = PolygonCategory::Outside; - categorized.outside.push(polygon); - } - (false, true) | (false, false) => inside_pieces.push(piece), - (true, true) => { - cap_points.clear(); - let outside = clip_polygon(&piece.polygon.vertices, plane, KeepSide::Front, cap_points); - if outside.len() >= 3 { - let mut outside_polygon = piece.polygon.with_vertices(outside); - outside_polygon.category = PolygonCategory::Outside; - categorized.outside.push(outside_polygon); - } - - let inside = clip_polygon(&piece.polygon.vertices, plane, KeepSide::Back, cap_points); - if inside.len() >= 3 { - inside_pieces.push(CategorizationPiece { - polygon: piece.polygon.with_vertices(inside), - aligned_category: piece.aligned_category, - }); - } - } - } -} - -fn clip_polygon( - vertices: &[Vec3], - plane: Plane, - keep: KeepSide, - cap_points: &mut Vec, -) -> Vec { - let mut output = Vec::with_capacity(vertices.len() + 1); - if vertices.is_empty() { - return output; - } - - let mut previous = *vertices.last().expect("checked non-empty"); - let mut previous_distance = plane.signed_distance(previous); - let mut previous_inside = inside(previous_distance, keep); - - for ¤t in vertices { - let current_distance = plane.signed_distance(current); - let current_inside = inside(current_distance, keep); - - if current_inside != previous_inside { - let t = previous_distance / (previous_distance - current_distance); - let intersection = previous + (current - previous) * t; - output.push(intersection); - cap_points.push(intersection); - } - - if current_inside { - output.push(current); - } - - previous = current; - previous_distance = current_distance; - previous_inside = current_inside; - } - - output -} - -fn inside(distance: f32, keep: KeepSide) -> bool { - match keep { - KeepSide::Front => distance >= -EPSILON, - KeepSide::Back => distance <= EPSILON, - } -} - -fn make_cap_polygon(points: &[Vec3], normal: Vec3, material: MaterialId) -> ConvexPolygon { - let center = points.iter().copied().sum::() / points.len() as f32; - let axis = if normal.z.abs() < 0.9 { - Vec3::Z - } else { - Vec3::Y - }; - let tangent = normal.cross(axis).normalize_or_zero(); - let bitangent = normal.cross(tangent).normalize_or_zero(); - let mut vertices = points.to_vec(); - - vertices.sort_by(|a, b| { - let da = *a - center; - let db = *b - center; - let aa = da.dot(bitangent).atan2(da.dot(tangent)); - let ab = db.dot(bitangent).atan2(db.dot(tangent)); - aa.total_cmp(&ab) - }); - - if polygon_normal(&vertices).dot(normal) < 0.0 { - vertices.reverse(); - } - - ConvexPolygon::new(vertices, normal, material) -} - -fn unique_points_on_plane(points: Vec) -> Vec { - let mut unique = Vec::new(); - 'outer: for point in points { - for seen in &unique { - let delta: Vec3 = point - *seen; - if delta.length_squared() < EPSILON * EPSILON { - continue 'outer; - } - } - unique.push(point); - } - unique -} - -fn planes_from_polygons(polygons: &[ConvexPolygon]) -> Vec { - polygons - .iter() - .filter_map(|polygon| { - polygon - .vertices - .first() - .map(|point| Plane::from_point_normal(*point, polygon.normal)) - }) - .collect() -} - -fn bounds_from_polygons(polygons: &[ConvexPolygon]) -> Aabb { - polygons.iter().fold(Aabb::empty(), |bounds, polygon| { - bounds.union(polygon.bounds) - }) -} - -fn whole_polygon_category_against( - polygon: &ConvexPolygon, - cutter: &ConvexSolid, -) -> Option { - let polygon_plane = polygon - .vertices - .first() - .map(|point| Plane::from_point_normal(*point, polygon.normal))?; - - for cutter_plane in &cutter.clip_planes { - if polygon_plane.is_coplanar_with(*cutter_plane) - && polygon - .vertices - .iter() - .all(|vertex| cutter_plane.signed_distance(*vertex).abs() <= EPSILON) - { - return Some(if polygon.normal.dot(cutter_plane.normal) >= 0.0 { - PolygonCategory::Aligned - } else { - PolygonCategory::ReverseAligned - }); - } - } - - let mut fully_inside = true; - for cutter_plane in &cutter.clip_planes { - let mut all_outside_this_plane = true; - for vertex in &polygon.vertices { - let distance = cutter_plane.signed_distance(*vertex); - if distance > EPSILON { - fully_inside = false; - } else { - all_outside_this_plane = false; - } - } - if all_outside_this_plane { - return Some(PolygonCategory::Outside); - } - } - - fully_inside.then_some(PolygonCategory::Inside) -} - -fn polygon_normal(vertices: &[Vec3]) -> Vec3 { - if vertices.len() < 3 { - return Vec3::ZERO; - } - (vertices[1] - vertices[0]) - .cross(vertices[2] - vertices[0]) - .normalize_or_zero() -} - -#[cfg(test)] -mod tests { - use super::*; - - fn assert_box_parity(solid: &ConvexSolid, material: MaterialId) { - assert_eq!(solid.polygons.len(), 6); - assert_eq!(solid.clip_planes.len(), 6); - assert!( - solid - .polygons - .iter() - .all(|polygon| polygon.vertices.len() == 4) - ); - assert!( - solid - .polygons - .iter() - .all(|polygon| polygon.material == material) - ); - } - - #[test] - fn parity_box_brush_from_planes_has_six_quad_polygons() { - let solid = ConvexSolid::box_from_center_size( - Vec3::ZERO, - Vec3::splat(2.0), - Quat::IDENTITY, - MaterialId(9), - ); - - assert_box_parity(&solid, MaterialId(9)); - } - - #[test] - fn subtract_center_box_emits_six_convex_fragments() { - let source = ConvexSolid::box_from_center_size( - Vec3::ZERO, - Vec3::splat(4.0), - Quat::IDENTITY, - MaterialId(1), - ); - let cutter = ConvexSolid::box_from_center_size( - Vec3::ZERO, - Vec3::splat(2.0), - Quat::IDENTITY, - MaterialId(0), - ); - - let fragments = source.subtract_convex(&cutter); - assert_eq!(fragments.len(), 6); - assert!( - fragments - .iter() - .all(|fragment| !fragment.polygons.is_empty()) - ); - } - - #[test] - fn intersect_overlapping_boxes_emits_single_common_solid() { - let source = ConvexSolid::box_from_center_size( - Vec3::ZERO, - Vec3::splat(4.0), - Quat::IDENTITY, - MaterialId(1), - ); - let cutter = ConvexSolid::box_from_center_size( - Vec3::X, - Vec3::splat(4.0), - Quat::IDENTITY, - MaterialId(0), - ); - - let common = source.intersect_convex(&cutter).expect("intersection"); - - assert!(common.bounds.is_valid()); - assert_eq!(common.polygons.len(), 6); - assert!(common.bounds.contains_point(Vec3::ZERO)); - assert!(common.bounds.contains_point(Vec3::new(2.0, 0.0, 0.0))); - assert!(!common.bounds.contains_point(Vec3::new(-2.0, 0.0, 0.0))); - } - - #[test] - fn categorize_crossing_coplanar_face_splits_visible_categories() { - let source = ConvexSolid::box_from_center_size( - Vec3::ZERO, - Vec3::splat(4.0), - Quat::IDENTITY, - MaterialId(1), - ); - let cutter = ConvexSolid::box_from_center_size( - Vec3::new(0.0, 0.0, 1.0), - Vec3::new(2.0, 2.0, 2.0), - Quat::IDENTITY, - MaterialId(0), - ); - - let categorized = source.categorize_polygons_against(&cutter); - - assert_eq!(categorized.aligned.len(), 1); - assert!(categorized.outside.len() > 6); - assert!(categorized.inside.is_empty()); - assert!(categorized.reverse_aligned.is_empty()); - assert!( - categorized - .aligned - .iter() - .all(|polygon| polygon.category == PolygonCategory::Aligned) - ); - assert!( - categorized - .outside - .iter() - .all(|polygon| polygon.category == PolygonCategory::Outside) - ); - } - - #[test] - fn subtract_rotated_box_creates_non_axis_aligned_faces() { - let source = ConvexSolid::box_from_center_size( - Vec3::ZERO, - Vec3::splat(4.0), - Quat::IDENTITY, - MaterialId(1), - ); - let cutter = ConvexSolid::box_from_center_size( - Vec3::ZERO, - Vec3::new(2.0, 5.0, 2.0), - Quat::from_rotation_z(std::f32::consts::FRAC_PI_4), - MaterialId(0), - ); - - let fragments = source.subtract_convex(&cutter); - assert!(fragments.len() > 2); - assert!(fragments.iter().any(|fragment| { - fragment - .polygons - .iter() - .any(|polygon| polygon.normal.x.abs() > 0.01 && polygon.normal.y.abs() > 0.01) - })); - } - - #[test] - fn routed_subtraction_emits_source_shell_and_reversed_cutter_shell() { - let source = ConvexSolid::box_from_center_size( - Vec3::ZERO, - Vec3::splat(4.0), - Quat::IDENTITY, - MaterialId(1), - ); - let cutter = ConvexSolid::box_from_center_size( - Vec3::ZERO, - Vec3::splat(2.0), - Quat::IDENTITY, - MaterialId(0), - ); - - let routed = source.subtract_convex_routed_polygons(&cutter); - - assert_eq!(routed.len(), 24); - assert_eq!( - routed - .iter() - .filter(|polygon| polygon.material == MaterialId(1)) - .count(), - 18 - ); - assert_eq!( - routed - .iter() - .filter(|polygon| polygon.material == MaterialId(0)) - .count(), - 6 - ); - assert_eq!( - routed - .iter() - .filter(|polygon| polygon.material == MaterialId(0) && polygon.reversed) - .count(), - 6 - ); - } - - #[test] - fn routed_subtraction_clips_crossing_cutter_surfaces_to_source() { - let source = ConvexSolid::box_from_center_size( - Vec3::ZERO, - Vec3::splat(4.0), - Quat::IDENTITY, - MaterialId(1), - ); - let cutter = ConvexSolid::box_from_center_size( - Vec3::new(1.5, 0.0, 0.0), - Vec3::splat(2.0), - Quat::IDENTITY, - MaterialId(0), - ); - - let routed = source.subtract_convex_routed_polygons(&cutter); - - assert!(!routed.is_empty()); - assert!( - routed - .iter() - .flat_map(|polygon| polygon.vertices.iter()) - .all(|vertex| source.bounds.contains_point(*vertex)) - ); - assert!( - routed - .iter() - .any(|polygon| polygon.material == MaterialId(0) && polygon.reversed) - ); - } -} diff --git a/packages/gamecult-geometry-rs/src/domain.rs b/packages/gamecult-geometry-rs/src/domain.rs deleted file mode 100644 index a946305e..00000000 --- a/packages/gamecult-geometry-rs/src/domain.rs +++ /dev/null @@ -1,1560 +0,0 @@ -#![allow(dead_code)] -use bevy_math::{Quat, Vec3}; - -use serde::{Deserialize, Serialize}; - -use crate::{ - Aabb, BrushOp, BuildReport, CsgBranchOp, CsgNodeId, CsgOperationType, CsgTreeArena, MaterialId, - Primitive, TriangleMesh, -}; - -#[derive(Clone, Debug, PartialEq, Eq, Hash)] -pub struct DomainKey(pub String); - -impl DomainKey { - pub fn new(value: impl Into) -> Self { - Self(value.into()) - } - - pub fn child(&self, name: impl AsRef) -> Self { - Self(format!("{}/{}", self.0, name.as_ref())) - } -} - -#[derive(Clone, Copy, Debug, PartialEq)] -pub struct DomainFrame { - pub translation: Vec3, - pub rotation: Quat, -} - -impl DomainFrame { - pub const IDENTITY: Self = Self { - translation: Vec3::ZERO, - rotation: Quat::IDENTITY, - }; - - pub fn translated(translation: Vec3) -> Self { - Self { - translation, - rotation: Quat::IDENTITY, - } - } - - pub fn rotated_z(translation: Vec3, radians: f32) -> Self { - Self { - translation, - rotation: Quat::from_rotation_z(radians), - } - } - - pub fn transform_point(self, point: Vec3) -> Vec3 { - self.translation + self.rotation * point - } - - pub fn transform_bounds(self, bounds: Aabb) -> Aabb { - let min = bounds.min; - let max = bounds.max; - let corners = [ - Vec3::new(min.x, min.y, min.z), - Vec3::new(max.x, min.y, min.z), - Vec3::new(max.x, max.y, min.z), - Vec3::new(min.x, max.y, min.z), - Vec3::new(min.x, min.y, max.z), - Vec3::new(max.x, min.y, max.z), - Vec3::new(max.x, max.y, max.z), - Vec3::new(min.x, max.y, max.z), - ]; - let world = corners.map(|corner| self.transform_point(corner)); - Aabb::from_points(&world) - } -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] -pub enum DomainKind { - Root, - Column, - AltitudeBand, - RoadSpine, - BranchRoad, - Roundabout, - SupportMass, - ClearanceVolume, - Chunk, -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] -pub enum FieldLayer { - Form, - Appearance, - Transport, -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] -pub enum FieldEncoding { - Mesh, - Sdf3d, - Occupancy, - Material, - Confidence, -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] -pub enum FeatureClaimKind { - SolidBrush, - VoidBrush, - RoadSurfaceSlab, - ClearanceVolume, - SupportShell, - ColliderProxy, -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] -pub enum FeatureLoweringPolicy { - RenderOnly, - ColliderOnly, - RenderAndCollider, - BooleanOperator, -} - -impl FeatureLoweringPolicy { - pub fn default_for_kind(kind: FeatureClaimKind) -> Self { - match kind { - FeatureClaimKind::VoidBrush | FeatureClaimKind::ClearanceVolume => { - Self::BooleanOperator - } - FeatureClaimKind::ColliderProxy => Self::ColliderOnly, - FeatureClaimKind::SolidBrush - | FeatureClaimKind::RoadSurfaceSlab - | FeatureClaimKind::SupportShell => Self::RenderAndCollider, - } - } - - pub fn emits_to(self, target: ClaimLoweringTarget) -> bool { - match self { - Self::RenderOnly => target == ClaimLoweringTarget::Render, - Self::ColliderOnly => target == ClaimLoweringTarget::Collider, - Self::RenderAndCollider | Self::BooleanOperator => true, - } - } -} - -impl FeatureClaimKind { - pub fn field_layer(self) -> FieldLayer { - match self { - Self::SolidBrush - | Self::VoidBrush - | Self::RoadSurfaceSlab - | Self::ClearanceVolume - | Self::SupportShell - | Self::ColliderProxy => FieldLayer::Form, - } - } - - pub fn field_encoding(self) -> FieldEncoding { - match self { - Self::VoidBrush | Self::ClearanceVolume => FieldEncoding::Sdf3d, - Self::ColliderProxy => FieldEncoding::Occupancy, - _ => FieldEncoding::Mesh, - } - } - - pub fn brush_op(self) -> BrushOp { - match self { - Self::VoidBrush | Self::ClearanceVolume => BrushOp::Subtract, - _ => BrushOp::Add, - } - } - - pub fn emits_render(self) -> bool { - !matches!(self, Self::ColliderProxy | Self::ClearanceVolume) - } - - pub fn emits_collider(self) -> bool { - matches!( - self, - Self::SolidBrush | Self::RoadSurfaceSlab | Self::SupportShell | Self::ColliderProxy - ) - } -} - -#[derive(Clone, Debug, PartialEq)] -pub struct FeatureClaim { - pub key: String, - pub domain_key: DomainKey, - pub frame: DomainFrame, - pub support: Aabb, - pub kind: FeatureClaimKind, - pub material: MaterialId, - pub lowering: FeatureLoweringPolicy, -} - -impl FeatureClaim { - pub fn world_bounds(&self) -> Aabb { - self.frame.transform_bounds(self.support) - } - - pub fn primitive(&self) -> Primitive { - Primitive::OrientedBox { - center: self.frame.transform_point(self.support.center()), - size: self.support.size(), - rotation: self.frame.rotation, - } - } -} - -#[derive(Clone, Debug, PartialEq)] -pub struct DomainSummary { - pub bounds: Aabb, - pub estimated_triangle_cost: usize, - pub estimated_brush_cost: usize, - pub projected_error: f32, - pub contribution_weight: f32, - pub has_children: bool, - pub fallback_claims: Vec, -} - -#[derive(Clone, Debug, PartialEq)] -pub struct DomainNode { - pub key: DomainKey, - pub parent: Option, - pub kind: DomainKind, - pub frame: DomainFrame, - pub bounds: Aabb, - pub seed: u64, - pub summary: DomainSummary, - pub claims: Vec, - pub children: Vec, -} - -#[derive(Clone, Debug, PartialEq)] -pub struct FeatureClaimSpec { - pub name: String, - pub frame: DomainFrame, - pub support: Aabb, - pub kind: FeatureClaimKind, - pub material: MaterialId, - pub lowering: FeatureLoweringPolicy, -} - -impl FeatureClaimSpec { - pub fn new( - name: impl Into, - frame: DomainFrame, - support: Aabb, - kind: FeatureClaimKind, - material: MaterialId, - ) -> Self { - Self { - name: name.into(), - frame, - support, - kind, - material, - lowering: FeatureLoweringPolicy::default_for_kind(kind), - } - } - - pub fn with_lowering_policy(mut self, lowering: FeatureLoweringPolicy) -> Self { - self.lowering = lowering; - self - } - - pub fn compile(&self, domain_key: &DomainKey) -> FeatureClaim { - FeatureClaim { - key: format!("{}/claim/{}", domain_key.0, self.name), - domain_key: domain_key.clone(), - frame: self.frame, - support: self.support, - kind: self.kind, - material: self.material, - lowering: self.lowering, - } - } -} - -#[derive(Clone, Debug, PartialEq)] -pub struct DomainNodeSpec { - pub name: String, - pub kind: DomainKind, - pub frame: DomainFrame, - pub seed: u64, - pub claims: Vec, - pub children: Vec, -} - -impl DomainNodeSpec { - pub fn new(name: impl Into, kind: DomainKind, frame: DomainFrame, seed: u64) -> Self { - Self { - name: name.into(), - kind, - frame, - seed, - claims: Vec::new(), - children: Vec::new(), - } - } - - pub fn with_claim(mut self, claim: FeatureClaimSpec) -> Self { - self.claims.push(claim); - self - } - - pub fn with_child(mut self, child: DomainNodeSpec) -> Self { - self.children.push(child); - self - } - - pub fn compile_root(&self) -> DomainNode { - self.compile(DomainKey::new(self.name.clone()), None) - } - - fn compile(&self, key: DomainKey, parent: Option) -> DomainNode { - let claims = self - .claims - .iter() - .map(|claim| claim.compile(&key)) - .collect::>(); - let children = self - .children - .iter() - .map(|child| child.compile(key.child(&child.name), Some(key.clone()))) - .collect::>(); - DomainNode::new( - key, parent, self.kind, self.frame, self.seed, claims, children, - ) - } -} - -impl DomainNode { - pub fn new( - key: DomainKey, - parent: Option, - kind: DomainKind, - frame: DomainFrame, - seed: u64, - claims: Vec, - children: Vec, - ) -> Self { - let claim_bounds = claims - .iter() - .map(FeatureClaim::world_bounds) - .fold(Aabb::empty(), Aabb::union); - let child_bounds = children - .iter() - .map(|child| child.summary.bounds) - .fold(Aabb::empty(), Aabb::union); - let bounds = claim_bounds.union(child_bounds); - let has_children = !children.is_empty(); - let estimated_brush_cost = claims.len() - + children - .iter() - .map(|child| child.summary.estimated_brush_cost) - .sum::(); - let estimated_triangle_cost = claims.len().max(1) * 12 - + children - .iter() - .map(|child| child.summary.estimated_triangle_cost) - .sum::(); - let contribution_weight = domain_priority(kind) * bounds.size().length().max(1.0); - let fallback_claims = if claims.is_empty() { - fallback_box_claim(&key, kind, frame, bounds) - } else { - claims.clone() - }; - - Self { - key, - parent, - kind, - frame, - bounds, - seed, - summary: DomainSummary { - bounds, - estimated_triangle_cost, - estimated_brush_cost, - projected_error: bounds.size().length().max(1.0), - contribution_weight, - has_children, - fallback_claims, - }, - claims, - children, - } - } - - pub fn find(&self, key: &DomainKey) -> Option<&Self> { - if &self.key == key { - return Some(self); - } - self.children.iter().find_map(|child| child.find(key)) - } -} - -#[derive(Clone, Debug, PartialEq)] -pub struct DomainQuery { - pub camera_position: Vec3, - pub frustum: Aabb, - pub viewport_height_px: f32, - pub vertical_fov_radians: f32, - pub target_error: f32, - pub triangle_budget: usize, - pub collider_budget: usize, - pub semantic_filter: Vec, - pub requested_chunk_keys: Vec, - pub dirty_domain_keys: Vec, -} - -impl DomainQuery { - pub fn accepts_kind(&self, kind: DomainKind) -> bool { - self.semantic_filter.is_empty() || self.semantic_filter.contains(&kind) - } -} - -#[derive(Clone, Debug, PartialEq)] -pub struct ContributionRow { - pub domain_key: DomainKey, - pub kind: DomainKind, - pub contribution: f32, - pub projected_screen_error: f32, - pub semantic_priority: f32, - pub estimated_triangle_cost: usize, - pub child_cost: usize, - pub remaining_triangle_budget: usize, - pub requested: bool, - pub dirty: bool, - pub selected: bool, - pub used_fallback: bool, - pub deferred_by_budget: bool, -} - -#[derive(Clone, Debug, PartialEq)] -pub struct SelectedCut { - pub id: String, - pub selected_nodes: Vec, - pub emitted_claims: Vec, - pub deferred_children: Vec, - pub fallback_nodes: Vec, - pub diagnostics: Vec, -} - -#[derive(Clone, Debug)] -pub struct DomainChunkBuild { - pub cut: SelectedCut, - pub chunks: Vec, - pub manifest: SelectedCutManifest, -} - -impl DomainChunkBuild {} - -#[derive(Clone, Debug)] -pub struct TriangleChunk { - pub key: DomainKey, - pub selected_cut_id: String, - pub bounds: Aabb, - pub mesh: TriangleMesh, - pub collider_mesh: Option, - pub source_domain_keys: Vec, - pub source_claim_keys: Vec, - pub report: BuildReport, -} - -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -pub struct TriangleChunkManifest { - pub key: String, - pub selected_cut_id: String, - pub bounds_min: [f32; 3], - pub bounds_max: [f32; 3], - pub render_vertices: usize, - pub render_triangles: usize, - pub collider_triangles: usize, - pub source_domain_keys: Vec, - pub source_claim_keys: Vec, - pub input_brushes: usize, - pub candidate_pairs: usize, - pub rejected_pairs: usize, - pub transition_hint: ChunkTransitionHint, -} - -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -pub struct SelectedCutManifest { - pub id: String, - pub selected_nodes: Vec, - pub deferred_child_requests: Vec, - pub parent_fallback_nodes: Vec, - pub diagnostics: Vec, - pub chunks: Vec, -} - -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -pub struct ContributionManifest { - pub domain_key: String, - pub kind: DomainKind, - pub contribution: f32, - pub projected_screen_error: f32, - pub semantic_priority: f32, - pub estimated_triangle_cost: usize, - pub child_cost: usize, - pub remaining_triangle_budget: usize, - pub requested: bool, - pub dirty: bool, - pub selected: bool, - pub used_fallback: bool, - pub deferred_by_budget: bool, -} - -impl SelectedCut { - pub fn manifest(&self, chunks: &[TriangleChunk]) -> SelectedCutManifest { - SelectedCutManifest { - id: self.id.clone(), - selected_nodes: self - .selected_nodes - .iter() - .map(|key| key.0.clone()) - .collect(), - deferred_child_requests: self - .deferred_children - .iter() - .map(|key| key.0.clone()) - .collect(), - parent_fallback_nodes: self - .fallback_nodes - .iter() - .map(|key| key.0.clone()) - .collect(), - diagnostics: self - .diagnostics - .iter() - .map(ContributionManifest::from_row) - .collect(), - chunks: chunks.iter().map(TriangleChunk::manifest).collect(), - } - } -} - -impl ContributionManifest { - pub fn from_row(row: &ContributionRow) -> Self { - Self { - domain_key: row.domain_key.0.clone(), - kind: row.kind, - contribution: row.contribution, - projected_screen_error: row.projected_screen_error, - semantic_priority: row.semantic_priority, - estimated_triangle_cost: row.estimated_triangle_cost, - child_cost: row.child_cost, - remaining_triangle_budget: row.remaining_triangle_budget, - requested: row.requested, - dirty: row.dirty, - selected: row.selected, - used_fallback: row.used_fallback, - deferred_by_budget: row.deferred_by_budget, - } - } -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct ChunkTransitionHint { - pub stable_clip_seed: u64, - pub supports_parent_child_coexistence: bool, -} - -impl TriangleChunk { - pub fn manifest(&self) -> TriangleChunkManifest { - TriangleChunkManifest { - key: self.key.0.clone(), - selected_cut_id: self.selected_cut_id.clone(), - bounds_min: vec3_to_array(self.bounds.min), - bounds_max: vec3_to_array(self.bounds.max), - render_vertices: self.mesh.vertex_count(), - render_triangles: self.mesh.triangle_count(), - collider_triangles: self - .collider_mesh - .as_ref() - .map_or(0, TriangleMesh::triangle_count), - source_domain_keys: self - .source_domain_keys - .iter() - .map(|key| key.0.clone()) - .collect(), - source_claim_keys: self.source_claim_keys.clone(), - input_brushes: self.report.input_brushes, - candidate_pairs: self.report.candidate_pairs, - rejected_pairs: self.report.rejected_pairs, - transition_hint: ChunkTransitionHint { - stable_clip_seed: stable_str_hash(&self.key.0), - supports_parent_child_coexistence: true, - }, - } - } -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] -pub enum ClaimLoweringTarget { - Render, - Collider, -} - -#[derive(Clone, Debug)] -pub struct CsgClaimLowering { - pub arena: CsgTreeArena, - pub root: Option, - pub bounds: Aabb, - pub source_domain_keys: Vec, - pub source_claim_keys: Vec, -} - -pub fn select_domain_cut(root: &DomainNode, query: &DomainQuery) -> SelectedCut { - let mut state = CutState { - query, - selected: Vec::new(), - claims: Vec::new(), - deferred: Vec::new(), - fallback: Vec::new(), - diagnostics: Vec::new(), - remaining_triangles: query.triangle_budget, - remaining_colliders: query.collider_budget, - }; - state.visit(root, true); - SelectedCut { - id: stable_cut_id(&state.selected), - selected_nodes: state.selected, - emitted_claims: state.claims, - deferred_children: state.deferred, - fallback_nodes: state.fallback, - diagnostics: state.diagnostics, - } -} - -pub fn build_domain_chunks(root: &DomainNode, query: &DomainQuery) -> DomainChunkBuild { - let cut = select_domain_cut(root, query); - let chunks = lower_selected_cut_chunks(&cut); - let manifest = cut.manifest(&chunks); - DomainChunkBuild { - cut, - chunks, - manifest, - } -} - -pub fn lower_selected_cut(cut: &SelectedCut) -> TriangleChunk { - lower_claims_to_chunk( - DomainKey::new(format!("chunk/{}", cut.id)), - &cut.id, - &cut.emitted_claims, - ) -} - -pub fn lower_selected_cut_chunks(cut: &SelectedCut) -> Vec { - let mut chunks = Vec::new(); - for domain_key in &cut.selected_nodes { - let claims = cut - .emitted_claims - .iter() - .filter(|claim| &claim.domain_key == domain_key) - .cloned() - .collect::>(); - if claims.is_empty() { - continue; - } - chunks.push(lower_claims_to_chunk( - DomainKey::new(format!("chunk/{}/{}", cut.id, domain_key.0)), - &cut.id, - &claims, - )); - } - chunks -} - -pub fn lower_feature_claims_to_csg_tree( - claims: &[FeatureClaim], - target: ClaimLoweringTarget, -) -> CsgClaimLowering { - let mut arena = CsgTreeArena::new(); - let mut additive_nodes = Vec::new(); - let mut subtractive_nodes = Vec::new(); - let mut source_domain_keys = Vec::::new(); - let mut source_claim_keys = Vec::::new(); - let mut bounds = Aabb::empty(); - - for claim in claims.iter().filter(|claim| claim_in_target(claim, target)) { - source_claim_keys.push(claim.key.clone()); - if !source_domain_keys.contains(&claim.domain_key) { - source_domain_keys.push(claim.domain_key.clone()); - } - bounds = bounds.union(claim.world_bounds()); - let brush = arena.generate_brush( - claim.key.clone(), - CsgOperationType::Additive, - claim.primitive(), - claim.material, - ); - match claim.kind.brush_op() { - BrushOp::Add => additive_nodes.push(brush.node), - BrushOp::Subtract => subtractive_nodes.push(brush.node), - BrushOp::Intersect => additive_nodes.push(brush.node), - } - } - - let root = build_claim_tree(&mut arena, additive_nodes, subtractive_nodes); - CsgClaimLowering { - arena, - root, - bounds, - source_domain_keys, - source_claim_keys, - } -} - -fn lower_claims_to_chunk( - key: DomainKey, - selected_cut_id: &str, - claims: &[FeatureClaim], -) -> TriangleChunk { - let render = lower_feature_claims_to_csg_tree(claims, ClaimLoweringTarget::Render); - let collider = lower_feature_claims_to_csg_tree(claims, ClaimLoweringTarget::Collider); - let (source_domain_keys, source_claim_keys) = claim_metadata(claims); - - let output = render - .root - .map(|root| { - render - .arena - .compile_tree_to_assembler(render.arena.generate_tree(root)) - .build() - }) - .unwrap_or_else(|| crate::Assembler::new().build()); - let collider_output = collider.root.map(|root| { - collider - .arena - .compile_tree_to_assembler(collider.arena.generate_tree(root)) - .build() - }); - - TriangleChunk { - key, - selected_cut_id: selected_cut_id.to_owned(), - bounds: render.bounds.union(collider.bounds), - mesh: output.mesh, - collider_mesh: collider_output.map(|output| output.mesh), - source_domain_keys, - source_claim_keys, - report: output.report, - } -} - -fn claim_metadata(claims: &[FeatureClaim]) -> (Vec, Vec) { - let mut source_domain_keys = Vec::::new(); - let mut source_claim_keys = Vec::::new(); - for claim in claims { - source_claim_keys.push(claim.key.clone()); - if !source_domain_keys.contains(&claim.domain_key) { - source_domain_keys.push(claim.domain_key.clone()); - } - } - (source_domain_keys, source_claim_keys) -} - -fn claim_in_target(claim: &FeatureClaim, target: ClaimLoweringTarget) -> bool { - claim.lowering.emits_to(target) -} - -fn build_claim_tree( - arena: &mut CsgTreeArena, - additive_nodes: Vec, - subtractive_nodes: Vec, -) -> Option { - let additive_root = match additive_nodes.len() { - 0 => None, - 1 => additive_nodes.first().copied(), - _ => Some( - arena - .generate_branch( - "domain-additive-claims", - CsgBranchOp::Addition, - additive_nodes, - ) - .node, - ), - }?; - - if subtractive_nodes.is_empty() { - return Some(additive_root); - } - - let subtractive_root = match subtractive_nodes.len() { - 0 => unreachable!(), - 1 => subtractive_nodes[0], - _ => { - arena - .generate_branch( - "domain-subtractive-claims", - CsgBranchOp::Addition, - subtractive_nodes, - ) - .node - } - }; - - Some( - arena - .generate_branch( - "domain-claims", - CsgBranchOp::Subtraction, - [additive_root, subtractive_root], - ) - .node, - ) -} - -pub fn ragnarok_column_fixture() -> DomainNode { - ragnarok_column_spec().compile_root() -} - -pub fn ragnarok_column_spec() -> DomainNodeSpec { - let mut column = DomainNodeSpec::new( - "stellarator-column-00", - DomainKind::Column, - DomainFrame::IDENTITY, - 0xC011_0000, - ) - .with_claim(claim_spec( - "column-support-shell", - DomainFrame::IDENTITY, - Aabb::from_center_size(Vec3::new(0.0, 0.0, 45.0), Vec3::new(18.0, 18.0, 96.0)), - FeatureClaimKind::SupportShell, - MaterialId(10), - )) - .with_claim(claim_spec( - "column-core-clearance", - DomainFrame::IDENTITY, - Aabb::from_center_size(Vec3::new(0.0, 0.0, 45.0), Vec3::new(7.0, 7.0, 100.0)), - FeatureClaimKind::ClearanceVolume, - MaterialId(0), - )); - for index in 0..3 { - column = column.with_child(ragnarok_band(index)); - } - - DomainNodeSpec::new( - "ragnarok-column", - DomainKind::Root, - DomainFrame::IDENTITY, - 0x5EED, - ) - .with_child(column) -} - -fn ragnarok_band(index: usize) -> DomainNodeSpec { - let z = 15.0 + index as f32 * 30.0; - let frame = DomainFrame::translated(Vec3::new(0.0, 0.0, z)); - let mut band = DomainNodeSpec::new( - format!("altitude-band-{index}"), - DomainKind::AltitudeBand, - frame, - 0xBADD_0000 + index as u64, - ) - .with_claim(claim_spec( - "coarse-ring-road", - frame, - Aabb::from_center_size(Vec3::ZERO, Vec3::new(34.0, 5.0, 1.0)), - FeatureClaimKind::RoadSurfaceSlab, - MaterialId(20 + index as u32), - )); - for lane in 0..2 { - band = band.with_child(ragnarok_branch(index, lane)); - } - band.with_child(ragnarok_roundabout(index)) -} - -fn ragnarok_branch(band: usize, lane: usize) -> DomainNodeSpec { - let angle = band as f32 * 0.41 + lane as f32 * std::f32::consts::PI; - let radius = 16.0 + lane as f32 * 5.0; - let frame = DomainFrame::rotated_z( - Vec3::new(angle.cos() * radius, angle.sin() * radius, 0.0), - angle, - ); - let mut branch = DomainNodeSpec::new( - format!("branch-road-{lane}"), - DomainKind::BranchRoad, - frame, - 0xA11E_0000 + (band * 10 + lane) as u64, - ) - .with_claim(claim_spec( - "road-slab", - frame, - Aabb::from_center_size(Vec3::new(7.0, 0.0, 0.0), Vec3::new(18.0, 4.0, 0.8)), - FeatureClaimKind::RoadSurfaceSlab, - MaterialId(40 + band as u32), - )) - .with_claim(claim_spec( - "hover-clearance", - frame, - Aabb::from_center_size(Vec3::new(7.0, 0.0, 2.2), Vec3::new(17.0, 3.0, 2.0)), - FeatureClaimKind::ClearanceVolume, - MaterialId(0), - )) - .with_claim(claim_spec( - "coarse-support-rib", - frame, - Aabb::from_center_size(Vec3::new(7.0, 0.0, -1.0), Vec3::new(18.5, 1.0, 1.5)), - FeatureClaimKind::SupportShell, - MaterialId(12), - )); - for segment in 0..3 { - branch = branch.with_child(ragnarok_road_chunk(band, lane, segment, frame)); - } - branch -} - -fn ragnarok_roundabout(band: usize) -> DomainNodeSpec { - let frame = DomainFrame::translated(Vec3::ZERO); - let mut roundabout = DomainNodeSpec::new( - "roundabout", - DomainKind::Roundabout, - frame, - 0xF00D_0000 + band as u64, - ) - .with_claim(claim_spec( - "roundabout-east-west", - frame, - Aabb::from_center_size(Vec3::ZERO, Vec3::new(26.0, 4.5, 0.9)), - FeatureClaimKind::RoadSurfaceSlab, - MaterialId(60 + band as u32), - )) - .with_claim(claim_spec( - "roundabout-north-south", - DomainFrame::rotated_z(Vec3::ZERO, std::f32::consts::FRAC_PI_2), - Aabb::from_center_size(Vec3::ZERO, Vec3::new(26.0, 4.5, 0.9)), - FeatureClaimKind::RoadSurfaceSlab, - MaterialId(60 + band as u32), - )); - for quadrant in 0..4 { - roundabout = roundabout.with_child(ragnarok_roundabout_chunk(band, quadrant)); - } - roundabout -} - -fn ragnarok_road_chunk( - band: usize, - lane: usize, - segment: usize, - frame: DomainFrame, -) -> DomainNodeSpec { - let x = 1.5 + segment as f32 * 5.5; - DomainNodeSpec::new( - format!("chunk-{segment}"), - DomainKind::Chunk, - frame, - 0xC40B_0000 + (band * 100 + lane * 10 + segment) as u64, - ) - .with_claim(claim_spec( - "road-slab", - frame, - Aabb::from_center_size(Vec3::new(x, 0.0, 0.0), Vec3::new(6.5, 4.0, 0.75)), - FeatureClaimKind::RoadSurfaceSlab, - MaterialId(80 + band as u32), - )) - .with_claim(claim_spec( - "hover-clearance", - frame, - Aabb::from_center_size(Vec3::new(x, 0.0, 2.1), Vec3::new(6.0, 3.0, 1.9)), - FeatureClaimKind::ClearanceVolume, - MaterialId(0), - )) - .with_claim(claim_spec( - "support-rib", - frame, - Aabb::from_center_size(Vec3::new(x, 0.0, -1.05), Vec3::new(6.8, 0.9, 1.4)), - FeatureClaimKind::SupportShell, - MaterialId(90 + lane as u32), - )) - .with_claim(claim_spec( - "collider-proxy", - frame, - Aabb::from_center_size(Vec3::new(x, 0.0, 0.35), Vec3::new(6.8, 4.2, 0.35)), - FeatureClaimKind::ColliderProxy, - MaterialId(0), - )) -} - -fn ragnarok_roundabout_chunk(band: usize, quadrant: usize) -> DomainNodeSpec { - let angle = quadrant as f32 * std::f32::consts::FRAC_PI_2; - let frame = DomainFrame::rotated_z(Vec3::ZERO, angle); - DomainNodeSpec::new( - format!("chunk-{quadrant}"), - DomainKind::Chunk, - frame, - 0xC10C_0000 + (band * 10 + quadrant) as u64, - ) - .with_claim(claim_spec( - "arc-road-slab", - frame, - Aabb::from_center_size(Vec3::new(5.0, 5.0, 0.0), Vec3::new(12.0, 4.0, 0.8)), - FeatureClaimKind::RoadSurfaceSlab, - MaterialId(100 + band as u32), - )) - .with_claim(claim_spec( - "arc-clearance", - frame, - Aabb::from_center_size(Vec3::new(5.0, 5.0, 2.1), Vec3::new(11.0, 3.0, 1.8)), - FeatureClaimKind::ClearanceVolume, - MaterialId(0), - )) - .with_claim(claim_spec( - "arc-support", - frame, - Aabb::from_center_size(Vec3::new(5.0, 5.0, -1.0), Vec3::new(12.5, 0.8, 1.2)), - FeatureClaimKind::SupportShell, - MaterialId(90), - )) -} - -fn claim_spec( - name: &str, - frame: DomainFrame, - support: Aabb, - kind: FeatureClaimKind, - material: MaterialId, -) -> FeatureClaimSpec { - FeatureClaimSpec::new(name, frame, support, kind, material) -} - -fn vec3_to_array(value: Vec3) -> [f32; 3] { - [value.x, value.y, value.z] -} - -fn array_to_vec3(value: [f32; 3]) -> Vec3 { - Vec3::new(value[0], value[1], value[2]) -} - -fn quat_to_array(value: Quat) -> [f32; 4] { - [value.x, value.y, value.z, value.w] -} - -fn array_to_quat(value: [f32; 4]) -> Quat { - Quat::from_xyzw(value[0], value[1], value[2], value[3]) -} - -fn fallback_box_claim( - key: &DomainKey, - kind: DomainKind, - frame: DomainFrame, - bounds: Aabb, -) -> Vec { - if !bounds.is_valid() { - return Vec::new(); - } - let kind = match kind { - DomainKind::ClearanceVolume => FeatureClaimKind::ClearanceVolume, - DomainKind::RoadSpine | DomainKind::BranchRoad | DomainKind::Roundabout => { - FeatureClaimKind::RoadSurfaceSlab - } - _ => FeatureClaimKind::SupportShell, - }; - vec![FeatureClaim { - key: format!("{}/claim/fallback-summary", key.0), - domain_key: key.clone(), - frame, - support: Aabb::from_center_size(Vec3::ZERO, bounds.size()), - kind, - material: MaterialId(1), - lowering: FeatureLoweringPolicy::default_for_kind(kind), - }] -} - -fn domain_priority(kind: DomainKind) -> f32 { - match kind { - DomainKind::Root => 0.5, - DomainKind::Column => 1.0, - DomainKind::AltitudeBand => 1.4, - DomainKind::RoadSpine | DomainKind::BranchRoad | DomainKind::Roundabout => 2.0, - DomainKind::SupportMass => 1.1, - DomainKind::ClearanceVolume => 1.8, - DomainKind::Chunk => 1.0, - } -} - -fn contribution(node: &DomainNode, query: &DomainQuery) -> f32 { - let projected_error = projected_screen_error(node, query); - let stale_multiplier = if query.dirty_domain_keys.contains(&node.key) { - 1.5 - } else { - 1.0 - }; - let cost_pressure = selection_triangle_cost(node).max(1) as f32; - (projected_error * domain_priority(node.kind) * stale_multiplier) / cost_pressure.sqrt() -} - -fn projected_screen_error(node: &DomainNode, query: &DomainQuery) -> f32 { - let center = node.summary.bounds.center(); - let distance = center.distance(query.camera_position).max(1.0); - let radius = node.summary.bounds.size().length() * 0.5; - let focal_pixels = query.viewport_height_px.max(1.0) - / (2.0 * (query.vertical_fov_radians.max(0.01) * 0.5).tan()); - (radius / distance) * focal_pixels -} - -fn stable_cut_id(keys: &[DomainKey]) -> String { - let mut hash = FNV_OFFSET_BASIS; - for key in keys { - hash = stable_hash_bytes(hash, key.0.as_bytes()); - } - format!("cut-{hash:016x}") -} - -const FNV_OFFSET_BASIS: u64 = 0xcbf2_9ce4_8422_2325; -const FNV_PRIME: u64 = 0x1000_0000_01b3; - -fn stable_str_hash(value: &str) -> u64 { - stable_hash_bytes(FNV_OFFSET_BASIS, value.as_bytes()) -} - -fn stable_hash_bytes(mut hash: u64, bytes: &[u8]) -> u64 { - for byte in bytes { - hash ^= u64::from(*byte); - hash = hash.wrapping_mul(FNV_PRIME); - } - hash -} - -struct CutState<'a> { - query: &'a DomainQuery, - selected: Vec, - claims: Vec, - deferred: Vec, - fallback: Vec, - diagnostics: Vec, - remaining_triangles: usize, - remaining_colliders: usize, -} - -impl CutState<'_> { - fn visit(&mut self, node: &DomainNode, force_visit: bool) { - let requested_self = self.query.requested_chunk_keys.contains(&node.key); - let requested_descendant = has_requested_descendant(node, &self.query.requested_chunk_keys); - let dirty_descendant = has_requested_descendant(node, &self.query.dirty_domain_keys); - if !force_visit - && !requested_self - && !requested_descendant - && !dirty_descendant - && !node.summary.bounds.intersects(self.query.frustum) - { - return; - } - let score = contribution(node, self.query); - let projected_error = projected_screen_error(node, self.query); - let semantic_priority = domain_priority(node.kind); - let dirty = self.query.dirty_domain_keys.contains(&node.key); - let child_cost = node - .children - .iter() - .map(selection_triangle_cost) - .sum::(); - let child_collider_cost = node - .children - .iter() - .map(selection_collider_cost) - .sum::(); - let budget_allows_children = child_cost <= self.remaining_triangles - && child_collider_cost <= self.remaining_colliders; - let descend_for_detail = score >= self.query.target_error && budget_allows_children; - let descend_for_request = requested_descendant && !requested_self; - let descend_for_dirty = dirty_descendant && budget_allows_children; - let should_descend = node.summary.has_children - && !requested_self - && (descend_for_request || descend_for_dirty || descend_for_detail); - self.diagnostics.push(ContributionRow { - domain_key: node.key.clone(), - kind: node.kind, - contribution: score, - projected_screen_error: projected_error, - semantic_priority, - estimated_triangle_cost: node.summary.estimated_triangle_cost, - child_cost, - remaining_triangle_budget: self.remaining_triangles, - requested: requested_self, - dirty, - selected: !should_descend && self.query.accepts_kind(node.kind), - used_fallback: !should_descend && node.summary.has_children, - deferred_by_budget: node.summary.has_children - && !requested_self - && score >= self.query.target_error - && !budget_allows_children, - }); - - if should_descend { - if budget_allows_children { - self.remaining_triangles = self.remaining_triangles.saturating_sub(child_cost); - self.remaining_colliders = - self.remaining_colliders.saturating_sub(child_collider_cost); - } - for child in &node.children { - let child_requested = self.query.requested_chunk_keys.contains(&child.key) - || has_requested_descendant(child, &self.query.requested_chunk_keys); - let child_dirty = self.query.dirty_domain_keys.contains(&child.key) - || has_requested_descendant(child, &self.query.dirty_domain_keys); - self.visit(child, child_requested || child_dirty); - } - return; - } - - for child in &node.children { - self.deferred.push(child.key.clone()); - } - if self.query.accepts_kind(node.kind) { - self.selected.push(node.key.clone()); - let claims = if node.summary.has_children { - self.fallback.push(node.key.clone()); - &node.summary.fallback_claims - } else { - &node.claims - }; - self.claims.extend(claims.iter().cloned()); - } - } -} - -fn selection_triangle_cost(node: &DomainNode) -> usize { - let claim_count = if node.summary.has_children { - node.summary.fallback_claims.len() - } else { - node.claims.len() - }; - claim_count.max(1) * 12 -} - -fn selection_collider_cost(node: &DomainNode) -> usize { - let claims = if node.summary.has_children { - &node.summary.fallback_claims - } else { - &node.claims - }; - claims - .iter() - .filter(|claim| claim.lowering.emits_to(ClaimLoweringTarget::Collider)) - .count() - .max(1) - * 12 -} - -fn has_requested_descendant(node: &DomainNode, requested: &[DomainKey]) -> bool { - node.children - .iter() - .any(|child| requested.contains(&child.key) || has_requested_descendant(child, requested)) -} - -#[cfg(test)] -mod tests { - use super::*; - - fn query(target_error: f32, triangle_budget: usize) -> DomainQuery { - DomainQuery { - camera_position: Vec3::new(36.0, -42.0, 30.0), - frustum: Aabb::from_center_size(Vec3::new(0.0, 0.0, 45.0), Vec3::splat(150.0)), - viewport_height_px: 1080.0, - vertical_fov_radians: std::f32::consts::FRAC_PI_3, - target_error, - triangle_budget, - collider_budget: triangle_budget, - semantic_filter: Vec::new(), - requested_chunk_keys: Vec::new(), - dirty_domain_keys: Vec::new(), - } - } - - #[test] - fn ragnarok_domain_keys_are_stable() { - let a = ragnarok_column_fixture(); - let b = ragnarok_column_fixture(); - assert_eq!(a.key, b.key); - assert_eq!(a.children[0].children[1].key, b.children[0].children[1].key); - } - - #[test] - fn domain_tree_spec_compiles_stable_parent_claim_identity() { - let spec = DomainNodeSpec::new("root", DomainKind::Root, DomainFrame::IDENTITY, 1) - .with_child( - DomainNodeSpec::new("road", DomainKind::BranchRoad, DomainFrame::IDENTITY, 2) - .with_claim(FeatureClaimSpec::new( - "slab", - DomainFrame::IDENTITY, - Aabb::from_center_size(Vec3::ZERO, Vec3::ONE), - FeatureClaimKind::RoadSurfaceSlab, - MaterialId(1), - )), - ); - let domain = spec.compile_root(); - let road = &domain.children[0]; - assert_eq!(road.key, DomainKey::new("root/road")); - assert_eq!(road.parent, Some(domain.key.clone())); - assert_eq!(road.claims[0].key, "root/road/claim/slab"); - assert_eq!(road.claims[0].domain_key, road.key); - assert_eq!( - road.claims[0].lowering, - FeatureLoweringPolicy::RenderAndCollider - ); - } - - #[test] - fn domain_summary_contains_child_bounds() { - let fixture = ragnarok_column_fixture(); - let column = &fixture.children[0]; - for band in &column.children { - assert!( - column - .summary - .bounds - .contains_point(band.summary.bounds.center()) - ); - } - } - - #[test] - fn selected_cut_chooses_parent_under_tight_budget_and_children_when_relaxed() { - let fixture = ragnarok_column_fixture(); - let tight = select_domain_cut(&fixture, &query(10_000.0, 100)); - let relaxed = select_domain_cut(&fixture, &query(0.01, 10_000)); - assert!(tight.selected_nodes.len() < relaxed.selected_nodes.len()); - assert!( - tight - .fallback_nodes - .iter() - .any(|key| key.0.contains("ragnarok-column")) - ); - assert!( - relaxed - .selected_nodes - .iter() - .any(|key| key.0.contains("branch-road")) - ); - } - - #[test] - fn selected_cut_honors_collider_budget() { - let fixture = ragnarok_column_fixture(); - let mut collider_tight = query(0.01, 10_000); - collider_tight.collider_budget = 1; - let cut = select_domain_cut(&fixture, &collider_tight); - assert_eq!(cut.selected_nodes, vec![fixture.key.clone()]); - assert!(cut.diagnostics[0].deferred_by_budget); - } - - #[test] - fn selected_cut_reports_projected_error_and_dirty_pressure() { - let fixture = ragnarok_column_fixture(); - let dirty_key = fixture.key.clone(); - let mut request = query(10_000.0, 100); - request.dirty_domain_keys.push(dirty_key.clone()); - let cut = select_domain_cut(&fixture, &request); - let row = cut - .diagnostics - .iter() - .find(|row| row.domain_key == dirty_key) - .unwrap(); - assert!(row.projected_screen_error > 0.0); - assert_eq!(row.semantic_priority, domain_priority(row.kind)); - assert!(row.dirty); - } - - #[test] - fn dirty_descendant_pulls_selection_down_when_budget_allows() { - let fixture = ragnarok_column_fixture(); - let dirty_key = fixture.children[0].children[0].children[0].children[0] - .key - .clone(); - let mut request = query(10_000.0, 10_000); - request.dirty_domain_keys.push(dirty_key.clone()); - let cut = select_domain_cut(&fixture, &request); - assert!(cut.selected_nodes.contains(&dirty_key)); - assert!( - cut.diagnostics - .iter() - .any(|row| row.domain_key == dirty_key && row.dirty) - ); - } - - #[test] - fn missing_children_degrade_to_parent_fallback_claims() { - let fixture = ragnarok_column_fixture(); - let cut = select_domain_cut(&fixture, &query(10_000.0, 100)); - assert!(!cut.fallback_nodes.is_empty()); - assert!(!cut.emitted_claims.is_empty()); - } - - #[test] - fn feature_claims_lower_deterministically_to_triangle_chunks() { - let fixture = ragnarok_column_fixture(); - let cut = select_domain_cut(&fixture, &query(0.01, 10_000)); - let a = lower_selected_cut(&cut); - let b = lower_selected_cut(&cut); - assert_eq!(a.source_claim_keys, b.source_claim_keys); - assert_eq!(a.mesh.positions, b.mesh.positions); - assert_eq!(a.mesh.indices, b.mesh.indices); - } - - #[test] - fn triangle_chunks_preserve_source_domain_and_claim_ids() { - let fixture = ragnarok_column_fixture(); - let cut = select_domain_cut(&fixture, &query(0.01, 10_000)); - let chunk = lower_selected_cut(&cut); - assert!(!chunk.source_domain_keys.is_empty()); - assert!(!chunk.source_claim_keys.is_empty()); - assert!(chunk.collider_mesh.is_some()); - } - - #[test] - fn feature_claims_lower_through_csg_tree_branches() { - let fixture = ragnarok_column_fixture(); - let cut = select_domain_cut(&fixture, &query(0.01, 10_000)); - let lowering = - lower_feature_claims_to_csg_tree(&cut.emitted_claims, ClaimLoweringTarget::Render); - assert!(lowering.root.is_some()); - assert!(lowering.arena.brush_count() > 0); - assert!(lowering.arena.branch_count() > 0); - } - - #[test] - fn lowering_policy_controls_render_and_collider_streams() { - let fixture = ragnarok_column_fixture(); - let cut = select_domain_cut(&fixture, &query(0.01, 10_000)); - let render = - lower_feature_claims_to_csg_tree(&cut.emitted_claims, ClaimLoweringTarget::Render); - let collider = - lower_feature_claims_to_csg_tree(&cut.emitted_claims, ClaimLoweringTarget::Collider); - assert!( - !render - .source_claim_keys - .iter() - .any(|key| key.ends_with("/claim/collider-proxy")) - ); - assert!( - collider - .source_claim_keys - .iter() - .any(|key| key.ends_with("/claim/collider-proxy")) - ); - assert!( - render - .source_claim_keys - .iter() - .any(|key| key.ends_with("/claim/hover-clearance")) - ); - } - - #[test] - fn selected_cut_chunks_emit_per_selected_domain() { - let fixture = ragnarok_column_fixture(); - let cut = select_domain_cut(&fixture, &query(0.01, 10_000)); - let chunks = lower_selected_cut_chunks(&cut); - assert_eq!(chunks.len(), cut.selected_nodes.len()); - for chunk in chunks { - assert_eq!(chunk.source_domain_keys.len(), 1); - assert!(cut.selected_nodes.contains(&chunk.source_domain_keys[0])); - assert!(chunk.mesh.triangle_count() > 0); - } - } - - #[test] - fn triangle_chunk_manifest_preserves_streaming_metadata() { - let fixture = ragnarok_column_fixture(); - let cut = select_domain_cut(&fixture, &query(0.01, 10_000)); - let chunks = lower_selected_cut_chunks(&cut); - let manifest = chunks[0].manifest(); - assert_eq!(manifest.selected_cut_id, cut.id); - assert!(manifest.render_triangles > 0); - assert!(!manifest.source_domain_keys.is_empty()); - assert!(manifest.transition_hint.supports_parent_child_coexistence); - assert_ne!(manifest.transition_hint.stable_clip_seed, 0); - } - - #[test] - fn selected_cut_manifest_carries_worker_artifact_context() { - let fixture = ragnarok_column_fixture(); - let cut = select_domain_cut(&fixture, &query(0.01, 10_000)); - let chunks = lower_selected_cut_chunks(&cut); - let manifest = cut.manifest(&chunks); - assert_eq!(manifest.id, cut.id); - assert_eq!(manifest.chunks.len(), chunks.len()); - assert_eq!(manifest.selected_nodes.len(), cut.selected_nodes.len()); - assert!(!manifest.diagnostics.is_empty()); - } - - #[test] - fn build_domain_chunks_runs_full_runtime_pipeline() { - let fixture = ragnarok_column_fixture(); - let build = build_domain_chunks(&fixture, &query(0.01, 10_000)); - assert_eq!(build.manifest.id, build.cut.id); - assert_eq!(build.manifest.chunks.len(), build.chunks.len()); - assert!(!build.chunks.is_empty()); - assert!( - build - .chunks - .iter() - .all(|chunk| !chunk.mesh.indices.is_empty()) - ); - } - - #[test] - fn requested_chunk_bypasses_frustum_and_budget() { - let fixture = ragnarok_column_fixture(); - let requested = fixture.children[0].children[2].children[0].children[1] - .key - .clone(); - let mut request = query(0.01, 1); - request.frustum = - Aabb::from_center_size(Vec3::new(10_000.0, 10_000.0, 10_000.0), Vec3::splat(1.0)); - request.requested_chunk_keys.push(requested.clone()); - let cut = select_domain_cut(&fixture, &request); - assert_eq!(cut.selected_nodes, vec![requested.clone()]); - assert!( - cut.diagnostics - .iter() - .any(|row| row.domain_key == requested && row.requested) - ); - } - - #[test] - fn ragnarok_fixture_emits_lod_chunks_and_preserves_parent_for_transition() { - let fixture = ragnarok_column_fixture(); - let tight_cut = select_domain_cut(&fixture, &query(10_000.0, 100)); - let medium_cut = select_domain_cut(&fixture, &query(0.01, 560)); - let relaxed_cut = select_domain_cut(&fixture, &query(0.01, 10_000)); - let tight = lower_selected_cut(&tight_cut); - let medium = lower_selected_cut(&medium_cut); - let relaxed = lower_selected_cut(&relaxed_cut); - assert!(tight.mesh.triangle_count() > 0); - assert!(medium.mesh.triangle_count() > tight.mesh.triangle_count()); - assert!(relaxed.mesh.triangle_count() > medium.mesh.triangle_count()); - assert!(medium_cut.selected_nodes.len() > tight_cut.selected_nodes.len()); - assert!(relaxed_cut.selected_nodes.len() > medium_cut.selected_nodes.len()); - assert!(!tight_cut.fallback_nodes.is_empty()); - assert!(!relaxed_cut.selected_nodes.is_empty()); - assert!( - relaxed_cut - .selected_nodes - .iter() - .any(|key| key.0.contains("/chunk-")) - ); - } - - #[test] - fn clearance_voids_leave_no_triangle_centers_inside_strict_bounds() { - let fixture = ragnarok_column_fixture(); - let cut = select_domain_cut(&fixture, &query(0.01, 10_000)); - let chunk = lower_selected_cut(&cut); - for clearance in cut - .emitted_claims - .iter() - .filter(|claim| claim.kind == FeatureClaimKind::ClearanceVolume) - { - let bounds = clearance.world_bounds(); - for tri in chunk.mesh.indices.chunks_exact(3) { - let center = (chunk.mesh.positions[tri[0] as usize] - + chunk.mesh.positions[tri[1] as usize] - + chunk.mesh.positions[tri[2] as usize]) - / 3.0; - assert!(!bounds.contains_point_strict(center, 1.0e-4)); - } - } - } -} diff --git a/packages/gamecult-geometry-rs/src/dsl.rs b/packages/gamecult-geometry-rs/src/dsl.rs deleted file mode 100644 index 77e31e68..00000000 --- a/packages/gamecult-geometry-rs/src/dsl.rs +++ /dev/null @@ -1,164 +0,0 @@ -#![allow(dead_code)] -use bevy_math::{Quat, Vec3}; - -use crate::{ - Aabb, Assembler, BrushId, BrushOp, BuildOutput, DomeCapZSpec, FloretArmSpec, MaterialId, - Primitive, -}; - -#[derive(Clone, Debug, Default)] -pub struct LevelDsl { - assembler: Assembler, -} - -impl LevelDsl { - pub fn new() -> Self { - Self::default() - } - - pub fn solid_box( - &mut self, - name: impl Into, - center: Vec3, - size: Vec3, - material: MaterialId, - ) -> BrushId { - self.assembler - .solid_box(name, Aabb::from_center_size(center, size), material) - } - - pub fn cut_box(&mut self, name: impl Into, center: Vec3, size: Vec3) -> BrushId { - self.assembler - .cut_box(name, Aabb::from_center_size(center, size)) - } - - pub fn solid_oriented_box( - &mut self, - name: impl Into, - center: Vec3, - size: Vec3, - rotation: Quat, - material: MaterialId, - ) -> BrushId { - self.assembler - .solid_oriented_box(name, center, size, rotation, material) - } - - pub fn cut_oriented_box( - &mut self, - name: impl Into, - center: Vec3, - size: Vec3, - rotation: Quat, - ) -> BrushId { - self.assembler - .cut_oriented_box(name, center, size, rotation) - } - - pub fn cylinder_z( - &mut self, - name: impl Into, - center: Vec3, - radius: f32, - depth: f32, - segments: usize, - material: MaterialId, - ) -> BrushId { - self.assembler.add_brush( - name, - BrushOp::Add, - Primitive::CylinderZ { - center, - radius, - depth, - segments, - }, - material, - ) - } - - pub fn dome_cap_z(&mut self, name: impl Into, spec: DomeCapZSpec) -> BrushId { - self.assembler.add_brush( - name, - BrushOp::Add, - Primitive::DomeCapZ { - center: spec.center, - radius: spec.radius, - height: spec.height, - rings: spec.rings, - segments: spec.segments, - }, - spec.material, - ) - } - - pub fn floret_arm(&mut self, name: impl Into, spec: FloretArmSpec) -> BrushId { - self.assembler.add_brush( - name, - BrushOp::Add, - Primitive::FloretArm { - anchor: spec.anchor, - direction: spec.direction, - length: spec.length, - root_width: spec.root_width, - tip_width: spec.tip_width, - thickness: spec.thickness, - tip_lift: spec.tip_lift, - }, - spec.material, - ) - } - - pub fn assemble(&self) -> BuildOutput { - self.assembler.build() - } - - pub fn into_assembler(self) -> Assembler { - self.assembler - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn dsl_builds_room_and_habitat_primitives() { - let mut dsl = LevelDsl::new(); - dsl.solid_box( - "wall", - Vec3::new(0.0, 0.0, 1.5), - Vec3::new(5.0, 0.25, 3.0), - MaterialId(1), - ); - dsl.cut_box("door", Vec3::new(0.0, 0.0, 1.0), Vec3::new(1.0, 0.5, 2.0)); - dsl.dome_cap_z( - "city dome", - DomeCapZSpec { - center: Vec3::new(0.0, 0.0, 0.0), - radius: 3.0, - height: 1.5, - rings: 4, - segments: 16, - material: MaterialId(2), - }, - ); - dsl.floret_arm( - "solar petal", - FloretArmSpec { - anchor: Vec3::new(3.0, 0.0, 0.0), - direction: Vec3::X, - length: 12.0, - root_width: 1.2, - tip_width: 3.0, - thickness: 0.08, - tip_lift: 0.6, - material: MaterialId(3), - }, - ); - - let output = dsl.assemble(); - assert!(output.mesh.triangle_count() > 12); - assert!(output.report.warnings.is_empty()); - } -} diff --git a/packages/gamecult-geometry-rs/src/frontier.rs b/packages/gamecult-geometry-rs/src/frontier.rs deleted file mode 100644 index e48af14f..00000000 --- a/packages/gamecult-geometry-rs/src/frontier.rs +++ /dev/null @@ -1,321 +0,0 @@ -#![allow(dead_code)] -use crate::{Brush, BrushOp}; - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct DemandPair { - pub source_index: usize, - pub operator_index: usize, -} - -#[derive(Clone, Debug, Default, PartialEq, Eq)] -pub struct DemandFrontier { - pub pairs: Vec, - pub rejected_pairs: usize, - pub operator_brushes: usize, -} - -#[derive(Clone, Debug, Default, PartialEq, Eq)] -pub struct DirtyDemandFrontier { - pub first_dirty_index: Option, - pub pairs: Vec, - pub rejected_pairs: usize, - pub operator_brushes: usize, - pub skipped_prefix_pairs: usize, -} - -impl DemandFrontier { - pub fn from_ordered_brushes(brushes: &[Brush]) -> Self { - let mut frontier = Self::default(); - let mut active_sources = Vec::::new(); - - for (operator_index, brush) in brushes.iter().enumerate() { - match brush.op { - BrushOp::Add => active_sources.push(operator_index), - BrushOp::Subtract => { - frontier.operator_brushes += 1; - frontier.collect_pairs(brushes, &active_sources, operator_index); - } - BrushOp::Intersect => { - frontier.operator_brushes += 1; - let retained = frontier.collect_pairs(brushes, &active_sources, operator_index); - active_sources = retained; - } - } - } - - frontier - } - - pub fn candidate_pairs(&self) -> usize { - self.pairs.len() - } - - fn collect_pairs( - &mut self, - brushes: &[Brush], - active_sources: &[usize], - operator_index: usize, - ) -> Vec { - let operator_bounds = brushes[operator_index].bounds(); - let mut retained = Vec::new(); - - for &source_index in active_sources { - if brushes[source_index].bounds().intersects(operator_bounds) { - self.pairs.push(DemandPair { - source_index, - operator_index, - }); - retained.push(source_index); - } else { - self.rejected_pairs += 1; - } - } - - retained - } -} - -impl DirtyDemandFrontier { - pub fn from_dirty_indices( - brushes: &[Brush], - dirty_indices: impl IntoIterator, - ) -> Self { - let first_dirty_index = dirty_indices - .into_iter() - .filter(|index| *index < brushes.len()) - .min(); - let Some(first_dirty_index) = first_dirty_index else { - return Self::default(); - }; - - let mut frontier = Self { - first_dirty_index: Some(first_dirty_index), - ..Self::default() - }; - let mut active_sources = Vec::::new(); - - for (operator_index, brush) in brushes.iter().enumerate() { - match brush.op { - BrushOp::Add => active_sources.push(operator_index), - BrushOp::Subtract => { - frontier.operator_brushes += usize::from(operator_index >= first_dirty_index); - frontier.collect_pairs( - brushes, - &active_sources, - operator_index, - first_dirty_index, - ); - } - BrushOp::Intersect => { - frontier.operator_brushes += usize::from(operator_index >= first_dirty_index); - let retained = frontier.collect_pairs( - brushes, - &active_sources, - operator_index, - first_dirty_index, - ); - active_sources = retained; - } - } - } - - frontier - } - - pub fn candidate_pairs(&self) -> usize { - self.pairs.len() - } - - fn collect_pairs( - &mut self, - brushes: &[Brush], - active_sources: &[usize], - operator_index: usize, - first_dirty_index: usize, - ) -> Vec { - let operator_bounds = brushes[operator_index].bounds(); - let mut retained = Vec::new(); - - for &source_index in active_sources { - if brushes[source_index].bounds().intersects(operator_bounds) { - if operator_index >= first_dirty_index { - self.pairs.push(DemandPair { - source_index, - operator_index, - }); - } else { - self.skipped_prefix_pairs += 1; - } - retained.push(source_index); - } else if operator_index >= first_dirty_index { - self.rejected_pairs += 1; - } else { - self.skipped_prefix_pairs += 1; - } - } - - retained - } -} - -#[cfg(test)] -mod tests { - use bevy_math::{Quat, Vec3}; - - use super::*; - use crate::{Aabb, Brush, MaterialId, Primitive}; - - fn brush(name: &str, op: BrushOp, bounds: Aabb) -> Brush { - Brush::new( - crate::BrushId(0), - name, - op, - Primitive::Box { bounds }, - MaterialId(1), - ) - } - - #[test] - fn frontier_keeps_only_touching_ordered_pairs() { - let brushes = vec![ - brush( - "source", - BrushOp::Add, - Aabb::from_center_size(Vec3::ZERO, Vec3::splat(4.0)), - ), - brush( - "far cutter", - BrushOp::Subtract, - Aabb::from_center_size(Vec3::splat(20.0), Vec3::ONE), - ), - brush( - "near cutter", - BrushOp::Subtract, - Aabb::from_center_size(Vec3::ZERO, Vec3::ONE), - ), - ]; - - let frontier = DemandFrontier::from_ordered_brushes(&brushes); - - assert_eq!(frontier.operator_brushes, 2); - assert_eq!(frontier.rejected_pairs, 1); - assert_eq!( - frontier.pairs, - vec![DemandPair { - source_index: 0, - operator_index: 2, - }] - ); - } - - #[test] - fn intersection_frontier_reduces_active_sources() { - let brushes = vec![ - brush( - "a", - BrushOp::Add, - Aabb::from_center_size(Vec3::ZERO, Vec3::splat(2.0)), - ), - brush( - "b", - BrushOp::Add, - Aabb::from_center_size(Vec3::new(20.0, 0.0, 0.0), Vec3::splat(2.0)), - ), - Brush::new( - crate::BrushId(2), - "common", - BrushOp::Intersect, - Primitive::OrientedBox { - center: Vec3::ZERO, - size: Vec3::splat(3.0), - rotation: Quat::IDENTITY, - }, - MaterialId(1), - ), - brush( - "cut", - BrushOp::Subtract, - Aabb::from_center_size(Vec3::ZERO, Vec3::ONE), - ), - ]; - - let frontier = DemandFrontier::from_ordered_brushes(&brushes); - - assert_eq!(frontier.operator_brushes, 2); - assert_eq!(frontier.rejected_pairs, 1); - assert_eq!( - frontier.pairs, - vec![ - DemandPair { - source_index: 0, - operator_index: 2, - }, - DemandPair { - source_index: 0, - operator_index: 3, - }, - ] - ); - } - - #[test] - fn dirty_frontier_preserves_prefix_before_first_dirty_brush() { - let brushes = vec![ - brush( - "source", - BrushOp::Add, - Aabb::from_center_size(Vec3::ZERO, Vec3::splat(8.0)), - ), - brush( - "early", - BrushOp::Subtract, - Aabb::from_center_size(Vec3::new(-2.0, 0.0, 0.0), Vec3::splat(2.0)), - ), - brush( - "dirty", - BrushOp::Subtract, - Aabb::from_center_size(Vec3::ZERO, Vec3::splat(2.0)), - ), - brush( - "later", - BrushOp::Subtract, - Aabb::from_center_size(Vec3::new(2.0, 0.0, 0.0), Vec3::splat(2.0)), - ), - ]; - - let frontier = DirtyDemandFrontier::from_dirty_indices(&brushes, [2]); - - assert_eq!(frontier.first_dirty_index, Some(2)); - assert_eq!(frontier.operator_brushes, 2); - assert_eq!(frontier.skipped_prefix_pairs, 1); - assert_eq!(frontier.candidate_pairs(), 2); - assert_eq!( - frontier.pairs, - vec![ - DemandPair { - source_index: 0, - operator_index: 2, - }, - DemandPair { - source_index: 0, - operator_index: 3, - }, - ] - ); - } - - #[test] - fn dirty_frontier_ignores_out_of_range_dirty_indices() { - let brushes = vec![brush( - "source", - BrushOp::Add, - Aabb::from_center_size(Vec3::ZERO, Vec3::splat(2.0)), - )]; - - let frontier = DirtyDemandFrontier::from_dirty_indices(&brushes, [99]); - - assert_eq!(frontier.first_dirty_index, None); - assert_eq!(frontier.candidate_pairs(), 0); - assert_eq!(frontier.operator_brushes, 0); - } -} diff --git a/packages/gamecult-geometry-rs/src/lib.rs b/packages/gamecult-geometry-rs/src/lib.rs deleted file mode 100644 index 04727ed1..00000000 --- a/packages/gamecult-geometry-rs/src/lib.rs +++ /dev/null @@ -1,695 +0,0 @@ -//! GameCult.Geometry's Rust contract and runtime surface. -//! -//! The v2 contract was mined from the MIT-licensed GameCult `vg-csg` crate at -//! commit `8f070f4`, then rewritten around typed vector values. This crate has -//! no runtime or build dependency on VibeGeometry or `vg-csg`. - -mod assembler; -mod brush; -mod convex; -mod domain; -mod dsl; -mod frontier; -mod mesh; -mod primitives; -#[cfg(test)] -mod realtime_csg_parity; -mod tree; - -#[allow(unused_imports)] -pub(crate) use assembler::{Assembler, BuildOutput, BuildReport, BuildWarning}; -#[allow(unused_imports)] -pub(crate) use brush::{Aabb, Brush, BrushId, BrushOp, MaterialId, PolygonCategory, Primitive}; -#[allow(unused_imports)] -pub(crate) use convex::{ - CategorizedPolygons, ConvexPolygon, ConvexSolid, Plane, PolygonRouteScratch, -}; -#[allow(unused_imports)] -pub(crate) use domain::{ - ClaimLoweringTarget, ContributionManifest, ContributionRow, CsgClaimLowering, DomainChunkBuild, - DomainFrame, DomainKey, DomainKind, DomainNode, DomainNodeSpec, DomainQuery, DomainSummary, - FeatureClaim, FeatureClaimKind, FeatureClaimSpec, FeatureLoweringPolicy, FieldEncoding, - FieldLayer, SelectedCut, SelectedCutManifest, TriangleChunk, TriangleChunkManifest, - build_domain_chunks, lower_feature_claims_to_csg_tree, lower_selected_cut, - lower_selected_cut_chunks, ragnarok_column_fixture, ragnarok_column_spec, select_domain_cut, -}; -#[allow(unused_imports)] -pub(crate) use dsl::LevelDsl; -#[allow(unused_imports)] -pub(crate) use frontier::{DemandFrontier, DemandPair, DirtyDemandFrontier}; -pub(crate) use mesh::TriangleMesh; -#[allow(unused_imports)] -pub(crate) use primitives::{ - DomeCapZSpec, FloretArmSpec, append_cylinder_z, append_dome_cap_z, append_floret_arm, -}; -#[allow(unused_imports)] -pub(crate) use tree::{ - CsgBranchOp, CsgNode, CsgNodeId, CsgOperationType, CsgTree, CsgTreeArena, CsgTreeBranch, - CsgTreeBrush, -}; - -use std::fmt::Write; - -use serde::{Deserialize, Serialize, de::DeserializeOwned}; -use sha2::{Digest, Sha256}; - -pub const GEOMETRY_DOMAIN_SCHEMA: &str = "gamecult.geometry.domain.v2"; -pub const GEOMETRY_BUILD_REQUEST_SCHEMA: &str = "gamecult.geometry.build_request.v2"; -pub const GEOMETRY_SELECTED_CUT_SCHEMA: &str = "gamecult.geometry.selected_cut.v2"; -pub const GEOMETRY_CHUNK_ARTIFACT_SCHEMA: &str = "gamecult.geometry.chunk_artifact.v2"; - -#[derive(Clone, Copy, Debug, Default, PartialEq, Serialize, Deserialize)] -pub struct Float2(pub f32, pub f32); - -#[derive(Clone, Copy, Debug, Default, PartialEq, Serialize, Deserialize)] -pub struct Float3(pub f32, pub f32, pub f32); - -#[derive(Clone, Copy, Debug, PartialEq, Serialize, Deserialize)] -pub struct Quaternion(pub f32, pub f32, pub f32, pub f32); - -impl Default for Quaternion { - fn default() -> Self { - Self(0.0, 0.0, 0.0, 1.0) - } -} - -pub trait GeometryDocument: Serialize + DeserializeOwned + Sized { - const SCHEMA_VERSION: &'static str; - fn record_key(&self) -> String; - fn to_msgpack(&self) -> Result, rmp_serde::encode::Error> { - rmp_serde::to_vec(self) - } - fn from_msgpack(payload: &[u8]) -> Result { - rmp_serde::from_slice(payload) - } -} - -/// Geometry-owned CSG facade. The implementation uses an internal math kernel, -/// but no `bevy_math` type crosses this public boundary. -#[derive(Default)] -pub struct GeometryBrushAssembler(Assembler); - -impl GeometryBrushAssembler { - pub fn new() -> Self { - Self(Assembler::new()) - } - pub fn add_box( - &mut self, - name: impl Into, - center: Float3, - size: Float3, - material: u32, - ) { - self.0 - .solid_box(name, internal_aabb(center, size), MaterialId(material)); - } - pub fn subtract_box(&mut self, name: impl Into, center: Float3, size: Float3) { - self.0.cut_box(name, internal_aabb(center, size)); - } - pub fn build(&self) -> GeometryTriangleMesh { - internal_mesh(&self.0.build().mesh) - } -} - -fn internal_aabb(center: Float3, size: Float3) -> Aabb { - use bevy_math::Vec3; - Aabb::from_center_size( - Vec3::new(center.0, center.1, center.2), - Vec3::new(size.0, size.1, size.2), - ) -} - -fn internal_mesh(mesh: &TriangleMesh) -> GeometryTriangleMesh { - GeometryTriangleMesh( - mesh.positions - .iter() - .map(|v| Float3(v.x, v.y, v.z)) - .collect(), - mesh.normals.iter().map(|v| Float3(v.x, v.y, v.z)).collect(), - mesh.uvs.iter().map(|v| Float2(v.x, v.y)).collect(), - mesh.indices.clone(), - mesh.triangle_materials.iter().map(|v| v.0).collect(), - ) -} - -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -pub struct GeometryDomainDocument( - pub String, - pub String, - pub String, - pub GeometryDomainNode, - pub String, -); - -impl GeometryDocument for GeometryDomainDocument { - const SCHEMA_VERSION: &'static str = GEOMETRY_DOMAIN_SCHEMA; - fn record_key(&self) -> String { - format!( - "geometry:domain:{}", - stable_hash(&[&self.1, &self.2, &self.3.stable_fingerprint()]) - ) - } -} - -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -pub struct GeometryDomainNode( - pub String, - pub String, - pub Float3, - pub Quaternion, - pub u64, - pub Vec, - pub Vec, -); - -impl GeometryDomainNode { - pub fn stable_fingerprint(&self) -> String { - let mut parts = vec![ - self.0.clone(), - self.1.clone(), - stable_float3(self.2), - stable_quaternion(self.3), - self.4.to_string(), - ]; - parts.extend(self.5.iter().map(GeometryFeatureClaim::stable_fingerprint)); - parts.extend(self.6.iter().map(GeometryDomainNode::stable_fingerprint)); - stable_join(&parts) - } -} - -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -pub struct GeometryFeatureClaim( - pub String, - pub Float3, - pub Quaternion, - pub Float3, - pub Float3, - pub String, - pub u32, - pub String, -); - -impl GeometryFeatureClaim { - pub fn stable_fingerprint(&self) -> String { - stable_join(&[ - self.0.clone(), - stable_float3(self.1), - stable_quaternion(self.2), - stable_float3(self.3), - stable_float3(self.4), - self.5.clone(), - self.6.to_string(), - self.7.clone(), - ]) - } -} - -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -pub struct GeometryBuildRequest( - pub String, - pub String, - pub String, - pub Float3, - pub Float3, - pub Float3, - pub f32, - pub f32, - pub f32, - pub i32, - pub i32, - pub Vec, - pub Vec, - pub Vec, - pub String, -); - -impl GeometryDocument for GeometryBuildRequest { - const SCHEMA_VERSION: &'static str = GEOMETRY_BUILD_REQUEST_SCHEMA; - fn record_key(&self) -> String { - format!( - "geometry:request:{}", - stable_hash(&[ - &self.1, - &self.2, - &stable_float3(self.3), - &stable_float3(self.4), - &stable_float3(self.5), - &stable_f32(self.6), - &stable_f32(self.7), - &stable_f32(self.8), - &self.9.to_string(), - &self.10.to_string(), - &stable_join(&self.11), - &stable_join(&self.12), - &stable_join(&self.13), - ]) - ) - } -} - -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -pub struct GeometrySelectedCutManifest( - pub String, - pub String, - pub Vec, - pub Vec, - pub Vec, - pub Vec, -); - -impl GeometryDocument for GeometrySelectedCutManifest { - const SCHEMA_VERSION: &'static str = GEOMETRY_SELECTED_CUT_SCHEMA; - fn record_key(&self) -> String { - format!( - "geometry:cut:{}", - stable_hash(&[ - &self.1, - &self.0, - &stable_join(&self.2), - &stable_join(&self.3), - &stable_join(&self.4), - ]) - ) - } -} - -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -pub struct GeometryContributionRow( - pub String, - pub String, - pub f32, - pub f32, - pub f32, - pub i32, - pub i32, - pub i32, - pub bool, - pub bool, - pub bool, - pub bool, - pub bool, -); - -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -pub struct GeometryChunkArtifact( - pub String, - pub String, - pub String, - pub Float3, - pub Float3, - pub Vec, - pub Vec, - pub GeometryTriangleMesh, - pub Option, - pub i32, - pub i32, - pub i32, - pub u64, - pub bool, -); - -impl GeometryDocument for GeometryChunkArtifact { - const SCHEMA_VERSION: &'static str = GEOMETRY_CHUNK_ARTIFACT_SCHEMA; - fn record_key(&self) -> String { - format!( - "geometry:chunk:{}", - stable_hash(&[ - &self.1, - &self.0, - &self.2, - &stable_join(&self.5), - &stable_join(&self.6), - &self.7.stable_fingerprint(), - self.8 - .as_ref() - .map(GeometryTriangleMesh::stable_fingerprint) - .as_deref() - .unwrap_or(""), - ]) - ) - } -} - -#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] -pub struct GeometryTriangleMesh( - pub Vec, - pub Vec, - pub Vec, - pub Vec, - pub Vec, -); - -impl GeometryTriangleMesh { - pub fn triangle_count(&self) -> usize { - self.3.len() / 3 - } - pub fn stable_fingerprint(&self) -> String { - stable_hash(&[ - &stable_float3_array(&self.0), - &stable_float3_array(&self.1), - &stable_float2_array(&self.2), - &stable_u32_array(&self.3), - &stable_u32_array(&self.4), - ]) - } -} - -fn stable_hash(parts: &[&str]) -> String { - let mut hasher = Sha256::new(); - hasher.update(parts.join("\u{1f}").as_bytes()); - let mut output = String::with_capacity(64); - for byte in hasher.finalize() { - write!(&mut output, "{byte:02x}").expect("string write"); - } - output -} - -fn stable_join(parts: &[String]) -> String { - parts.join("\u{1e}") -} -fn stable_f32(value: f32) -> String { - format!("{:08x}", value.to_bits()) -} -fn stable_float2(value: Float2) -> String { - format!("{},{}", stable_f32(value.0), stable_f32(value.1)) -} -fn stable_float3(value: Float3) -> String { - format!( - "{},{},{}", - stable_f32(value.0), - stable_f32(value.1), - stable_f32(value.2) - ) -} -fn stable_quaternion(value: Quaternion) -> String { - format!( - "{},{},{},{}", - stable_f32(value.0), - stable_f32(value.1), - stable_f32(value.2), - stable_f32(value.3) - ) -} -fn stable_float2_array(values: &[Float2]) -> String { - values - .iter() - .map(|v| stable_float2(*v)) - .collect::>() - .join(",") -} -fn stable_float3_array(values: &[Float3]) -> String { - values - .iter() - .map(|v| stable_float3(*v)) - .collect::>() - .join(",") -} -fn stable_u32_array(values: &[u32]) -> String { - values - .iter() - .map(u32::to_string) - .collect::>() - .join(",") -} - -#[cfg(test)] -mod tests { - use super::*; - - fn domain() -> GeometryDomainDocument { - GeometryDomainDocument( - "fixture-domain".into(), - "fixture/root".into(), - "gamecult-geometry-rs".into(), - GeometryDomainNode( - "root".into(), - "Root".into(), - Float3(1.0, -2.5, 0.0), - Quaternion::default(), - 42, - vec![], - vec![], - ), - "2026-07-22T00:00:00Z".into(), - ) - } - - fn chunk() -> GeometryChunkArtifact { - GeometryChunkArtifact( - "fixture/chunk".into(), - "geometry:cut:fixture".into(), - "cut-fixture".into(), - Float3(-1.0, -2.0, -3.0), - Float3(1.0, 2.0, 3.0), - vec!["fixture/root".into()], - vec!["fixture/root/claim".into()], - GeometryTriangleMesh( - vec![ - Float3(0.0, 0.0, 0.0), - Float3(1.0, 0.0, 0.0), - Float3(0.0, 1.0, 0.0), - ], - vec![Float3(0.0, 0.0, 1.0); 3], - vec![Float2(0.0, 0.0), Float2(1.0, 0.0), Float2(0.0, 1.0)], - vec![0, 1, 2], - vec![7], - ), - None, - 1, - 0, - 0, - 0x0123_4567_89ab_cdef, - true, - ) - } - - fn request() -> GeometryBuildRequest { - GeometryBuildRequest( - "request-fixture".into(), - domain().record_key(), - "workers".into(), - Float3(1.0, 2.0, 3.0), - Float3(-4.0, -5.0, -6.0), - Float3(4.0, 5.0, 6.0), - 1080.0, - 1.0, - 0.25, - 100, - 50, - vec!["Root".into()], - vec![], - vec![], - "2026-07-22T00:00:01Z".into(), - ) - } - - fn cut() -> GeometrySelectedCutManifest { - GeometrySelectedCutManifest( - "cut-fixture".into(), - "geometry:request:fixture".into(), - vec!["fixture/root".into()], - vec![], - vec![], - vec![], - ) - } - - #[test] - fn schema_versions_are_v2() { - assert_eq!( - GeometryDomainDocument::SCHEMA_VERSION, - "gamecult.geometry.domain.v2" - ); - assert_eq!( - GeometryBuildRequest::SCHEMA_VERSION, - "gamecult.geometry.build_request.v2" - ); - assert_eq!( - GeometrySelectedCutManifest::SCHEMA_VERSION, - "gamecult.geometry.selected_cut.v2" - ); - assert_eq!( - GeometryChunkArtifact::SCHEMA_VERSION, - "gamecult.geometry.chunk_artifact.v2" - ); - } - - #[test] - fn domain_record_key_matches_csharp_v2_fixture() { - let document = GeometryDomainDocument( - "ragnarok-column".into(), - "ragnarok-column".into(), - "vg-csg".into(), - GeometryDomainNode( - "ragnarok-column".into(), - "Root".into(), - Float3::default(), - Quaternion::default(), - 0x5eed, - vec![], - vec![GeometryDomainNode( - "stellarator-column-00".into(), - "Column".into(), - Float3::default(), - Quaternion::default(), - 0xc011_0000, - vec![GeometryFeatureClaim( - "column-support-shell".into(), - Float3::default(), - Quaternion::default(), - Float3(0.0, 0.0, 45.0), - Float3(18.0, 18.0, 96.0), - "SupportShell".into(), - 10, - "RenderAndCollider".into(), - )], - vec![], - )], - ), - "2026-05-29T00:00:00.0000000Z".into(), - ); - - assert_eq!( - document.record_key(), - "geometry:domain:175899ea97548da0599e12bcaccd07fa1d6009ed450fadb3de229d47bab04431" - ); - } - - #[test] - fn mined_csg_performance_fixture_keeps_large_distant_cutters_bounded() { - use bevy_math::Vec3; - use std::time::{Duration, Instant}; - - let mut assembler = Assembler::new(); - assembler.solid_box( - "source", - Aabb::from_center_size(Vec3::ZERO, Vec3::splat(4.0)), - MaterialId(1), - ); - for index in 0..512 { - assembler.cut_box( - format!("distant-{index}"), - Aabb::from_center_size(Vec3::new(100.0 + index as f32 * 2.0, 0.0, 0.0), Vec3::ONE), - ); - } - let started = Instant::now(); - let output = assembler.build(); - assert_eq!(output.mesh.triangle_count(), 12); - assert_eq!(output.report.candidate_pairs, 0); - assert_eq!(output.report.rejected_pairs, 512); - assert!(started.elapsed() < Duration::from_secs(2)); - } - - #[test] - fn exact_v2_messagepack_and_record_key_witnesses() { - let domain = domain(); - let chunk = chunk(); - let request = request(); - let cut = cut(); - assert_eq!( - domain.record_key(), - "geometry:domain:505c94e7393580a4e5b048b169aa4da955da606ffd28c054b766a4b94e50502e" - ); - assert_eq!( - request.record_key(), - "geometry:request:36e25b80c04a6c54c8b3137abb4a8c961468a9b1c0e7dd4d5632cf5a6f847966" - ); - assert_eq!( - cut.record_key(), - "geometry:cut:5614eae3fe2a17dca651d1e32f401c50e1c3b9cf941f5789655990325da64fb3" - ); - assert_eq!( - chunk.record_key(), - "geometry:chunk:ca2874d3638800179c906087ec817ea715c43b70b99ead04edccce2a1b3d6ebb" - ); - assert_eq!( - domain.to_msgpack().unwrap(), - [ - 149, 174, 102, 105, 120, 116, 117, 114, 101, 45, 100, 111, 109, 97, 105, 110, 172, - 102, 105, 120, 116, 117, 114, 101, 47, 114, 111, 111, 116, 180, 103, 97, 109, 101, - 99, 117, 108, 116, 45, 103, 101, 111, 109, 101, 116, 114, 121, 45, 114, 115, 151, - 164, 114, 111, 111, 116, 164, 82, 111, 111, 116, 147, 202, 63, 128, 0, 0, 202, 192, - 32, 0, 0, 202, 0, 0, 0, 0, 148, 202, 0, 0, 0, 0, 202, 0, 0, 0, 0, 202, 0, 0, 0, 0, - 202, 63, 128, 0, 0, 42, 144, 144, 180, 50, 48, 50, 54, 45, 48, 55, 45, 50, 50, 84, - 48, 48, 58, 48, 48, 58, 48, 48, 90 - ] - ); - assert_eq!( - request.to_msgpack().unwrap(), - [ - 159, 175, 114, 101, 113, 117, 101, 115, 116, 45, 102, 105, 120, 116, 117, 114, 101, - 217, 80, 103, 101, 111, 109, 101, 116, 114, 121, 58, 100, 111, 109, 97, 105, 110, - 58, 53, 48, 53, 99, 57, 52, 101, 55, 51, 57, 51, 53, 56, 48, 97, 52, 101, 53, 98, - 48, 52, 56, 98, 49, 54, 57, 97, 97, 52, 100, 97, 57, 53, 53, 100, 97, 54, 48, 54, - 102, 102, 100, 50, 56, 99, 48, 53, 52, 98, 55, 54, 54, 97, 52, 98, 57, 52, 101, 53, - 48, 53, 48, 50, 101, 167, 119, 111, 114, 107, 101, 114, 115, 147, 202, 63, 128, 0, - 0, 202, 64, 0, 0, 0, 202, 64, 64, 0, 0, 147, 202, 192, 128, 0, 0, 202, 192, 160, 0, - 0, 202, 192, 192, 0, 0, 147, 202, 64, 128, 0, 0, 202, 64, 160, 0, 0, 202, 64, 192, - 0, 0, 202, 68, 135, 0, 0, 202, 63, 128, 0, 0, 202, 62, 128, 0, 0, 100, 50, 145, - 164, 82, 111, 111, 116, 144, 144, 180, 50, 48, 50, 54, 45, 48, 55, 45, 50, 50, 84, - 48, 48, 58, 48, 48, 58, 48, 49, 90 - ] - ); - assert_eq!( - cut.to_msgpack().unwrap(), - [ - 150, 171, 99, 117, 116, 45, 102, 105, 120, 116, 117, 114, 101, 184, 103, 101, 111, - 109, 101, 116, 114, 121, 58, 114, 101, 113, 117, 101, 115, 116, 58, 102, 105, 120, - 116, 117, 114, 101, 145, 172, 102, 105, 120, 116, 117, 114, 101, 47, 114, 111, 111, - 116, 144, 144, 144 - ] - ); - assert_eq!( - chunk.to_msgpack().unwrap(), - [ - 158, 173, 102, 105, 120, 116, 117, 114, 101, 47, 99, 104, 117, 110, 107, 180, 103, - 101, 111, 109, 101, 116, 114, 121, 58, 99, 117, 116, 58, 102, 105, 120, 116, 117, - 114, 101, 171, 99, 117, 116, 45, 102, 105, 120, 116, 117, 114, 101, 147, 202, 191, - 128, 0, 0, 202, 192, 0, 0, 0, 202, 192, 64, 0, 0, 147, 202, 63, 128, 0, 0, 202, 64, - 0, 0, 0, 202, 64, 64, 0, 0, 145, 172, 102, 105, 120, 116, 117, 114, 101, 47, 114, - 111, 111, 116, 145, 178, 102, 105, 120, 116, 117, 114, 101, 47, 114, 111, 111, 116, - 47, 99, 108, 97, 105, 109, 149, 147, 147, 202, 0, 0, 0, 0, 202, 0, 0, 0, 0, 202, 0, - 0, 0, 0, 147, 202, 63, 128, 0, 0, 202, 0, 0, 0, 0, 202, 0, 0, 0, 0, 147, 202, 0, 0, - 0, 0, 202, 63, 128, 0, 0, 202, 0, 0, 0, 0, 147, 147, 202, 0, 0, 0, 0, 202, 0, 0, 0, - 0, 202, 63, 128, 0, 0, 147, 202, 0, 0, 0, 0, 202, 0, 0, 0, 0, 202, 63, 128, 0, 0, - 147, 202, 0, 0, 0, 0, 202, 0, 0, 0, 0, 202, 63, 128, 0, 0, 147, 146, 202, 0, 0, 0, - 0, 202, 0, 0, 0, 0, 146, 202, 63, 128, 0, 0, 202, 0, 0, 0, 0, 146, 202, 0, 0, 0, 0, - 202, 63, 128, 0, 0, 147, 0, 1, 2, 145, 7, 192, 1, 0, 0, 207, 1, 35, 69, 103, 137, - 171, 205, 239, 195 - ] - ); - assert_eq!( - GeometryDomainDocument::from_msgpack(&domain.to_msgpack().unwrap()).unwrap(), - domain - ); - assert_eq!( - GeometryChunkArtifact::from_msgpack(&chunk.to_msgpack().unwrap()).unwrap(), - chunk - ); - assert_eq!( - GeometryBuildRequest::from_msgpack(&request.to_msgpack().unwrap()).unwrap(), - request - ); - assert_eq!( - GeometrySelectedCutManifest::from_msgpack(&cut.to_msgpack().unwrap()).unwrap(), - cut - ); - } - - #[test] - fn scalar_bit_identity_is_stable_through_messagepack() { - let mut signed_zero = request(); - signed_zero.6 = -0.0; - signed_zero.7 = f32::from_bits(0x7fc0_1234); - signed_zero.8 = f32::NEG_INFINITY; - assert_eq!( - signed_zero.record_key(), - "geometry:request:978dd0913fc3e12c04766e065320d485d172d60d61be920a9a32637d1c439c81" - ); - let decoded = - GeometryBuildRequest::from_msgpack(&signed_zero.to_msgpack().unwrap()).unwrap(); - assert_eq!(decoded.6.to_bits(), 0x8000_0000); - assert_eq!(decoded.7.to_bits(), 0x7fc0_1234); - assert_eq!(decoded.8.to_bits(), f32::NEG_INFINITY.to_bits()); - } -} diff --git a/packages/gamecult-geometry-rs/src/mesh.rs b/packages/gamecult-geometry-rs/src/mesh.rs deleted file mode 100644 index 9e23b7ab..00000000 --- a/packages/gamecult-geometry-rs/src/mesh.rs +++ /dev/null @@ -1,103 +0,0 @@ -#![allow(dead_code)] -use bevy_math::{Vec2, Vec3}; - -use crate::{Aabb, MaterialId}; - -#[derive(Clone, Debug, Default, PartialEq)] -pub struct TriangleMesh { - pub positions: Vec, - pub normals: Vec, - pub uvs: Vec, - pub indices: Vec, - pub triangle_materials: Vec, -} - -impl TriangleMesh { - pub fn new() -> Self { - Self::default() - } - - pub fn triangle_count(&self) -> usize { - self.indices.len() / 3 - } - - pub fn vertex_count(&self) -> usize { - self.positions.len() - } - - pub fn append_box(&mut self, bounds: Aabb, material: MaterialId) { - let min = bounds.min; - let max = bounds.max; - let corners = [ - Vec3::new(min.x, min.y, min.z), - Vec3::new(max.x, min.y, min.z), - Vec3::new(max.x, max.y, min.z), - Vec3::new(min.x, max.y, min.z), - Vec3::new(min.x, min.y, max.z), - Vec3::new(max.x, min.y, max.z), - Vec3::new(max.x, max.y, max.z), - Vec3::new(min.x, max.y, max.z), - ]; - - self.append_quad( - [corners[0], corners[3], corners[2], corners[1]], - Vec3::NEG_Z, - material, - ); - self.append_quad( - [corners[4], corners[5], corners[6], corners[7]], - Vec3::Z, - material, - ); - self.append_quad( - [corners[0], corners[1], corners[5], corners[4]], - Vec3::NEG_Y, - material, - ); - self.append_quad( - [corners[3], corners[7], corners[6], corners[2]], - Vec3::Y, - material, - ); - self.append_quad( - [corners[0], corners[4], corners[7], corners[3]], - Vec3::NEG_X, - material, - ); - self.append_quad( - [corners[1], corners[2], corners[6], corners[5]], - Vec3::X, - material, - ); - } - - pub fn append_quad(&mut self, points: [Vec3; 4], normal: Vec3, material: MaterialId) { - let base = self.positions.len() as u32; - self.positions.extend(points); - self.normals.extend([normal; 4]); - self.uvs.extend([ - Vec2::new(0.0, 0.0), - Vec2::new(1.0, 0.0), - Vec2::new(1.0, 1.0), - Vec2::new(0.0, 1.0), - ]); - self.indices - .extend([base, base + 1, base + 2, base, base + 2, base + 3]); - self.triangle_materials.extend([material; 2]); - } - - pub fn append_triangle( - &mut self, - points: [Vec3; 3], - normal: Vec3, - uvs: [Vec2; 3], - material: MaterialId, - ) { - let base = self.positions.len() as u32; - self.positions.extend(points); - self.normals.extend([normal; 3]); - self.uvs.extend(uvs); - self.indices.extend([base, base + 1, base + 2]); - self.triangle_materials.push(material); - } -} diff --git a/packages/gamecult-geometry-rs/src/primitives.rs b/packages/gamecult-geometry-rs/src/primitives.rs deleted file mode 100644 index bc100e59..00000000 --- a/packages/gamecult-geometry-rs/src/primitives.rs +++ /dev/null @@ -1,173 +0,0 @@ -#![allow(dead_code)] -use bevy_math::{Vec2, Vec3}; - -use crate::{MaterialId, TriangleMesh}; - -#[derive(Clone, Copy, Debug)] -pub struct DomeCapZSpec { - pub center: Vec3, - pub radius: f32, - pub height: f32, - pub rings: usize, - pub segments: usize, - pub material: MaterialId, -} - -#[derive(Clone, Copy, Debug)] -pub struct FloretArmSpec { - pub anchor: Vec3, - pub direction: Vec3, - pub length: f32, - pub root_width: f32, - pub tip_width: f32, - pub thickness: f32, - pub tip_lift: f32, - pub material: MaterialId, -} - -pub fn append_cylinder_z( - mesh: &mut TriangleMesh, - center: Vec3, - radius: f32, - depth: f32, - segments: usize, - material: MaterialId, -) { - let segments = segments.max(3); - let half = depth * 0.5; - let bottom_center = center - Vec3::Z * half; - let top_center = center + Vec3::Z * half; - - for i in 0..segments { - let a0 = ring_angle(i, segments); - let a1 = ring_angle(i + 1, segments); - let p0 = center + Vec3::new(a0.cos() * radius, a0.sin() * radius, -half); - let p1 = center + Vec3::new(a1.cos() * radius, a1.sin() * radius, -half); - let p2 = center + Vec3::new(a1.cos() * radius, a1.sin() * radius, half); - let p3 = center + Vec3::new(a0.cos() * radius, a0.sin() * radius, half); - let normal = Vec3::new(((a0 + a1) * 0.5).cos(), ((a0 + a1) * 0.5).sin(), 0.0); - mesh.append_quad([p0, p1, p2, p3], normal, material); - mesh.append_triangle( - [bottom_center, p1, p0], - Vec3::NEG_Z, - [Vec2::ZERO, Vec2::X, Vec2::Y], - material, - ); - mesh.append_triangle( - [top_center, p3, p2], - Vec3::Z, - [Vec2::ZERO, Vec2::X, Vec2::Y], - material, - ); - } -} - -pub fn append_dome_cap_z(mesh: &mut TriangleMesh, spec: DomeCapZSpec) { - let rings = spec.rings.max(2); - let segments = spec.segments.max(6); - let top = spec.center + Vec3::Z * spec.height; - - for ring in 0..rings { - let t0 = ring as f32 / rings as f32; - let t1 = (ring + 1) as f32 / rings as f32; - let r0 = spec.radius * (1.0 - t0 * t0).sqrt(); - let r1 = spec.radius * (1.0 - t1 * t1).sqrt(); - let z0 = spec.center.z + spec.height * t0; - let z1 = spec.center.z + spec.height * t1; - - for seg in 0..segments { - let a0 = ring_angle(seg, segments); - let a1 = ring_angle(seg + 1, segments); - let p0 = Vec3::new( - spec.center.x + a0.cos() * r0, - spec.center.y + a0.sin() * r0, - z0, - ); - let p1 = Vec3::new( - spec.center.x + a1.cos() * r0, - spec.center.y + a1.sin() * r0, - z0, - ); - - if ring + 1 == rings { - let normal = (p0 - spec.center) - .cross(top - spec.center) - .normalize_or_zero(); - mesh.append_triangle( - [p0, p1, top], - normal, - [Vec2::ZERO, Vec2::X, Vec2::Y], - spec.material, - ); - } else { - let p2 = Vec3::new( - spec.center.x + a1.cos() * r1, - spec.center.y + a1.sin() * r1, - z1, - ); - let p3 = Vec3::new( - spec.center.x + a0.cos() * r1, - spec.center.y + a0.sin() * r1, - z1, - ); - let normal = ((p0 + p1 + p2 + p3) * 0.25 - spec.center).normalize_or_zero(); - mesh.append_quad([p0, p1, p2, p3], normal, spec.material); - } - } - } -} - -pub fn append_floret_arm(mesh: &mut TriangleMesh, spec: FloretArmSpec) { - let forward = spec.direction.normalize_or_zero(); - let forward = if forward.length_squared() == 0.0 { - Vec3::X - } else { - forward - }; - let mut side = forward.cross(Vec3::Z).normalize_or_zero(); - if side.length_squared() == 0.0 { - side = Vec3::Y; - } - - let root_center = spec.anchor; - let tip_center = spec.anchor + forward * spec.length + Vec3::Z * spec.tip_lift; - let z = Vec3::Z * (spec.thickness * 0.5); - let r = side * (spec.root_width * 0.5); - let t = side * (spec.tip_width * 0.5); - - let bottom = [ - root_center - r - z, - root_center + r - z, - tip_center + t - z, - tip_center - t - z, - ]; - let top = [ - root_center - r + z, - root_center + r + z, - tip_center + t + z, - tip_center - t + z, - ]; - - mesh.append_quad( - [bottom[0], bottom[3], bottom[2], bottom[1]], - Vec3::NEG_Z, - spec.material, - ); - mesh.append_quad([top[0], top[1], top[2], top[3]], Vec3::Z, spec.material); - mesh.append_quad( - [bottom[0], bottom[1], top[1], top[0]], - -forward, - spec.material, - ); - mesh.append_quad( - [bottom[3], top[3], top[2], bottom[2]], - forward, - spec.material, - ); - mesh.append_quad([bottom[0], top[0], top[3], bottom[3]], -side, spec.material); - mesh.append_quad([bottom[1], bottom[2], top[2], top[1]], side, spec.material); -} - -fn ring_angle(index: usize, segments: usize) -> f32 { - index as f32 * std::f32::consts::TAU / segments as f32 -} diff --git a/packages/gamecult-geometry-rs/src/realtime_csg_parity.rs b/packages/gamecult-geometry-rs/src/realtime_csg_parity.rs deleted file mode 100644 index fbb1ac70..00000000 --- a/packages/gamecult-geometry-rs/src/realtime_csg_parity.rs +++ /dev/null @@ -1,166 +0,0 @@ -use crate::{ - Aabb, Assembler, ConvexSolid, CsgBranchOp, CsgOperationType, CsgTreeArena, MaterialId, - PolygonCategory, Primitive, -}; -use bevy_math::{Quat, Vec3}; - -#[test] -fn parity_polygon_category_vocabulary_matches_public_demo() { - let categories = [ - PolygonCategory::Inside, - PolygonCategory::Outside, - PolygonCategory::Aligned, - PolygonCategory::ReverseAligned, - ]; - - assert_eq!(categories.len(), 4); -} - -#[test] -fn parity_operation_type_ordinals_match_public_api() { - assert_eq!(CsgOperationType::Additive as u8, 0); - assert_eq!(CsgOperationType::Subtractive as u8, 1); - assert_eq!(CsgOperationType::Intersecting as u8, 2); -} - -#[test] -fn parity_demo_branch_vocabulary_matches_public_demo() { - let branches = [ - CsgBranchOp::Addition, - CsgBranchOp::Subtraction, - CsgBranchOp::Common, - ]; - - assert_eq!(branches.len(), 3); -} - -#[test] -fn parity_tree_api_exposes_brush_branch_tree_handles() { - let mut arena = CsgTreeArena::new(); - let brush = arena.generate_brush( - "brush", - CsgOperationType::Additive, - Primitive::Box { - bounds: Aabb::from_center_size(Vec3::ZERO, Vec3::ONE), - }, - MaterialId(1), - ); - let branch = arena.generate_branch("branch", CsgBranchOp::Addition, [brush.node]); - let tree = arena.generate_tree(branch.node); - - assert_eq!(arena.brush_count(), 1); - assert_eq!(arena.branch_count(), 1); - assert_eq!(arena.child_nodes(branch).expect("branch"), &[brush.node]); - assert_eq!(tree.root, branch.node); -} - -#[test] -fn parity_box_brush_observable_counts_match_realtime_csg_demo() { - let solid = ConvexSolid::box_from_center_size( - Vec3::ZERO, - Vec3::splat(2.0), - Quat::IDENTITY, - MaterialId(3), - ); - - assert_eq!(solid.clip_planes.len(), 6); - assert_eq!(solid.polygons.len(), 6); - assert_eq!( - solid - .polygons - .iter() - .map(|polygon| polygon.vertices.len()) - .sum::(), - 24 - ); -} - -#[test] -fn parity_center_box_subtraction_has_six_surviving_regions() { - let mut asm = Assembler::new(); - asm.solid_box( - "source", - Aabb::from_center_size(Vec3::ZERO, Vec3::splat(4.0)), - MaterialId(1), - ); - asm.cut_box("void", Aabb::from_center_size(Vec3::ZERO, Vec3::splat(2.0))); - - let output = asm.build(); - assert_eq!(output.report.emitted_convex_fragments, 6); - assert_eq!(output.report.warnings.len(), 0); -} - -#[test] -fn parity_identical_box_polygons_classify_as_aligned() { - let cutter = ConvexSolid::box_from_center_size( - Vec3::ZERO, - Vec3::splat(2.0), - Quat::IDENTITY, - MaterialId(0), - ); - let mut source = ConvexSolid::box_from_center_size( - Vec3::ZERO, - Vec3::splat(2.0), - Quat::IDENTITY, - MaterialId(1), - ); - - source.categorize_whole_polygons_against(&cutter); - - assert_eq!(source.polygons.len(), 6); - assert!( - source - .polygons - .iter() - .all(|polygon| polygon.category == PolygonCategory::Aligned) - ); -} - -#[test] -fn parity_outer_box_polygons_classify_as_outside_inner_box() { - let cutter = ConvexSolid::box_from_center_size( - Vec3::ZERO, - Vec3::splat(2.0), - Quat::IDENTITY, - MaterialId(0), - ); - let mut source = ConvexSolid::box_from_center_size( - Vec3::ZERO, - Vec3::splat(4.0), - Quat::IDENTITY, - MaterialId(1), - ); - - source.categorize_whole_polygons_against(&cutter); - - assert_eq!(source.polygons.len(), 6); - assert!( - source - .polygons - .iter() - .all(|polygon| polygon.category == PolygonCategory::Outside) - ); -} - -#[test] -fn parity_crossing_polygons_split_before_final_category() { - let source = ConvexSolid::box_from_center_size( - Vec3::ZERO, - Vec3::splat(4.0), - Quat::IDENTITY, - MaterialId(1), - ); - let cutter = ConvexSolid::box_from_center_size( - Vec3::new(0.0, 0.0, 1.0), - Vec3::new(2.0, 2.0, 2.0), - Quat::IDENTITY, - MaterialId(0), - ); - - let categorized = source.categorize_polygons_against(&cutter); - - assert_eq!(categorized.aligned.len(), 1); - assert!(categorized.outside.len() > source.polygons.len()); - assert_eq!(categorized.inside.len(), 0); - assert_eq!(categorized.reverse_aligned.len(), 0); -} diff --git a/packages/gamecult-geometry-rs/src/tree.rs b/packages/gamecult-geometry-rs/src/tree.rs deleted file mode 100644 index 14bdc226..00000000 --- a/packages/gamecult-geometry-rs/src/tree.rs +++ /dev/null @@ -1,402 +0,0 @@ -#![allow(dead_code)] -use crate::{Aabb, Assembler, Brush, BrushId, BrushOp, MaterialId, Primitive}; - -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] -pub struct CsgNodeId(pub u32); - -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] -pub struct CsgTreeBrush { - pub node: CsgNodeId, -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] -pub struct CsgTreeBranch { - pub node: CsgNodeId, -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] -pub struct CsgTree { - pub root: CsgNodeId, -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] -pub enum CsgOperationType { - Additive = 0, - Subtractive = 1, - Intersecting = 2, -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] -pub enum CsgBranchOp { - Addition, - Subtraction, - Common, -} - -#[derive(Clone, Debug)] -pub enum CsgNode { - Brush(Brush), - Branch { - name: String, - op: CsgBranchOp, - children: Vec, - }, -} - -#[derive(Clone, Debug, Default)] -pub struct CsgTreeArena { - nodes: Vec, -} - -impl CsgTreeArena { - pub fn new() -> Self { - Self::default() - } - - pub fn nodes(&self) -> &[CsgNode] { - &self.nodes - } - - pub fn node(&self, id: CsgNodeId) -> Option<&CsgNode> { - self.nodes.get(id.0 as usize) - } - - pub fn node_mut(&mut self, id: CsgNodeId) -> Option<&mut CsgNode> { - self.nodes.get_mut(id.0 as usize) - } - - pub fn brush_count(&self) -> usize { - self.nodes - .iter() - .filter(|node| matches!(node, CsgNode::Brush(_))) - .count() - } - - pub fn branch_count(&self) -> usize { - self.nodes - .iter() - .filter(|node| matches!(node, CsgNode::Branch { .. })) - .count() - } - - pub fn generate_brush( - &mut self, - name: impl Into, - operation: CsgOperationType, - primitive: Primitive, - material: MaterialId, - ) -> CsgTreeBrush { - let id = CsgNodeId(self.nodes.len() as u32); - let brush = Brush::new( - BrushId(id.0), - name, - operation.into_brush_op(), - primitive, - material, - ); - self.nodes.push(CsgNode::Brush(brush)); - CsgTreeBrush { node: id } - } - - pub fn generate_branch( - &mut self, - name: impl Into, - op: CsgBranchOp, - children: impl IntoIterator, - ) -> CsgTreeBranch { - let id = CsgNodeId(self.nodes.len() as u32); - self.nodes.push(CsgNode::Branch { - name: name.into(), - op, - children: children.into_iter().collect(), - }); - CsgTreeBranch { node: id } - } - - pub fn generate_tree(&self, root: CsgNodeId) -> CsgTree { - CsgTree { root } - } - - pub fn set_brush_operation_type( - &mut self, - brush: CsgTreeBrush, - operation: CsgOperationType, - ) -> bool { - let Some(CsgNode::Brush(node)) = self.node_mut(brush.node) else { - return false; - }; - node.op = operation.into_brush_op(); - node.set_dirty(); - true - } - - pub fn set_branch_operation_type(&mut self, branch: CsgTreeBranch, op: CsgBranchOp) -> bool { - let Some(CsgNode::Branch { op: current_op, .. }) = self.node_mut(branch.node) else { - return false; - }; - *current_op = op; - true - } - - pub fn set_child_nodes( - &mut self, - branch: CsgTreeBranch, - children: impl IntoIterator, - ) -> bool { - let Some(CsgNode::Branch { - children: current_children, - .. - }) = self.node_mut(branch.node) - else { - return false; - }; - *current_children = children.into_iter().collect(); - true - } - - pub fn child_nodes(&self, branch: CsgTreeBranch) -> Option<&[CsgNodeId]> { - let Some(CsgNode::Branch { children, .. }) = self.node(branch.node) else { - return None; - }; - Some(children) - } - - pub fn node_bounds(&self, id: CsgNodeId) -> Option { - match self.node(id)? { - CsgNode::Brush(brush) => Some(brush.bounds()), - CsgNode::Branch { op, children, .. } => self.branch_bounds(*op, children), - } - } - - pub fn compile_tree_to_assembler(&self, tree: CsgTree) -> Assembler { - let mut assembler = Assembler::new(); - self.compile_node(tree.root, &mut assembler, None); - assembler - } - - fn compile_node( - &self, - id: CsgNodeId, - assembler: &mut Assembler, - operation_override: Option, - ) { - let Some(node) = self.node(id) else { - return; - }; - - match node { - CsgNode::Brush(brush) => { - let op = operation_override - .map(CsgOperationType::into_brush_op) - .unwrap_or(brush.op); - assembler.add_brush( - brush.name.clone(), - op, - brush.primitive.clone(), - brush.material, - ); - } - CsgNode::Branch { op, children, .. } => match op { - CsgBranchOp::Addition => { - for child in children { - self.compile_node(*child, assembler, operation_override); - } - } - CsgBranchOp::Subtraction => { - if let Some((first, rest)) = children.split_first() { - self.compile_node(*first, assembler, operation_override); - for child in rest { - self.compile_node( - *child, - assembler, - Some(CsgOperationType::Subtractive), - ); - } - } - } - CsgBranchOp::Common => { - if let Some((first, rest)) = children.split_first() { - self.compile_node(*first, assembler, operation_override); - for child in rest { - self.compile_node( - *child, - assembler, - Some(CsgOperationType::Intersecting), - ); - } - } - } - }, - } - } - - fn branch_bounds(&self, op: CsgBranchOp, children: &[CsgNodeId]) -> Option { - let mut child_bounds = children.iter().filter_map(|child| self.node_bounds(*child)); - match op { - CsgBranchOp::Addition => child_bounds.reduce(Aabb::union), - CsgBranchOp::Subtraction => child_bounds.next(), - CsgBranchOp::Common => { - let first = child_bounds.next()?; - child_bounds.try_fold(first, |bounds, child| bounds.intersection(child)) - } - } - } -} - -impl CsgOperationType { - pub fn into_brush_op(self) -> BrushOp { - match self { - Self::Additive => BrushOp::Add, - Self::Subtractive => BrushOp::Subtract, - Self::Intersecting => BrushOp::Intersect, - } - } -} - -impl From for CsgOperationType { - fn from(value: BrushOp) -> Self { - match value { - BrushOp::Add => Self::Additive, - BrushOp::Subtract => Self::Subtractive, - BrushOp::Intersect => Self::Intersecting, - } - } -} - -#[cfg(test)] -mod tests { - use bevy_math::Vec3; - - use super::*; - use crate::{Aabb, Primitive}; - - #[test] - fn tree_subtraction_compiles_to_ordered_brush_stream() { - let mut arena = CsgTreeArena::new(); - let solid = arena.generate_brush( - "solid", - CsgOperationType::Additive, - Primitive::Box { - bounds: Aabb::from_center_size(Vec3::ZERO, Vec3::splat(4.0)), - }, - MaterialId(1), - ); - let void = arena.generate_brush( - "void", - CsgOperationType::Additive, - Primitive::Box { - bounds: Aabb::from_center_size(Vec3::ZERO, Vec3::splat(2.0)), - }, - MaterialId(0), - ); - let branch = arena.generate_branch( - "solid minus void", - CsgBranchOp::Subtraction, - [solid.node, void.node], - ); - let assembler = arena.compile_tree_to_assembler(arena.generate_tree(branch.node)); - - assert_eq!(assembler.brushes().len(), 2); - assert_eq!(assembler.brushes()[0].op, BrushOp::Add); - assert_eq!(assembler.brushes()[1].op, BrushOp::Subtract); - assert_eq!(assembler.build().report.emitted_convex_fragments, 6); - } - - #[test] - fn tree_edit_api_replaces_children_and_operations() { - let mut arena = CsgTreeArena::new(); - let a = arena.generate_brush( - "a", - CsgOperationType::Additive, - Primitive::Box { - bounds: Aabb::from_center_size(Vec3::ZERO, Vec3::splat(2.0)), - }, - MaterialId(1), - ); - let b = arena.generate_brush( - "b", - CsgOperationType::Additive, - Primitive::Box { - bounds: Aabb::from_center_size(Vec3::X, Vec3::splat(1.0)), - }, - MaterialId(1), - ); - let branch = arena.generate_branch("branch", CsgBranchOp::Addition, [a.node]); - - assert!(arena.set_child_nodes(branch, [a.node, b.node])); - assert_eq!(arena.child_nodes(branch).expect("branch").len(), 2); - assert!(arena.set_branch_operation_type(branch, CsgBranchOp::Subtraction)); - assert!(arena.set_brush_operation_type(b, CsgOperationType::Subtractive)); - assert_eq!(arena.brush_count(), 2); - assert_eq!(arena.branch_count(), 1); - } - - #[test] - fn tree_bounds_follow_boolean_branch_shape() { - let mut arena = CsgTreeArena::new(); - let a = arena.generate_brush( - "a", - CsgOperationType::Additive, - Primitive::Box { - bounds: Aabb::from_center_size(Vec3::ZERO, Vec3::splat(4.0)), - }, - MaterialId(1), - ); - let b = arena.generate_brush( - "b", - CsgOperationType::Additive, - Primitive::Box { - bounds: Aabb::from_center_size(Vec3::X * 2.5, Vec3::splat(2.0)), - }, - MaterialId(1), - ); - let union = arena.generate_branch("union", CsgBranchOp::Addition, [a.node, b.node]); - let common = arena.generate_branch("common", CsgBranchOp::Common, [a.node, b.node]); - let subtract = - arena.generate_branch("subtract", CsgBranchOp::Subtraction, [a.node, b.node]); - - assert_eq!( - arena.node_bounds(union.node).expect("union"), - Aabb::new(Vec3::splat(-2.0), Vec3::new(3.5, 2.0, 2.0)) - ); - assert_eq!( - arena.node_bounds(common.node).expect("common"), - Aabb::new(Vec3::new(1.5, -1.0, -1.0), Vec3::new(2.0, 1.0, 1.0)) - ); - assert_eq!( - arena.node_bounds(subtract.node).expect("subtract"), - Aabb::from_center_size(Vec3::ZERO, Vec3::splat(4.0)) - ); - } - - #[test] - fn tree_common_compiles_to_additive_source_then_intersections() { - let mut arena = CsgTreeArena::new(); - let a = arena.generate_brush( - "a", - CsgOperationType::Additive, - Primitive::Box { - bounds: Aabb::from_center_size(Vec3::ZERO, Vec3::splat(4.0)), - }, - MaterialId(1), - ); - let b = arena.generate_brush( - "b", - CsgOperationType::Additive, - Primitive::Box { - bounds: Aabb::from_center_size(Vec3::X, Vec3::splat(4.0)), - }, - MaterialId(1), - ); - let branch = arena.generate_branch("common", CsgBranchOp::Common, [a.node, b.node]); - - let assembler = arena.compile_tree_to_assembler(arena.generate_tree(branch.node)); - let output = assembler.build(); - - assert_eq!(assembler.brushes()[0].op, BrushOp::Add); - assert_eq!(assembler.brushes()[1].op, BrushOp::Intersect); - assert_eq!(output.report.emitted_convex_fragments, 1); - assert_eq!(output.report.warnings.len(), 0); - assert_eq!(output.mesh.triangle_count(), 12); - } -} diff --git a/packages/gamecult-geometry-ts/package.json b/packages/gamecult-geometry-ts/package.json deleted file mode 100644 index ea7f8a32..00000000 --- a/packages/gamecult-geometry-ts/package.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "name": "@gamecult/geometry", - "version": "0.1.0", - "private": true, - "type": "module", - "scripts": { - "build": "node --no-warnings --experimental-strip-types --check src/planetary-tile.ts", - "test": "node --no-warnings --experimental-strip-types test/planetary-tile.test.mjs" - } -} diff --git a/packages/gamecult-geometry-ts/src/planetary-tile.ts b/packages/gamecult-geometry-ts/src/planetary-tile.ts deleted file mode 100644 index 82849c82..00000000 --- a/packages/gamecult-geometry-ts/src/planetary-tile.ts +++ /dev/null @@ -1,47 +0,0 @@ -export const GAMECULT_GEOMETRY_PLANETARY_TILE_VERSION = 1; - -export interface PlanetaryTileHeader { - fieldVersion: bigint; projectionVersion: number; layerId: number; projectionKind: number; - centerLongitude: number; centerLatitude: number; projectionScale: number; - level: number; x: number; y: number; interiorSize: number; borderSize: number; - footprintMeters: number; maximumUnresolvedHeight: number; -} -export interface PlanetaryTileSample { - direction: readonly [number, number, number]; radius: number; radialDisplacement: number; - tangentGradient: readonly [number, number, number]; surfaceNormal: readonly [number, number, number]; - slope: number; ridge: number; gully: number; finestResolvedWavelength: number; unresolvedHeightBound: number; -} -export interface PlanetarySurfaceTile { header: PlanetaryTileHeader; storageSize: number; samples: readonly (PlanetaryTileSample | null)[]; } - -export function decodePlanetarySurfaceTile(bytes: ArrayBuffer): PlanetarySurfaceTile { - const view = new DataView(bytes); let offset = 0; - const u8 = () => view.getUint8(offset++); - const u32 = () => { const value = view.getUint32(offset, true); offset += 4; return value; }; - const i32 = () => { const value = view.getInt32(offset, true); offset += 4; return value; }; - const f32 = () => { const value = view.getFloat32(offset, true); offset += 4; return value; }; - const f64 = () => { const value = view.getFloat64(offset, true); offset += 8; return value; }; - const u64 = () => { const value = view.getBigUint64(offset, true); offset += 8; return value; }; - const vector3 = (): [number, number, number] => [f32(), f32(), f32()]; - const requireRemaining = (count: number) => { if (offset + count > view.byteLength) throw new Error("Truncated GameCult.Geometry planetary tile"); }; - requireRemaining(8); - if (String.fromCharCode(u8(), u8(), u8(), u8()) !== "CMPT") throw new Error("Not a GameCult.Geometry planetary tile"); - const version = u32(); - if (version !== GAMECULT_GEOMETRY_PLANETARY_TILE_VERSION) throw new Error(`Unsupported GameCult.Geometry planetary tile version ${version}`); - requireRemaining(76); - const header: PlanetaryTileHeader = { - fieldVersion: u64(), projectionVersion: u32(), layerId: u32(), projectionKind: i32(), - centerLongitude: f64(), centerLatitude: f64(), projectionScale: f64(), - level: i32(), x: i32(), y: i32(), interiorSize: i32(), borderSize: i32(), - footprintMeters: f32(), maximumUnresolvedHeight: f32(), - }; - const count = i32(); const storageSize = header.interiorSize + header.borderSize * 2; - if (count !== storageSize * storageSize) throw new Error("GameCult.Geometry planetary tile sample count does not match layout"); - const samples: (PlanetaryTileSample | null)[] = new Array(count); - for (let index = 0; index < count; index++) { - requireRemaining(1); if (u8() === 0) { samples[index] = null; continue; } - requireRemaining(64); - samples[index] = { direction: vector3(), radius: f32(), radialDisplacement: f32(), tangentGradient: vector3(), surfaceNormal: vector3(), slope: f32(), ridge: f32(), gully: f32(), finestResolvedWavelength: f32(), unresolvedHeightBound: f32() }; - } - if (offset !== view.byteLength) throw new Error("GameCult.Geometry planetary tile has trailing bytes"); - return { header, storageSize, samples }; -} diff --git a/packages/gamecult-geometry-ts/test/planetary-tile.cross-runtime.test.mjs b/packages/gamecult-geometry-ts/test/planetary-tile.cross-runtime.test.mjs deleted file mode 100644 index 04243ce8..00000000 --- a/packages/gamecult-geometry-ts/test/planetary-tile.cross-runtime.test.mjs +++ /dev/null @@ -1,16 +0,0 @@ -import assert from "node:assert/strict"; -import { createHash } from "node:crypto"; -import { readFile } from "node:fs/promises"; -import { decodePlanetarySurfaceTile } from "../src/planetary-tile.ts"; -if (process.argv.length !== 3) throw new Error("Expected a generated CMPT fixture path"); -const bytes = await readFile(process.argv[2]); -assert.equal(bytes.length, 1221); -assert.equal(createHash("sha256").update(bytes).digest("hex"), "8b2bc46e4123b8ac8936b43f8d08a34e3f2e74f38d0bb3f101cf3728cd201962"); -const tile = decodePlanetarySurfaceTile(bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength)); -assert.equal(tile.header.projectionVersion, 3); assert.equal(tile.header.layerId, 29); -assert.equal(tile.header.projectionKind, 2); assert.equal(tile.header.interiorSize, 5); -assert.equal(tile.header.borderSize, 1); assert.equal(tile.storageSize, 7); assert.equal(tile.samples.length, 49); -assert.ok(tile.samples.some(sample => sample !== null)); -for (const sample of tile.samples) if (sample !== null) { - assert.ok(Number.isFinite(sample.radialDisplacement)); assert.ok(Number.isFinite(sample.unresolvedHeightBound)); -} diff --git a/packages/gamecult-geometry-ts/test/planetary-tile.test.mjs b/packages/gamecult-geometry-ts/test/planetary-tile.test.mjs deleted file mode 100644 index c5c27027..00000000 --- a/packages/gamecult-geometry-ts/test/planetary-tile.test.mjs +++ /dev/null @@ -1,24 +0,0 @@ -import assert from "node:assert/strict"; -import { decodePlanetarySurfaceTile } from "../src/planetary-tile.ts"; -const bytes = new ArrayBuffer(152); const view = new DataView(bytes); let offset = 0; -const u8 = value => view.setUint8(offset++, value); -const u32 = value => { view.setUint32(offset, value, true); offset += 4; }; -const i32 = value => { view.setInt32(offset, value, true); offset += 4; }; -const u64 = value => { view.setBigUint64(offset, value, true); offset += 8; }; -const f32 = value => { view.setFloat32(offset, value, true); offset += 4; }; -const f64 = value => { view.setFloat64(offset, value, true); offset += 8; }; -for (const character of "CMPT") u8(character.charCodeAt(0)); -u32(1); u64(0x1234n); u32(2); u32(17); i32(0); f64(0); f64(0); f64(1); -i32(0); i32(0); i32(0); i32(2); i32(0); f32(10); f32(0.5); i32(4); u8(1); -for (const value of [1, 0, 0]) f32(value); f32(8.4); f32(0.25); -for (const value of [0, 0.1, 0]) f32(value); -for (const value of [0.99, -0.1, 0]) f32(value); -for (const value of [0.1, 0.8, 0.2, 2.5, 0.05]) f32(value); -u8(0); u8(0); u8(0); assert.equal(offset, bytes.byteLength); -const tile = decodePlanetarySurfaceTile(bytes); -assert.equal(tile.header.fieldVersion, 0x1234n); assert.equal(tile.header.layerId, 17); -assert.equal(tile.storageSize, 2); assert.equal(tile.samples.length, 4); -assert.equal(tile.samples[0]?.radialDisplacement, 0.25); assert.equal(tile.samples[1], null); -assert.throws(() => decodePlanetarySurfaceTile(bytes.slice(0, -1)), /Truncated/); -const trailing = new Uint8Array(bytes.byteLength + 1); trailing.set(new Uint8Array(bytes)); -assert.throws(() => decodePlanetarySurfaceTile(trailing.buffer), /trailing/); diff --git a/samples/eve-browser-network/EveBrowserNetworkSample.csproj b/samples/eve-browser-network/EveBrowserNetworkSample.csproj new file mode 100644 index 00000000..50b41781 --- /dev/null +++ b/samples/eve-browser-network/EveBrowserNetworkSample.csproj @@ -0,0 +1,23 @@ + + + net10.0 + enable + enable + $(MSBuildThisFileDirectory)..\..\..\Eve + false + 1.0.35 + 0.3.3 + $(PackageFeed) + + + + + + + + + + + + diff --git a/samples/eve-browser-network/Program.cs b/samples/eve-browser-network/Program.cs new file mode 100644 index 00000000..c26d654b --- /dev/null +++ b/samples/eve-browser-network/Program.cs @@ -0,0 +1,520 @@ +using System.Net; +using System.Diagnostics; +using System.Text.Json; +using GameCult.Caching; +using GameCult.Caching.MessagePack; +using GameCult.Eve.Surface; +using GameCult.Mesh; +using GameCult.Networking; +using GameCult.Networking.WebSockets; +using MessagePack; +using Microsoft.AspNetCore.Hosting.Server; +using Microsoft.AspNetCore.Hosting.Server.Features; +using Microsoft.Extensions.DependencyInjection; + +const string counterKey = "counter:main"; +const string surfaceKey = "sample.counter"; +const string providerKey = "sample.counter-ui"; +const string commandBoundary = "sample.counter.commands"; +var arguments = Args.Parse(args); +if (arguments.Mode == "provider") +{ + await RunProviderAsync(arguments); + return; +} +if (arguments.Mode == "headless") +{ + await RunHeadlessAsync(arguments); + return; +} +if (arguments.Mode == "odin") +{ + await RunOdinAsync(arguments); + return; +} +throw new InvalidOperationException("Use 'provider', 'headless', or 'odin'."); + +static async Task RunOdinAsync(Args arguments) +{ + if (string.IsNullOrWhiteSpace(arguments.ProviderEndpoint)) + throw new ArgumentException("The Odin fixture requires --provider-endpoint."); + using var catalog = new CultMeshVerseCatalog(); + var routes = new List + { + new( + arguments.AuthorityRuntimeId, + arguments.ProviderEndpoint, + [CultMeshProtocols.Documents.Value], + priority: 10, + generation: arguments.RouteGeneration) + }; + if (!string.IsNullOrWhiteSpace(arguments.DecoyEndpoint)) + { + routes.Add(new CultMeshAuthorityRoute( + arguments.DecoyAuthorityRuntimeId, + arguments.DecoyEndpoint, + [CultMeshProtocols.Documents.Value], + priority: 0, + generation: arguments.DecoyRouteGeneration)); + } + catalog.Upsert(new CultMeshVerseDescriptor( + arguments.VerseId, + arguments.VerseName, + CultMeshVerseAuthorityModel.OperatorCluster, + new CultMeshVerseCompatibility(arguments.TransportVersion, arguments.RulesHash), + authorityRoutes: routes, + description: "Local executable Odin fixture for the browser/network onboarding witness.")); + + await using var schemaServer = new CultNetWebSocketSchemaServer(); + schemaServer.OnCultNet((request, peer) => + { + peer.SendCultNet(CultMeshVerseMessages.CreateCatalogResponse(catalog, request)); + return Task.CompletedTask; + }); + var builder = WebApplication.CreateSlimBuilder(); + builder.WebHost.ConfigureKestrel(options => options.Listen(IPAddress.Loopback, arguments.Port)); + var app = builder.Build(); + app.UseWebSockets(); + app.MapCultNetWebSocket("/odin", schemaServer, new CultNetWebSocketEndpointOptions + { + AuthorizeAsync = context => ValueTask.FromResult( + context.Request.Cookies.TryGetValue("cultnet_session", out var token) && + string.Equals(token, arguments.Token, StringComparison.Ordinal)) + }); + await app.StartAsync(); + var address = app.Services.GetRequiredService() + .Features.Get()!.Addresses.Single(); + Console.WriteLine("ODIN_READY " + address.Replace("http://", "ws://", StringComparison.Ordinal) + "/odin"); + await app.WaitForShutdownAsync(); + await app.DisposeAsync(); +} + +static async Task RunProviderAsync(Args arguments) +{ + var documentTypes = new[] { typeof(CounterState), typeof(EveSurfaceDocument), typeof(EveProviderAdvertisementDocument) }; + var registry = CultMesh.CreateCultCacheDocumentRegistry(documentTypes); + var documents = CultMesh.CreateCultNetDocumentRegistry(documentTypes, registry); + using var node = await CultMesh.CreateNodeAsync(arguments.StatePath, new CultMeshNodeOptions + { + StartServer = false, + CacheOptions = new CultCacheOpenOptions + { + Registry = registry, + FlushOnDispose = true, + StoreFlushOnDispose = true + }, + DatabaseOptions = new CultNetDatabaseOptions + { + RuntimeId = arguments.AuthorityRuntimeId, + DocumentRegistry = documents + } + }); + if (node.Cache.Get(new CultRecordKey(counterKey)) == null) + await node.Database.PutAsync(new CultRecordKey(counterKey), CounterState.Initial(counterKey)); + if (node.Cache.Get(new CultRecordKey(surfaceKey)) == null) + await node.Database.PutAsync(new CultRecordKey(surfaceKey), CreateCounterSurface()); + // The advertisement is how a lowering learns where this surface's commands + // go and what comes back. Eve refuses to build a command intent without it, + // and the browser must not have to restate it: the provider is the owner. + await node.Database.PutAsync(new CultRecordKey(providerKey), CreateProviderAdvertisement(arguments)); + await node.FlushAsync(); + + await using var schemaServer = new CultNetWebSocketSchemaServer(); + using var identity = new CultMeshSessionIdentityServer( + schemaServer, + arguments.AuthorityRuntimeId, + [arguments.VerseId], + [CultMeshProtocols.Documents.Value], + [arguments.RouteGeneration]); + using var subscriptions = new CultNetDatabaseSubscriptionServer(schemaServer, node.Database); + using var operationGate = new SemaphoreSlim(1, 1); + using var operations = new CultNetOperationServer(schemaServer, arguments.AuthorityRuntimeId) + .Register( + "sample.counter", + "sample.counter.ping", + "sample.ping_request.v1", + "sample.ping_receipt.v1", + context => Task.FromResult(new PingReceipt { Sequence = context.Value.Sequence })) + .Register( + "sample.counter", + "sample.counter.increment", + "sample.increment.v1", + "gamecult.eve.command_receipt.v1", + async context => + { + await operationGate.WaitAsync(); + try + { + var current = node.Cache.Get(new CultRecordKey(counterKey)) + ?? throw new InvalidOperationException("Counter state is unavailable."); + if (current.Receipts.TryGetValue(context.IdempotencyKey, out var existing)) + return existing; + + var receipt = IncrementReceipt.Accepted( + context.IdempotencyKey, + current.Count + context.Value.Amount); + var receipts = new Dictionary(current.Receipts, StringComparer.Ordinal) + { + [context.IdempotencyKey] = receipt + }; + await node.Database.PutAsync(new CultRecordKey(counterKey), new CounterState + { + CounterId = counterKey, + Count = receipt.Count, + Receipts = receipts + }); + await node.FlushAsync(); + return receipt; + } + finally + { + operationGate.Release(); + } + }); + + var builder = WebApplication.CreateSlimBuilder(); + builder.WebHost.ConfigureKestrel(options => options.Listen(IPAddress.Loopback, arguments.Port)); + var app = builder.Build(); + app.UseWebSockets(); + app.MapCultNetWebSocket("/cultmesh", schemaServer, new CultNetWebSocketEndpointOptions + { + AuthorizeAsync = context => ValueTask.FromResult( + context.Request.Cookies.TryGetValue("cultnet_session", out var token) && + string.Equals(token, arguments.Token, StringComparison.Ordinal)) + }); + await app.StartAsync(); + var address = app.Services.GetRequiredService() + .Features.Get()!.Addresses.Single(); + Console.WriteLine("PROVIDER_READY " + address.Replace("http://", "ws://", StringComparison.Ordinal) + "/cultmesh"); + await app.WaitForShutdownAsync(); + await app.DisposeAsync(); +} + +static async Task RunHeadlessAsync(Args arguments) +{ + if (string.IsNullOrWhiteSpace(arguments.OdinEndpoint)) + throw new ArgumentException("The headless client requires --odin."); + CultNetWebSocketSchemaClient CreateClient() => new(options => + options.SetRequestHeader("Cookie", $"cultnet_session={arguments.Token}")); + using var mesh = new CultMeshClient(new CultMeshClientOptions + { + RendezvousEndpoints = [arguments.OdinEndpoint], + Discovery = new CultMeshVerseDiscoveryClientOptions + { + CreateClient = CreateClient, + TransportVersion = arguments.TransportVersion + }, + Sessions = new CultMeshSessionManagerOptions + { + Trust = new CultMeshAuthorityTrustPolicy(CultMeshAuthorityTrustMode.LocalDevelopment) + }, + Connectors = + [ + new CultMeshUriSchemaTransportConnector( + "cultnet-websocket", + ["ws", "wss"], + _ => CreateClient()) + ] + }); + var target = new CultMeshSessionTarget(arguments.VerseId, arguments.AuthorityRuntimeId); + using var lease = await mesh.LeaseDocumentAsync(target, counterKey); + var completion = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var lastCount = -1; + using var watch = lease.Handle.Watch(counter => + { + if (counter.Count <= lastCount) return; + lastCount = counter.Count; + if (counter.Count == 0) + Console.WriteLine("HEADLESS_READY " + JsonSerializer.Serialize(new { count = counter.Count })); + else + Console.WriteLine($"HEADLESS_UPDATE_{counter.Count} " + JsonSerializer.Serialize(new + { + count = counter.Count, + receiptIds = counter.Receipts.Values + .Select(receipt => receipt.ReceiptId) + .OrderBy(value => value, StringComparer.Ordinal) + .ToArray() + })); + if (counter.Count >= arguments.ExpectedCount) completion.TrySetResult(); + }); + var initial = await lease.Handle.LatestAsync(); + if (lastCount < 0) + { + lastCount = initial.Count; + Console.WriteLine("HEADLESS_READY " + JsonSerializer.Serialize(new { count = initial.Count })); + } + if (initial.Count >= arguments.ExpectedCount) completion.TrySetResult(); + var commandTask = Task.Run(async () => + { + var line = await Console.In.ReadLineAsync(); + if (line == null || !line.StartsWith("INVOKE ", StringComparison.Ordinal)) + throw new InvalidOperationException("The headless sample expected 'INVOKE ' on stdin."); + var commandId = line["INVOKE ".Length..].Trim(); + var result = await mesh.InvokeAsync( + target, + "sample.counter", + "sample.counter.increment", + "sample.increment.v1", + "gamecult.eve.command_receipt.v1", + new IncrementRequest { Amount = 1 }, + sourceRuntimeId: "sample.csharp", + idempotencyKey: commandId); + Console.WriteLine("HEADLESS_RECEIPT " + JsonSerializer.Serialize(new + { + receiptId = result.Value.ReceiptId, + count = result.Value.Count, + status = result.Status + })); + Console.WriteLine("HEADLESS_NETWORK_BENCHMARK " + JsonSerializer.Serialize( + await MeasureOperationSessionAsync(mesh, target, arguments))); + }); + await Task.WhenAll(completion.Task, commandTask).WaitAsync(TimeSpan.FromSeconds(45)); +} + +static async Task MeasureOperationSessionAsync( + CultMeshClient mesh, + CultMeshSessionTarget target, + Args arguments) +{ + const int warmupOperations = 100; + if (arguments.BenchmarkOperations <= 0) + throw new ArgumentOutOfRangeException(nameof(arguments.BenchmarkOperations)); + for (var index = 0; index < warmupOperations; index++) + await PingAsync(mesh, target, "warmup:" + index, index); + + ForceCollection(); + using var process = Process.GetCurrentProcess(); + process.Refresh(); + var managedHeapBefore = GC.GetTotalMemory(forceFullCollection: false); + var privateBytesBefore = process.PrivateMemorySize64; + var latencies = new double[arguments.BenchmarkOperations]; + var clock = Stopwatch.StartNew(); + for (var index = 0; index < arguments.BenchmarkOperations; index++) + { + var started = Stopwatch.GetTimestamp(); + await PingAsync(mesh, target, "benchmark:" + index, index); + latencies[index] = Stopwatch.GetElapsedTime(started).TotalMilliseconds; + } + clock.Stop(); + ForceCollection(); + process.Refresh(); + var managedHeapAfter = GC.GetTotalMemory(forceFullCollection: false); + var privateBytesAfter = process.PrivateMemorySize64; + Array.Sort(latencies); + var p99 = latencies[Math.Clamp((int)Math.Ceiling(latencies.Length * 0.99) - 1, 0, latencies.Length - 1)]; + var managedHeapGrowth = managedHeapAfter - managedHeapBefore; + if (managedHeapGrowth > 8 * 1024 * 1024) + throw new InvalidOperationException($"Retained operation session grew the managed heap by {managedHeapGrowth} bytes."); + if (p99 > 250) + throw new InvalidOperationException($"Retained operation session p99 was {p99:F2} ms."); + return new + { + operations = arguments.BenchmarkOperations, + p99Milliseconds = p99, + operationsPerSecond = arguments.BenchmarkOperations / clock.Elapsed.TotalSeconds, + managedHeapBefore, + managedHeapAfter, + managedHeapGrowth, + privateBytesBefore, + privateBytesAfter, + privateBytesGrowth = privateBytesAfter - privateBytesBefore + }; +} + +static async Task PingAsync( + CultMeshClient mesh, + CultMeshSessionTarget target, + string idempotencyKey, + int sequence) +{ + var result = await mesh.InvokeAsync( + target, + "sample.counter", + "sample.counter.ping", + "sample.ping_request.v1", + "sample.ping_receipt.v1", + new PingRequest { Sequence = sequence }, + sourceRuntimeId: "sample.csharp", + idempotencyKey: idempotencyKey); + if (result.Status != "accepted" || result.Value.Sequence != sequence) + throw new InvalidOperationException("The typed ping receipt did not match its request."); +} + +static void ForceCollection() +{ + GC.Collect(); + GC.WaitForPendingFinalizers(); + GC.Collect(); +} + +static EveProviderAdvertisementDocument CreateProviderAdvertisement(Args arguments) +{ + var worldInteraction = new EveWorldInteractionAdvertisement( + projectionKind: "surface-state", + stateSchemas: ["sample.counter_state.v1"], + commandBoundary: commandBoundary, + commandRecordRef: "", + receiptSchema: EveCommandReceiptDocument.SchemaId, + receiptRecordRef: "", + assetManifestRecordRef: "", + loweringTargets: ["browser"], + ownership: arguments.AuthorityRuntimeId); + return new EveProviderAdvertisementDocument( + providerId: providerKey, + serviceId: arguments.VerseId, + verseId: arguments.VerseId, + title: "CultMesh browser counter", + kind: "sample.daemon", + cultMeshAddress: arguments.AuthorityRuntimeId, + updatedAtUtc: DateTime.UtcNow.ToString("O"), + freshness: new EveProviderFreshness("live", DateTime.UtcNow.ToString("O"), 0), + schemas: ["sample.counter_state.v1", EveSurfaceDocument.SchemaId, EveCommandReceiptDocument.SchemaId], + witnesses: [], + surfaces: + [ + new EveAdvertisedSurface( + surfaceKey, + EveSurfaceDocument.SchemaId, + surfaceKey, + "cultmesh", + "live", + "surface", + worldInteraction) + ], + commands: [new EveAdvertisedCommand("sample.counter.increment", surfaceKey, "cultmesh", "Increment")]); +} + +static EveSurfaceDocument CreateCounterSurface() +{ + var networkRoute = new CultMeshRouteHint(CultMeshLocalityKind.Network, "cultmesh"); + var count = new CultMeshStateBindingDescriptor( + "value", + "sample.counter.count", + "sample.counter_state:counter:main", + "sample.counter_state.v1", + networkRoute); + var increment = CultMesh.OperationBinding( + "sample.counter.increment", + "Increment", + "sample.increment.v1", + networkRoute); + return EveSurface.Create(surfaceKey) + .Provider("sample.counter-ui", "sample.daemon") + .Title("CultMesh browser counter") + .Version(1) + .UpdatedAtUtc("2026-08-17T00:00:00Z") + .RootColumn("counter.root", root => root + .Title("counter.title", "CultMesh browser counter") + .Metric("counter.value", "Canonical count", "0", count) + .Button("counter.increment", "Increment", increment)) + .Build(); +} + +sealed class Args +{ + public string Mode { get; private init; } = ""; + public int Port { get; private init; } + public string StatePath { get; private init; } = "sample-counter.cc"; + public string OdinEndpoint { get; private init; } = ""; + public string ProviderEndpoint { get; private init; } = ""; + public string DecoyEndpoint { get; private init; } = ""; + public string Token { get; private init; } = "sample-session"; + public string VerseId { get; private init; } = "sample.counter"; + public string VerseName { get; private init; } = "CultMesh browser counter"; + public string AuthorityRuntimeId { get; private init; } = "sample.counter-daemon"; + public string DecoyAuthorityRuntimeId { get; private init; } = "sample.decoy-daemon"; + public string TransportVersion { get; private init; } = "cultmesh.v0"; + public string RulesHash { get; private init; } = "sample-counter-v1"; + public string RouteGeneration { get; private init; } = "sample-counter-route-1"; + public string DecoyRouteGeneration { get; private init; } = "sample-decoy-route-1"; + public int ExpectedCount { get; private init; } = 2; + public int BenchmarkOperations { get; private init; } = 10_000; + + public static Args Parse(string[] values) + { + if (values.Length == 0) return new Args(); + var named = new Dictionary(StringComparer.Ordinal); + for (var index = 1; index + 1 < values.Length; index += 2) + named[values[index]] = values[index + 1]; + return new Args + { + Mode = values[0], + Port = named.TryGetValue("--port", out var port) ? int.Parse(port) : 0, + StatePath = named.GetValueOrDefault("--state", "sample-counter.cc"), + OdinEndpoint = named.GetValueOrDefault("--odin", ""), + ProviderEndpoint = named.GetValueOrDefault("--provider-endpoint", ""), + DecoyEndpoint = named.GetValueOrDefault("--decoy-endpoint", ""), + Token = named.GetValueOrDefault("--token", "sample-session"), + VerseId = named.GetValueOrDefault("--verse-id", "sample.counter"), + VerseName = named.GetValueOrDefault("--verse-name", "CultMesh browser counter"), + AuthorityRuntimeId = named.GetValueOrDefault("--authority-runtime-id", "sample.counter-daemon"), + DecoyAuthorityRuntimeId = named.GetValueOrDefault("--decoy-authority-runtime-id", "sample.decoy-daemon"), + TransportVersion = named.GetValueOrDefault("--transport-version", "cultmesh.v0"), + RulesHash = named.GetValueOrDefault("--rules-hash", "sample-counter-v1"), + RouteGeneration = named.GetValueOrDefault("--route-generation", "sample-counter-route-1"), + DecoyRouteGeneration = named.GetValueOrDefault("--decoy-route-generation", "sample-decoy-route-1"), + ExpectedCount = named.TryGetValue("--expected-count", out var expectedCount) + ? int.Parse(expectedCount) + : 2, + BenchmarkOperations = named.TryGetValue("--benchmark-operations", out var benchmarkOperations) + ? int.Parse(benchmarkOperations) + : 10_000 + }; + } +} + +[CultDocument("sample.counter_state", "sample.counter_state.v1")] +[MessagePackObject] +public sealed class CounterState +{ + [Key(0)] public string CounterId { get; set; } = ""; + [Key(1)] public int Count { get; set; } + [Key(2)] public Dictionary Receipts { get; set; } = new(StringComparer.Ordinal); + public static CounterState Initial(string counterId) => new() { CounterId = counterId }; +} + +[MessagePackObject] +public sealed class IncrementRequest +{ + [Key("amount")] public int Amount { get; set; } = 1; +} + +[MessagePackObject] +public sealed class IncrementReceipt +{ + [Key("receiptId")] public string ReceiptId { get; set; } = ""; + [Key("schema")] public string Schema { get; set; } = "gamecult.eve.command_receipt.v1"; + [Key("commandId")] public string CommandId { get; set; } = ""; + [Key("command")] public string Command { get; set; } = "sample.counter.increment"; + [Key("state")] public string State { get; set; } = "accepted"; + [Key("ownerRepo")] public string OwnerRepo { get; set; } = "CultLib"; + [Key("authority")] public string Authority { get; set; } = "provider-daemon"; + [Key("providerId")] public string ProviderId { get; set; } = "sample.counter-ui"; + [Key("surfaceId")] public string SurfaceId { get; set; } = "sample.counter"; + [Key("issuedAtUtc")] public string IssuedAtUtc { get; set; } = ""; + [Key("sourceVersion")] public int SourceVersion { get; set; } + [Key("idempotencyKey")] public string IdempotencyKey { get; set; } = ""; + [Key("count")] public int Count { get; set; } + + public static IncrementReceipt Accepted(string id, int count) => new() + { + ReceiptId = $"receipt:{id}", + CommandId = id, + IssuedAtUtc = DateTimeOffset.UtcNow.ToString("O"), + SourceVersion = count, + IdempotencyKey = id, + Count = count + }; +} + +[MessagePackObject] +public sealed class PingRequest +{ + [Key("sequence")] public int Sequence { get; set; } +} + +[MessagePackObject] +public sealed class PingReceipt +{ + [Key("sequence")] public int Sequence { get; set; } +} diff --git a/samples/eve-browser-network/README.md b/samples/eve-browser-network/README.md new file mode 100644 index 00000000..445bfb9d --- /dev/null +++ b/samples/eve-browser-network/README.md @@ -0,0 +1,58 @@ +# Eve Browser + Headless Counter + +This is the real-process continuation of `samples/eve-two-runtime`: + +- one C# provider owns a durable `.cc` counter, the Eve surface, command + idempotency, and canonical receipts; +- one real Chromium page leases both documents through `cultmesh-browser` and + lowers the provider-owned Eve surface; +- one retained C# headless client discovers the provider through the same Odin + identity, observes the typed counter and canonical receipt ids, survives the + provider route replacement, and invokes through the generic CultMesh client + without application transport or reconnect code; +- one separate local Odin fixture answers the canonical CultMesh Verse catalog + over binary CultNet WebSocket messages; +- one wrong-authority provider is deliberately advertised on a better-priority + route, and both clients must ignore it because the route is bound to another + runtime; +- the browser and C# clients speak the same binary CultNet schema-v0 WebSocket + messages through the authenticated host adapter. + +The provider registers typed commands through `CultNetOperationServer`. +Application code receives `CultNetOperationContext` and returns a +typed receipt; it does not parse base64, switch on wire envelopes, or construct +correlation responses. The counter's durable transaction still owns +idempotency and state mutation. The provider opens one public `CultMeshNode`; +it does not assemble a parallel CultCache/CultNet database host by hand. + +Run the proof from the CultLib repository: + +```powershell +pwsh -File ./scripts/verify-eve-getting-started.ps1 +``` + +The wrapper first proves clean package consumption, then runs this real-process +checkpoint. Run `node scripts/verify-eve-browser-network.mjs --eve-root ../Eve` +directly only to diagnose the network layer. Browser discovery is portable: +`CHROME_PATH` wins when configured, then common Chrome/Chromium/Edge locations +and Playwright's installed Chromium are checked. + +The verifier clicks the Eve button twice with one idempotency key and requires +one state transition. It then restarts the provider on a different physical +port, updates the local Odin fixture, and requires both retained clients to +rediscover and resubscribe. The C# client invokes the second command through +`CultMeshClient.InvokeAsync`; Chromium observes the resulting count and both +clients observe both canonical receipt ids. +The retained C# client then sends 10,000 typed no-op operations over the same +real WebSocket session. The gate records p99 latency, throughput, managed heap, +and private memory, and rejects more than 8 MiB of post-GC managed growth or a +250 ms p99. The no-op provider handler stores no receipt history, so persistence +growth cannot impersonate a client leak. +The fixture speaks the production Verse-catalog contract; a deployment smoke +against the full Odin daemon remains a separate infrastructure gate. + +The `odin` fixture defaults to this sample's Verse, but its `--verse-id`, +`--verse-name`, `--authority-runtime-id`, `--transport-version`, and +`--rules-hash` arguments can advertise another real provider during a product +route-rotation witness. The provider endpoint remains data owned by Odin; the +consumer still connects only by stable Verse and provider identity. diff --git a/samples/eve-browser-network/browser/index.html b/samples/eve-browser-network/browser/index.html new file mode 100644 index 00000000..d23ad8ab --- /dev/null +++ b/samples/eve-browser-network/browser/index.html @@ -0,0 +1,13 @@ + + + + + + CultMesh Eve browser counter + + +
Connecting to the Verse…
+ + + + diff --git a/samples/eve-browser-network/browser/main.ts b/samples/eve-browser-network/browser/main.ts new file mode 100644 index 00000000..fac4cc43 --- /dev/null +++ b/samples/eve-browser-network/browser/main.ts @@ -0,0 +1,110 @@ +import { parseEveCommandReceipt, parseEveProviderAdvertisement, parseEveSurfaceDocument } from "@gamecult/eve-contracts"; +import { renderEveSurface } from "@gamecult/eve-browser-lowering"; +import { + CultMeshBrowserClient, + CultMeshBrowserOdinRendezvous, + decodeCultNetOperationPayload, + decodeCultNetPayload, +} from "cultmesh-browser"; + +// CounterState is an integer-keyed MessagePack object: [counterId, count, receipts]. +const counterCount = (record: Parameters[0]) => + decodeCultNetPayload<[string, number, unknown]>(record)[1]; + +declare global { + interface Window { + __sampleReady?: boolean; + __sampleCount?: number; + __sampleReceipt?: { receiptId: string; count: number }; + __sampleCommandId?: string; + __sampleConnectionStates?: string[]; + __sampleError?: string; + } +} + +const params = new URLSearchParams(location.search); +const odinEndpoint = params.get("odin"); +const token = params.get("token") || "sample-session"; +if (!odinEndpoint) throw new Error("The sample requires an Odin endpoint query parameter."); +document.cookie = `cultnet_session=${encodeURIComponent(token)}; Path=/; SameSite=Strict`; + +const host = document.querySelector("#surface")!; +const receiptOutput = document.querySelector("#receipt")!; + +try { + const mesh = await CultMeshBrowserClient.connect({ + verseId: "sample.counter", + authorityRuntimeId: "sample.counter-daemon", + runtimeId: "sample.chromium", + trust: { mode: "local-development" }, + rendezvous: new CultMeshBrowserOdinRendezvous({ + endpoints: [odinEndpoint], + runtimeId: "sample.chromium.discovery", + transportVersion: "cultmesh.v0", + }), + }); + window.__sampleConnectionStates = []; + mesh.watchState(state => window.__sampleConnectionStates!.push(state)); + window.__sampleCommandId = "browser-click-1"; + const counter = await mesh.leaseRawDocument({ + schemaId: "sample.counter_state.v1", + recordKey: "counter:main", + subscriptionId: "browser-counter", + }); + const surface = await mesh.leaseRawDocument({ + schemaId: "gamecult.eve.surface.v1", + recordKey: "sample.counter", + subscriptionId: "browser-surface", + }); + // The provider owns the declaration of where this surface's commands go and + // what comes back; the page reads it rather than restating it. + const advertisement = await mesh.leaseRawDocument({ + schemaId: "gamecult.eve.provider_advertisement.v1", + recordKey: "sample.counter-ui", + subscriptionId: "browser-provider", + }); + const provider = parseEveProviderAdvertisement(decodeCultNetPayload(advertisement.current!)); + const currentSurface = parseEveSurfaceDocument(decodeCultNetPayload(surface.current!)); + renderEveSurface(currentSurface, host, { + activeSurfaceId: "sample.counter", + clientId: "sample.chromium", + provider, + commandSink: async intent => { + const response = await mesh.invoke({ + serviceId: "sample.counter", + operation: intent.operation.operationId, + payloadSchema: "sample.increment.v1", + payload: { amount: 1 }, + idempotencyKey: window.__sampleCommandId, + }); + const receipt = parseEveCommandReceipt(decodeCultNetOperationPayload(response)) as { + receiptId: string; + count: number; + }; + window.__sampleReceipt = receipt; + receiptOutput.textContent = receipt.receiptId; + }, + stateBindingResolver: async binding => { + if (binding.pointerId !== "sample.counter.count") return undefined; + return { + latest: async () => counterCount(counter.current!), + watch: callback => counter.watch(record => { + if (!record) return; + const count = counterCount(record); + window.__sampleCount = count; + callback(count); + }), + }; + }, + }); + window.__sampleCount = counterCount(counter.current!); + window.__sampleReady = true; +} catch (error) { + const details = error instanceof AggregateError + ? error.errors.map(inner => inner instanceof Error ? inner.stack || inner.message : String(inner)).join("\n---\n") + : ""; + window.__sampleError = (error instanceof Error ? error.stack || error.message : String(error)) + + (details ? `\n${details}` : ""); + host.textContent = window.__sampleError; + throw error; +} diff --git a/samples/eve-two-runtime/README.md b/samples/eve-two-runtime/README.md new file mode 100644 index 00000000..0981ad1a --- /dev/null +++ b/samples/eve-two-runtime/README.md @@ -0,0 +1,37 @@ +# Eve Two-Consumer Counter + +This is the smallest executable package-consumer checkpoint for CultMesh plus +Eve. It intentionally has one owner for each decision: + +- the provider node owns the counter document and command transaction; +- Eve owns the validated surface and command-receipt contracts; +- CultMesh owns typed observation and the operation boundary; +- the DOM lowerer owns HTML only; +- the headless observer owns no state. + +Run it from the CultLib repository: + +```powershell +pwsh -File ./scripts/verify-eve-getting-started.ps1 +``` + +That command continues into the real Chromium/C# network checkpoint after this +artifact layer passes. Run `verify-eve-two-runtime-sample.ps1` directly only to +diagnose clean package consumption in isolation. + +From clean CultLib and Eve checkouts, the verifier installs missing build +dependencies, builds package artifacts, installs them into a new empty +temporary consumer, and runs `sample.mjs`. Success proves: + +1. no source-tree import is required by the consumer; +2. the provider surface and receipt pass Eve's generated runtime validators; +3. a DOM click executes one idempotent provider operation; +4. the DOM binding and a separate headless observer converge on the same count; +5. the canonical receipt and state survive reopening the `.cc` store. + +This artifact checkpoint is deliberately not the network proof. The DOM lowerer +is hosted by jsdom in Node and both consumers share one local CultMesh node. The +same wrapper continues into a separate real Chromium/C# process chronology with +Odin discovery, route replacement, and persistent provider state. Keeping those +evidence layers separate prevents a fast package test from impersonating the +network. diff --git a/samples/eve-two-runtime/sample.mjs b/samples/eve-two-runtime/sample.mjs new file mode 100644 index 00000000..93ed1dc8 --- /dev/null +++ b/samples/eve-two-runtime/sample.mjs @@ -0,0 +1,247 @@ +import assert from "node:assert/strict"; +import { mkdtemp } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { JSDOM } from "jsdom"; +import { z } from "zod"; +import { defineDocumentType } from "@gamecult/cultcache-ts"; +import { CultMesh } from "cultmesh-ts"; +import { + parseEveCommandReceipt, + parseEveSurfaceDocument, +} from "@gamecult/eve-contracts"; +import { renderEveSurface } from "@gamecult/eve-browser-lowering"; + +const providerId = "sample.counter-provider"; +const surfaceId = "sample.counter"; +const stateKey = "counter:main"; + +const counterDocument = defineDocumentType({ + type: "sample.counter_state", + schemaId: "sample.counter_state.v1", + schema: z.object({ counterId: z.string(), count: z.number().int() }), + name: "counterId", +}); + +const surfaceDocument = defineDocumentType({ + type: "gamecult.eve.surface", + schemaId: "gamecult.eve.surface.v1", + schema: z.unknown().transform(value => parseEveSurfaceDocument(value)), +}); + +const receiptDocument = defineDocumentType({ + type: "gamecult.eve.command_receipt", + schemaId: "gamecult.eve.command_receipt.v1", + schema: z.unknown().transform(value => parseEveCommandReceipt(value)), + name: "receiptId", +}); + +function counterSurface() { + return { + type: "surface-state", + schema: "gamecult.eve.surface.v1", + providerId, + providerKind: "sample.daemon", + title: "CultMesh two-runtime counter", + version: 1, + updatedAtUtc: "2026-08-17T00:00:00Z", + surface: { + id: surfaceId, + title: "Counter", + root: { + id: "counter.root", + kind: "column", + props: {}, + children: [ + { + id: "counter.value", + kind: "metric", + props: { label: "Canonical count", value: 0 }, + children: [], + stateBindings: [{ + targetProp: "value", + pointerId: "sample.counter.count", + sourceId: `${counterDocument.type}:${stateKey}`, + schemaId: counterDocument.schemaId, + routeKind: "shared-memory", + }], + }, + { + id: "counter.increment", + kind: "control.button", + props: { label: "Increment", command: "sample.counter.increment" }, + children: [], + }, + ], + }, + styles: {}, + }, + commands: [{ + schema: "gamecult.eve.command.v1", + command: "sample.counter.increment", + label: "Increment", + surfaceId, + transport: "cultmesh", + authority: "provider-daemon", + result: "gamecult.eve.command_receipt.v1", + }], + }; +} + +function installDom() { + const dom = new JSDOM("
", { + url: "https://eve.sample/", + pretendToBeVisual: true, + }); + for (const name of [ + "document", + "window", + "Element", + "HTMLElement", + "HTMLButtonElement", + "HTMLInputElement", + "HTMLTextAreaElement", + "HTMLImageElement", + ]) globalThis[name] = dom.window[name]; + return dom; +} + +async function waitFor(predicate, description) { + const deadline = Date.now() + 2_000; + while (!predicate()) { + if (Date.now() >= deadline) throw new Error(`Timed out waiting for ${description}.`); + await new Promise(resolve => setTimeout(resolve, 5)); + } +} + +assert.throws( + () => parseEveSurfaceDocument({ providerId, surface: { id: surfaceId } }), + /Invalid Eve surface/, +); +assert.throws( + () => parseEveCommandReceipt({ receiptId: "unowned" }), + /Invalid Eve commandReceipt/, +); + +const statePath = join(await mkdtemp(join(tmpdir(), "cultmesh-eve-two-runtime-")), "state.cc"); +const node = await CultMesh.startNode(statePath, { + documents: [counterDocument, surfaceDocument, receiptDocument], +}); +await node.put(counterDocument, stateKey, { counterId: stateKey, count: 0 }); +await node.put(surfaceDocument, surfaceId, counterSurface()); +await node.flush(); + +const state = node.document(counterDocument, stateKey, { pollMs: 5 }); +const headless = state.observe({ context: "headless-client" }); +await headless.ready; +const receiptsByIdempotencyKey = new Map(); +const increment = CultMesh.operation("sample.counter.increment", async (_request, context) => { + const idempotencyKey = context.idempotencyKey || "missing"; + const existing = receiptsByIdempotencyKey.get(idempotencyKey) + || node.get(receiptDocument, `receipt:${idempotencyKey}`); + if (existing) return existing; + const next = await state.authoritativeWriter("provider-daemon").update(current => ({ + ...current, + count: current.count + 1, + })); + const receipt = { + receiptId: `receipt:${idempotencyKey}`, + schema: "gamecult.eve.command_receipt.v1", + commandId: idempotencyKey, + command: "sample.counter.increment", + state: "accepted", + ownerRepo: "CultLib", + authority: "provider-daemon", + providerId, + surfaceId, + issuedAtUtc: new Date().toISOString(), + sourceVersion: next.count, + idempotencyKey, + count: next.count, + }; + receiptsByIdempotencyKey.set(idempotencyKey, receipt); + await node.put(receiptDocument, receipt.receiptId, receipt); + await node.flush(); + return receipt; +}); + +const dom = installDom(); +const host = document.querySelector("#surface"); +const browserVerse = CultMesh.verse("local", "browser-client"); +let browserReceipt; +renderEveSurface(await node.document(surfaceDocument, surfaceId).latest(), host, { + activeSurfaceId: surfaceId, + clientId: "browser-client", + // Eve refuses to build a command intent for a surface that has not + // advertised where its commands go and what comes back. The provider + // advertisement is that declaration; a lowering cannot infer it. + provider: { + providerId, + surfaces: [{ + surfaceId, + worldInteraction: { + commandBoundary: "sample.counter.commands", + receiptSchema: "gamecult.eve.command_receipt.v1", + }, + }], + }, + commandSink: async intent => { + browserReceipt = await increment.bind(browserVerse).invoke(intent.payload, { + idempotencyKey: "browser-click-1", + }); + }, + stateBindingResolver: async binding => { + if (binding.pointerId !== "sample.counter.count") return undefined; + return { + latest: async () => (await state.latest("browser-client")).count, + watch: callback => state.watch("browser-client", value => callback(value.count)), + }; + }, +}); + +await waitFor(() => host.querySelector(".metric-value")?.textContent === "0", "initial browser binding"); +host.querySelector("button").click(); +await waitFor(() => browserReceipt?.count === 1, "typed command receipt"); +await waitFor(() => headless.current.count === 1, "headless convergence"); +await waitFor(() => host.querySelector(".metric-value")?.textContent === "1", "browser convergence"); + +const duplicate = await increment.bind(browserVerse).invoke({}, { idempotencyKey: "browser-click-1" }); +assert.equal(duplicate.receiptId, browserReceipt.receiptId); +assert.equal(headless.current.count, 1); + +headless.dispose(); +renderEveSurface(parseEveSurfaceDocument({ + type: "surface-state", + schema: "gamecult.eve.surface.v1", + providerId, + providerKind: "sample.daemon", + title: "Closed counter", + version: 2, + updatedAtUtc: "2026-08-17T00:00:01Z", + surface: { + id: "sample.closed", + root: { id: "closed.root", kind: "column", props: {}, children: [] }, + styles: {}, + }, + commands: [], +}), host); +dom.window.close(); +await node.flush(); +const reopened = await CultMesh.startNode(statePath, { + documents: [counterDocument, surfaceDocument, receiptDocument], +}); +assert.equal(reopened.getRequired(counterDocument, stateKey).count, 1); +assert.equal( + reopened.getRequired(receiptDocument, "receipt:browser-click-1").receiptId, + browserReceipt.receiptId, +); + +console.log(JSON.stringify({ + providerId, + surfaceId, + receiptId: browserReceipt.receiptId, + loweringRuntime: "jsdom", + browserCount: 1, + headlessCount: 1, + restartCount: 1, +})); diff --git a/scripts/build-geometry-candidate.ps1 b/scripts/build-geometry-candidate.ps1 deleted file mode 100644 index 0643dd8b..00000000 --- a/scripts/build-geometry-candidate.ps1 +++ /dev/null @@ -1,37 +0,0 @@ -param( - [string] $GeometryVersion = "0.1.0", - [string] $SupportVersion = "1.0.3", - [string] $CultMathVersion = "0.1.0", - [string] $OutputDirectory = "artifacts\candidate-feed", - [string] $CultMathCandidateFeed = "..\CultMath\.tools\local-feed" -) -$ErrorActionPreference = "Stop" -$root = Split-Path -Parent $PSScriptRoot -$project = Join-Path $root "src\GameCult.Geometry\GameCult.Geometry.csproj" -$output = Join-Path $root $OutputDirectory -$feed = [IO.Path]::GetFullPath((Join-Path $root $CultMathCandidateFeed)) -New-Item -ItemType Directory -Force -Path $output | Out-Null -$cultMathPackage = Join-Path $feed "CultMath.$CultMathVersion.nupkg" -if (-not (Test-Path -LiteralPath $cultMathPackage)) { - throw "CultMath $CultMathVersion is missing from $feed. Pack the final CultMath artifact first." -} -$supportProjects = @( - "src\GameCult.Logging\GameCult.Logging.csproj", - "src\GameCult.Caching\GameCult.Caching.csproj", - "src\GameCult.Caching.MessagePack.Analyzers\GameCult.Caching.MessagePack.Analyzers.csproj", - "src\GameCult.Caching.MessagePack\GameCult.Caching.MessagePack.csproj", - "src\GameCult.Networking\GameCult.Networking.csproj", - "src\GameCult.Mesh\GameCult.Mesh.csproj" -) -foreach ($relativeProject in $supportProjects) { - $supportProject = Join-Path $root $relativeProject - dotnet pack $supportProject -c Release -o $output --disable-build-servers -p:PackageVersion=$SupportVersion -p:UseSharedCompilation=false -m:1 - if ($LASTEXITCODE -ne 0) { throw "Candidate support pack failed: $relativeProject" } -} -dotnet restore $project --disable-build-servers -p:CultMathCandidateFeed=$feed -p:CultMathVersion=$CultMathVersion -p:UseSharedCompilation=false -m:1 -if ($LASTEXITCODE -ne 0) { throw "Geometry candidate restore failed." } -dotnet pack $project -c Release -o $output --no-restore --disable-build-servers -p:CultMathVersion=$CultMathVersion -p:CultMathCandidateFeed=$feed -p:UseSharedCompilation=false -m:1 -if ($LASTEXITCODE -ne 0) { throw "Geometry candidate pack failed." } -$package = Join-Path $output "GameCult.Geometry.$GeometryVersion.nupkg" -if (-not (Test-Path -LiteralPath $package)) { throw "Geometry candidate package was not produced." } -Write-Host $package diff --git a/scripts/build-geometry-unity-package.ps1 b/scripts/build-geometry-unity-package.ps1 deleted file mode 100644 index 27f3c567..00000000 --- a/scripts/build-geometry-unity-package.ps1 +++ /dev/null @@ -1,117 +0,0 @@ -param( - [string] $Configuration = "Release", - [string] $OutputDirectory = "artifacts\unity\org.gamecult.geometry", - [string] $CultMathCandidateFeed = "..\CultMath\.tools\local-feed", - [string] $CultMathVersion = "0.1.0" -) - -$ErrorActionPreference = "Stop" - -$repoRoot = Split-Path -Parent $PSScriptRoot -$projectPath = Join-Path $repoRoot "src\GameCult.Geometry\GameCult.Geometry.csproj" -$templateRoot = Join-Path $repoRoot "unity\org.gamecult.geometry" -$publishRoot = Join-Path $repoRoot "artifacts\geometry-unity-publish" -$outputRoot = if ([System.IO.Path]::IsPathRooted($OutputDirectory)) { - $OutputDirectory -} else { - Join-Path $repoRoot $OutputDirectory -} -$candidateFeed = if ([System.IO.Path]::IsPathRooted($CultMathCandidateFeed)) { - $CultMathCandidateFeed -} else { - Join-Path $repoRoot $CultMathCandidateFeed -} -$pluginRoot = Join-Path $outputRoot "Runtime\Plugins" -$templatePluginRoot = Join-Path $templateRoot "Runtime\Plugins" - -if (-not (Test-Path -LiteralPath (Join-Path $candidateFeed "CultMath.$CultMathVersion.nupkg"))) { - throw "CultMath $CultMathVersion is missing from $candidateFeed. Pack CultMath before staging Geometry." -} - -if (Test-Path -LiteralPath $publishRoot) { - Remove-Item -LiteralPath $publishRoot -Recurse -Force -} -if (Test-Path -LiteralPath $outputRoot) { - Remove-Item -LiteralPath $outputRoot -Recurse -Force -} - -dotnet restore $projectPath --disable-build-servers -p:CultMathCandidateFeed=$candidateFeed -p:CultMathVersion=$CultMathVersion -p:UseSharedCompilation=false -m:1 -if ($LASTEXITCODE -ne 0) { - throw "GameCult.Geometry restore failed with exit code $LASTEXITCODE" -} - -dotnet publish $projectPath -c $Configuration -o $publishRoot --no-restore --disable-build-servers -p:UseSharedCompilation=false -m:1 -if ($LASTEXITCODE -ne 0) { - throw "GameCult.Geometry publish failed with exit code $LASTEXITCODE" -} - -$geometryAssembly = Join-Path $publishRoot "GameCult.Geometry.dll" -if (-not (Test-Path -LiteralPath $geometryAssembly)) { - throw "Geometry Unity package is missing GameCult.Geometry.dll" -} -$geometrySymbols = [System.IO.Path]::ChangeExtension($geometryAssembly, ".pdb") - -$requiredTemplateFiles = @( - "LICENSES.md", - "THIRD-PARTY-NOTICES.md", - "Runtime\Plugins\GameCult.Geometry.dll", - "Runtime\Plugins\GameCult.Geometry.dll.meta", - "Runtime\Plugins\GameCult.Geometry.pdb", - "Runtime\Plugins\GameCult.Geometry.pdb.meta", - "Runtime\Unity\GameCult.Geometry.Unity.asmdef", - "Runtime\Unity\PlanetaryPatchMeshAdapter.cs", - "Runtime\Unity\PlanetaryPageUpload.cs", - "Tests\Editor\GameCult.Geometry.Tests.Editor.asmdef", - "Tests\Editor\GeometryPackageTests.cs", - "Shaders\AdvancedErosionFilter.hlsl", - "Shaders\GameCult.Geometry.hlsl", - "Shaders\Planetary.hlsl", - "Shaders\PlanetaryRadialRefinement.hlsl", - "Shaders\SphericalErosion.hlsl" -) -foreach ($relativePath in $requiredTemplateFiles) { - if (-not (Test-Path -LiteralPath (Join-Path $templateRoot $relativePath))) { - throw "Tracked Geometry Unity package is incomplete: missing $relativePath" - } -} - -$unityAssetFiles = @(Get-ChildItem -LiteralPath $templateRoot -Recurse -File | Where-Object { - $_.Extension -in @(".asmdef", ".cs", ".dll", ".pdb", ".shader", ".hlsl") -}) -$missingAssetMetas = @($unityAssetFiles | Where-Object { -not (Test-Path -LiteralPath ($_.FullName + ".meta")) }) -$missingDirectoryMetas = @(Get-ChildItem -LiteralPath $templateRoot -Recurse -Directory | Where-Object { - -not (Test-Path -LiteralPath ($_.FullName + ".meta")) -}) -if ($missingAssetMetas.Count -ne 0 -or $missingDirectoryMetas.Count -ne 0) { - $missing = @($missingAssetMetas.FullName) + @($missingDirectoryMetas.FullName) - throw "Tracked Geometry Unity package has assets without stable .meta files: $($missing -join ', ')" -} - -$assemblyDirectories = @($unityAssetFiles | Where-Object { $_.Extension -eq ".asmdef" } | Group-Object DirectoryName) -$duplicateAssemblyDirectories = @($assemblyDirectories | Where-Object { $_.Count -ne 1 }) -if ($duplicateAssemblyDirectories.Count -ne 0) { - throw "Each Geometry Unity assembly directory must contain exactly one asmdef: $($duplicateAssemblyDirectories.Name -join ', ')" -} - -$templateManifest = Get-Content -LiteralPath (Join-Path $templateRoot "package.json") -Raw | ConvertFrom-Json -$expectedFileVersion = "$($templateManifest.version).0" -$trackedAssembly = Join-Path $templatePluginRoot "GameCult.Geometry.dll" -$trackedFileVersion = [Diagnostics.FileVersionInfo]::GetVersionInfo($trackedAssembly).FileVersion -$publishedFileVersion = [Diagnostics.FileVersionInfo]::GetVersionInfo($geometryAssembly).FileVersion -if ($trackedFileVersion -ne $expectedFileVersion -or $publishedFileVersion -ne $expectedFileVersion) { - throw "Geometry Unity assemblies must match package version $expectedFileVersion; tracked=$trackedFileVersion published=$publishedFileVersion" -} - -New-Item -ItemType Directory -Force -Path $outputRoot | Out-Null -Copy-Item -Path (Join-Path $templateRoot "*") -Destination $outputRoot -Recurse -Force - -$manifest = Get-Content -LiteralPath (Join-Path $outputRoot "package.json") -Raw | ConvertFrom-Json -$stagedAssemblies = @(Get-ChildItem -LiteralPath $pluginRoot -Filter "*.dll") -if ($stagedAssemblies.Count -ne 1 -or $stagedAssemblies[0].Name -ne "GameCult.Geometry.dll") { - throw "Geometry package must stage only its owned assembly; found: $($stagedAssemblies.Name -join ', ')" -} - -Write-Host "GameCult.Geometry Unity package: $outputRoot" -Write-Host "Package: $($manifest.name)@$($manifest.version)" -Write-Host "Dependencies: $($manifest.dependencies.PSObject.Properties.Name -join ', ')" -Write-Host "Managed assemblies: $($stagedAssemblies.Count)" diff --git a/scripts/build-quic-native.ps1 b/scripts/build-quic-native.ps1 index 2de12bde..68496180 100644 --- a/scripts/build-quic-native.ps1 +++ b/scripts/build-quic-native.ps1 @@ -30,7 +30,14 @@ if (-not (Test-Path -LiteralPath $packagePath)) { "https://www.nuget.org/api/v2/package/Microsoft.Native.Quic.MsQuic.Schannel/$version" if ($LASTEXITCODE -ne 0) { throw "MsQuic package download failed with exit code $LASTEXITCODE" } } -if ((Get-FileHash -LiteralPath $packagePath -Algorithm SHA256).Hash -ne $expectedPackageSha256) { +$sha256 = [Security.Cryptography.SHA256]::Create() +try { + $packageBytes = [IO.File]::ReadAllBytes($packagePath) + $packageSha256 = [BitConverter]::ToString($sha256.ComputeHash($packageBytes)).Replace("-", "") +} finally { + $sha256.Dispose() +} +if ($packageSha256 -ne $expectedPackageSha256) { throw "MsQuic package digest does not match the pinned release." } if (-not (Test-Path -LiteralPath (Join-Path $nativeRoot "include\msquic.h"))) { diff --git a/scripts/build-unity-package.ps1 b/scripts/build-unity-package.ps1 index dbc01233..804f78df 100644 --- a/scripts/build-unity-package.ps1 +++ b/scripts/build-unity-package.ps1 @@ -1,6 +1,8 @@ param( [string] $Configuration = "Release", [string] $OutputDirectory = "artifacts\unity\org.gamecult.cultlib", + [string] $NuGetConfig = "", + [switch] $NoRestore, [switch] $UpdateTemplate ) @@ -8,9 +10,11 @@ $ErrorActionPreference = "Stop" $repoRoot = Split-Path -Parent $PSScriptRoot $projectPath = Join-Path $repoRoot "src\GameCult.Mesh\GameCult.Mesh.csproj" +$webSocketProjectPath = Join-Path $repoRoot "src\GameCult.Networking.WebSockets\GameCult.Networking.WebSockets.csproj" $quicProjectPath = Join-Path $repoRoot "src\GameCult.Mesh.Quic.Native\GameCult.Mesh.Quic.Native.csproj" $templateRoot = Join-Path $repoRoot "unity\org.gamecult.cultlib" $publishRoot = Join-Path $repoRoot "artifacts\unity-publish" +$webSocketPublishRoot = Join-Path $repoRoot "artifacts\unity-websocket-publish" $quicPublishRoot = Join-Path $repoRoot "artifacts\unity-quic-publish" $quicNativeRoot = Join-Path $repoRoot "artifacts\unity-quic-native" $outputRoot = if ([System.IO.Path]::IsPathRooted($OutputDirectory)) { @@ -19,7 +23,55 @@ $outputRoot = if ([System.IO.Path]::IsPathRooted($OutputDirectory)) { Join-Path $repoRoot $OutputDirectory } $pluginRoot = Join-Path $outputRoot "Runtime\Plugins" -$manifest = Get-Content -LiteralPath (Join-Path $templateRoot "package.json") -Raw | ConvertFrom-Json + +if (Test-Path -LiteralPath $publishRoot) { + Remove-Item -LiteralPath $publishRoot -Recurse -Force +} +if (Test-Path -LiteralPath $quicPublishRoot) { + Remove-Item -LiteralPath $quicPublishRoot -Recurse -Force +} +if (Test-Path -LiteralPath $webSocketPublishRoot) { + Remove-Item -LiteralPath $webSocketPublishRoot -Recurse -Force +} +if (Test-Path -LiteralPath $outputRoot) { + Remove-Item -LiteralPath $outputRoot -Recurse -Force +} + +$unityPackageVersion = (Get-Content -LiteralPath (Join-Path $templateRoot "package.json") -Raw | ConvertFrom-Json).version +$publishArguments = @("publish", $projectPath, "-c", $Configuration, "-o", $publishRoot, "-p:CultLibPackageVersion=$unityPackageVersion") +if ($NoRestore) { $publishArguments += "--no-restore" } +if (-not [string]::IsNullOrWhiteSpace($NuGetConfig)) { + $publishArguments += "-p:RestoreConfigFile=$([IO.Path]::GetFullPath($NuGetConfig))" +} +& dotnet @publishArguments +if ($LASTEXITCODE -ne 0) { + throw "CultLib publish failed with exit code $LASTEXITCODE" +} +$webSocketPublishArguments = @("publish", $webSocketProjectPath, "-c", $Configuration, "-f", "netstandard2.1", "-o", $webSocketPublishRoot, "-p:CultLibPackageVersion=$unityPackageVersion") +if ($NoRestore) { $webSocketPublishArguments += "--no-restore" } +if (-not [string]::IsNullOrWhiteSpace($NuGetConfig)) { + $webSocketPublishArguments += "-p:RestoreConfigFile=$([IO.Path]::GetFullPath($NuGetConfig))" +} +& dotnet @webSocketPublishArguments +if ($LASTEXITCODE -ne 0) { + throw "CultLib WebSocket publish failed with exit code $LASTEXITCODE" +} +$quicPublishArguments = @("publish", $quicProjectPath, "-c", $Configuration, "-o", $quicPublishRoot, "-p:CultLibPackageVersion=$unityPackageVersion") +if ($NoRestore) { $quicPublishArguments += "--no-restore" } +if (-not [string]::IsNullOrWhiteSpace($NuGetConfig)) { + $quicPublishArguments += "-p:RestoreConfigFile=$([IO.Path]::GetFullPath($NuGetConfig))" +} +& dotnet @quicPublishArguments +if ($LASTEXITCODE -ne 0) { + throw "CultLib native QUIC managed publish failed with exit code $LASTEXITCODE" +} +& (Join-Path $PSScriptRoot "build-quic-native.ps1") ` + -Configuration $Configuration ` + -OutputDirectory $quicNativeRoot + +New-Item -ItemType Directory -Force -Path $pluginRoot | Out-Null +Copy-Item -LiteralPath (Join-Path $templateRoot "package.json") -Destination $outputRoot +Copy-Item -LiteralPath (Join-Path $templateRoot "README.md") -Destination $outputRoot $expectedAssemblies = @( "ConcurrentCollections.dll", @@ -29,6 +81,7 @@ $expectedAssemblies = @( "GameCult.Mesh.dll", "GameCult.Mesh.Quic.Native.dll", "GameCult.Networking.dll", + "GameCult.Networking.WebSockets.dll", "Isopoh.Cryptography.Argon2.dll", "Isopoh.Cryptography.Blake2b.dll", "Isopoh.Cryptography.SecureArray.dll", @@ -47,80 +100,6 @@ $expectedAssemblies = @( "System.Text.Json.dll", "System.Threading.Channels.dll" ) -$ownedAssemblies = @( - "GameCult.Caching.dll", - "GameCult.Caching.MessagePack.dll", - "GameCult.Logging.dll", - "GameCult.Mesh.dll", - "GameCult.Mesh.Quic.Native.dll", - "GameCult.Networking.dll" -) -$expectedPdbs = $ownedAssemblies | ForEach-Object { [System.IO.Path]::ChangeExtension($_, ".pdb") } - -function Assert-SameNames([string[]] $Actual, [string[]] $Expected, [string] $Description) { - $difference = Compare-Object ($Expected | Sort-Object) ($Actual | Sort-Object) - if ($difference) { - throw "$Description differs from the expected package contract:`n$($difference | Out-String)" - } -} - -function Assert-Meta([string] $Path) { - if (-not (Test-Path -LiteralPath "$Path.meta")) { - throw "Unity metadata is missing for: $Path" - } -} - -if ($manifest.name -ne "org.gamecult.cultlib" -or $manifest.version -notmatch '^\d+\.\d+\.\d+$') { - throw "CultLib package manifest has an invalid name or semantic version." -} - -$templatePluginRoot = Join-Path $templateRoot "Runtime\Plugins" -Assert-SameNames ` - (Get-ChildItem -LiteralPath $templatePluginRoot -Filter "*.dll" | Select-Object -ExpandProperty Name) ` - $expectedAssemblies ` - "Tracked Unity DLL set" -Assert-SameNames ` - (Get-ChildItem -LiteralPath $templatePluginRoot -Filter "*.pdb" | Select-Object -ExpandProperty Name) ` - $expectedPdbs ` - "Tracked Unity PDB set" - -@( - (Join-Path $templateRoot "package.json"), - (Join-Path $templateRoot "README.md"), - (Join-Path $templateRoot "Runtime"), - (Join-Path $templateRoot "Runtime\GameCult.CultLib.asmdef"), - $templatePluginRoot -) + ($expectedAssemblies | ForEach-Object { Join-Path $templatePluginRoot $_ }) + - ($expectedPdbs | ForEach-Object { Join-Path $templatePluginRoot $_ }) | - ForEach-Object { Assert-Meta $_ } - -$asmdef = Get-Content -LiteralPath (Join-Path $templateRoot "Runtime\GameCult.CultLib.asmdef") -Raw | ConvertFrom-Json -Assert-SameNames $asmdef.precompiledReferences $expectedAssemblies "asmdef precompiled reference set" - -if (Test-Path -LiteralPath $publishRoot) { - Remove-Item -LiteralPath $publishRoot -Recurse -Force -} -if (Test-Path -LiteralPath $quicPublishRoot) { - Remove-Item -LiteralPath $quicPublishRoot -Recurse -Force -} -if (Test-Path -LiteralPath $outputRoot) { - Remove-Item -LiteralPath $outputRoot -Recurse -Force -} - -dotnet publish $projectPath -c $Configuration -o $publishRoot --disable-build-servers ` - -p:UseSharedCompilation=false -m:1 -p:Version=$($manifest.version) -if ($LASTEXITCODE -ne 0) { - throw "CultLib publish failed with exit code $LASTEXITCODE" -} -dotnet publish $quicProjectPath -c $Configuration -o $quicPublishRoot --disable-build-servers ` - -p:UseSharedCompilation=false -m:1 -p:Version=$($manifest.version) -if ($LASTEXITCODE -ne 0) { - throw "CultLib native QUIC managed publish failed with exit code $LASTEXITCODE" -} -& (Join-Path $PSScriptRoot "build-quic-native.ps1") ` - -Configuration $Configuration ` - -OutputDirectory $quicNativeRoot - $publishedByName = @{} foreach ($assembly in Get-ChildItem -LiteralPath $publishRoot -Filter "*.dll") { $publishedByName[$assembly.Name] = $assembly @@ -128,85 +107,78 @@ foreach ($assembly in Get-ChildItem -LiteralPath $publishRoot -Filter "*.dll") { foreach ($assembly in Get-ChildItem -LiteralPath $quicPublishRoot -Filter "*.dll") { $publishedByName[$assembly.Name] = $assembly } +foreach ($assembly in Get-ChildItem -LiteralPath $webSocketPublishRoot -Filter "*.dll") { + $publishedByName[$assembly.Name] = $assembly +} foreach ($assemblyName in $expectedAssemblies) { if (-not $publishedByName.ContainsKey($assemblyName)) { throw "CultLib Unity package is missing expected assembly: $assemblyName" } } -Copy-Item -LiteralPath $templateRoot -Destination $outputRoot -Recurse +Copy-Item -LiteralPath (Join-Path $templateRoot "Runtime\GameCult.CultLib.asmdef") ` + -Destination (Join-Path $outputRoot "Runtime") foreach ($assemblyName in $expectedAssemblies) { - Copy-Item -LiteralPath $publishedByName[$assemblyName].FullName -Destination $pluginRoot -Force -} -foreach ($pdbName in $expectedPdbs) { - $pdbPath = @($publishRoot, $quicPublishRoot) | - ForEach-Object { Join-Path $_ $pdbName } | - Where-Object { Test-Path -LiteralPath $_ } | - Select-Object -First 1 - if (-not (Test-Path -LiteralPath $pdbPath)) { - throw "CultLib Unity package is missing expected symbols: $pdbName" + $assembly = $publishedByName[$assemblyName] + Copy-Item -LiteralPath $assembly.FullName -Destination $pluginRoot + $pdb = [System.IO.Path]::ChangeExtension($assembly.FullName, ".pdb") + if (Test-Path -LiteralPath $pdb) { + Copy-Item -LiteralPath $pdb -Destination $pluginRoot } - Copy-Item -LiteralPath $pdbPath -Destination $pluginRoot -Force } - $nativePluginRoot = Join-Path $pluginRoot "x86_64" -Copy-Item -LiteralPath (Join-Path $quicNativeRoot "gamecult_mesh_quic_native.dll") -Destination $nativePluginRoot -Force -Copy-Item -LiteralPath (Join-Path $quicNativeRoot "msquic.dll") -Destination $nativePluginRoot -Force +New-Item -ItemType Directory -Force -Path $nativePluginRoot | Out-Null +Copy-Item -LiteralPath (Join-Path $quicNativeRoot "gamecult_mesh_quic_native.dll") -Destination $nativePluginRoot +Copy-Item -LiteralPath (Join-Path $quicNativeRoot "msquic.dll") -Destination $nativePluginRoot +New-Item -ItemType Directory -Force -Path (Join-Path $outputRoot "Third Party Notices") | Out-Null Copy-Item -LiteralPath (Join-Path $quicNativeRoot "MSQUIC-LICENSE.txt") ` - -Destination (Join-Path $outputRoot "Third Party Notices\MSQUIC-LICENSE.txt") -Force + -Destination (Join-Path $outputRoot "Third Party Notices\MSQUIC-LICENSE.txt") -Assert-SameNames ` - (Get-ChildItem -LiteralPath $pluginRoot -Filter "*.dll" | Select-Object -ExpandProperty Name) ` - $expectedAssemblies ` - "Staged Unity DLL set" -Assert-SameNames ` - (Get-ChildItem -LiteralPath $pluginRoot -Filter "*.pdb" | Select-Object -ExpandProperty Name) ` - $expectedPdbs ` - "Staged Unity PDB set" - -$expectedFileVersion = "$($manifest.version).0" -foreach ($assemblyName in $ownedAssemblies) { - $assemblyPath = Join-Path $pluginRoot $assemblyName - $fileVersion = [System.Diagnostics.FileVersionInfo]::GetVersionInfo($assemblyPath).FileVersion - if ($fileVersion -ne $expectedFileVersion) { - throw "$assemblyName has file version $fileVersion; expected $expectedFileVersion." - } - $freshHash = (Get-FileHash -LiteralPath $publishedByName[$assemblyName].FullName -Algorithm SHA256).Hash - $stagedHash = (Get-FileHash -LiteralPath $assemblyPath -Algorithm SHA256).Hash - if ($freshHash -ne $stagedHash) { - throw "$assemblyName differs between the fresh publish and staged package." +$meshPackagePath = Join-Path $pluginRoot "GameCult.Mesh.dll" +$meshPackageVersion = [Reflection.AssemblyName]::GetAssemblyName($meshPackagePath).Version.ToString() +if ($meshPackageVersion -ne "$unityPackageVersion.0") { + throw "Packaged GameCult.Mesh.dll version '$meshPackageVersion' does not match package '$unityPackageVersion'." +} +$meshPackageMetadata = [Text.Encoding]::UTF8.GetString([IO.File]::ReadAllBytes($meshPackagePath)) +foreach ($requiredType in @( + "GameCult.Mesh.CultMeshAuthorityTrustPolicy", + "GameCult.Mesh.CultMeshRouteCertificate", + "GameCult.Mesh.CultMeshHttpsContentTransportConnector")) { + if (-not $meshPackageMetadata.Contains($requiredType.Split('.')[-1])) { + throw "Packaged GameCult.Mesh.dll is missing required API '$requiredType'." } } - -$resolverAssemblyPath = Join-Path $pluginRoot "GameCult.Caching.MessagePack.dll" -$resolverAssemblyMetadata = [System.Text.Encoding]::UTF8.GetString( - [System.IO.File]::ReadAllBytes($resolverAssemblyPath)) -if (-not $resolverAssemblyMetadata.Contains("CultDocumentMessagePackResolversAttribute")) { - throw "GameCult.Caching.MessagePack.dll lacks CultDocumentMessagePackResolversAttribute." -} -$meshAssemblyMetadata = [System.Text.Encoding]::UTF8.GetString( - [System.IO.File]::ReadAllBytes((Join-Path $pluginRoot "GameCult.Mesh.dll"))) -if (-not $meshAssemblyMetadata.Contains("ICultMeshBodyReadLease")) { - throw "GameCult.Mesh.dll lacks ICultMeshBodyReadLease." +$webSocketPackageMetadata = [Text.Encoding]::UTF8.GetString( + [IO.File]::ReadAllBytes((Join-Path $pluginRoot "GameCult.Networking.WebSockets.dll"))) +if (-not $webSocketPackageMetadata.Contains("CultNetWebSocketSchemaClient")) { + throw "Packaged WebSocket assembly is missing the Unity-compatible CultNet client." } if ($UpdateTemplate) { - foreach ($name in $expectedAssemblies + $expectedPdbs) { - Copy-Item -LiteralPath (Join-Path $pluginRoot $name) -Destination $templatePluginRoot -Force + $templatePluginRoot = Join-Path $templateRoot "Runtime\Plugins" + foreach ($assemblyName in $expectedAssemblies) { + Copy-Item -LiteralPath (Join-Path $pluginRoot $assemblyName) -Destination $templatePluginRoot -Force + $pdbName = [System.IO.Path]::ChangeExtension($assemblyName, ".pdb") + $pdb = Join-Path $pluginRoot $pdbName + if (Test-Path -LiteralPath $pdb) { + Copy-Item -LiteralPath $pdb -Destination $templatePluginRoot -Force + } } + $templateNativePluginRoot = Join-Path $templatePluginRoot "x86_64" + New-Item -ItemType Directory -Force -Path $templateNativePluginRoot | Out-Null Copy-Item -LiteralPath (Join-Path $nativePluginRoot "gamecult_mesh_quic_native.dll") ` - -Destination (Join-Path $templatePluginRoot "x86_64") -Force + -Destination $templateNativePluginRoot -Force Copy-Item -LiteralPath (Join-Path $nativePluginRoot "msquic.dll") ` - -Destination (Join-Path $templatePluginRoot "x86_64") -Force + -Destination $templateNativePluginRoot -Force + $templateNoticesRoot = Join-Path $templateRoot "Third Party Notices" + New-Item -ItemType Directory -Force -Path $templateNoticesRoot | Out-Null Copy-Item -LiteralPath (Join-Path $outputRoot "Third Party Notices\MSQUIC-LICENSE.txt") ` - -Destination (Join-Path $templateRoot "Third Party Notices") -Force + -Destination $templateNoticesRoot -Force Write-Host "Updated tracked Unity package assemblies: $templatePluginRoot" } +$manifest = Get-Content -LiteralPath (Join-Path $outputRoot "package.json") -Raw | ConvertFrom-Json Write-Host "CultLib Unity package: $outputRoot" Write-Host "Package: $($manifest.name)@$($manifest.version)" Write-Host "Managed assemblies: $($expectedAssemblies.Count)" -Write-Host "Verified owned assembly version: $expectedFileVersion" -Write-Host "Verified Stage 1 MessagePack resolver attribute." -Write-Host "Verified ICultMeshBodyReadLease." Write-Host "Native realtime: MsQuic Schannel 2.5.9 (Windows x64)" diff --git a/scripts/measure-reactive-document-performance.mjs b/scripts/measure-reactive-document-performance.mjs new file mode 100644 index 00000000..786cb3ed --- /dev/null +++ b/scripts/measure-reactive-document-performance.mjs @@ -0,0 +1,141 @@ +import { monitorEventLoopDelay, performance } from "node:perf_hooks"; +import process from "node:process"; + +import { CultMesh } from "../packages/cultmesh-ts/dist/index.js"; + +const quick = process.argv.includes("--quick"); +const idleMs = quick ? 250 : 10_000; +const activeMs = quick ? 500 : 10_000; +const results = []; + +for (const documentCount of [1, 100, 1000]) { + results.push(await measure(documentCount)); +} + +console.log(JSON.stringify({ + runtime: "typescript", + workload: { + documentCounts: [1, 100, 1000], + payloadBytes: 16 * 1024, + idleSeconds: idleMs / 1000, + activeSeconds: activeMs / 1000, + updateRateHz: 60, + changedFraction: 0.01, + }, + results, +}, undefined, 2)); + +if (results.some(result => + result.actualPublishes !== result.expectedPublishes || + result.p99PublishLatencyMilliseconds >= 250 || + result.activeHeapPeakGrowthBytes >= 128 * 1024 * 1024)) { + process.exitCode = 1; +} + +async function measure(documentCount) { + const payload = "x".repeat(16 * 1024); + const changedDocumentCount = Math.max(1, Math.floor(documentCount / 100)); + const reactiveDocuments = []; + const publishSignals = new Array(documentCount); + const updateStartedAt = new Float64Array(documentCount); + const latencies = []; + let payloadBytesPublished = 0; + let publishes = 0; + + for (let index = 0; index < documentCount; index++) { + let current = { id: `performance:${index}`, payload, revision: 1 }; + let watcher; + const document = CultMesh.document( + `performance:${index}`, + { schemaId: "gamecult.mesh.performance_probe.v1" }, + async () => current, + { + submitPrediction: async (_context, value) => { + payloadBytesPublished += Buffer.byteLength(JSON.stringify(value)); + latencies.push(performance.now() - updateStartedAt[index]); + publishes++; + current = value; + watcher?.(value); + publishSignals[index]?.(); + }, + watchDocument: (_context, callback) => { + watcher = callback; + return () => { watcher = undefined; }; + }, + }, + ); + const reactive = document.predictionWriter().reactive(); + reactiveDocuments.push(reactive); + } + + await Promise.all(reactiveDocuments.map(document => document.ready)); + collect(); + const idleHeapStart = process.memoryUsage().heapUsed; + const idleCpuStart = process.cpuUsage(); + await delay(idleMs); + collect(); + const idleHeapGrowthBytes = process.memoryUsage().heapUsed - idleHeapStart; + const idleCpu = process.cpuUsage(idleCpuStart); + + collect(); + const activeHeapStart = process.memoryUsage().heapUsed; + let activeHeapPeak = activeHeapStart; + const activeCpuStart = process.cpuUsage(); + const eventLoop = monitorEventLoopDelay({ resolution: 10 }); + eventLoop.enable(); + const activeStartedAt = performance.now(); + let frames = 0; + while (performance.now() - activeStartedAt < activeMs) { + const completions = []; + for (let index = 0; index < changedDocumentCount; index++) { + completions.push(new Promise(resolve => { publishSignals[index] = resolve; })); + updateStartedAt[index] = performance.now(); + reactiveDocuments[index].update(value => { value.revision++; }); + } + await Promise.all(completions); + frames++; + activeHeapPeak = Math.max(activeHeapPeak, process.memoryUsage().heapUsed); + const remaining = frames * (1000 / 60) - (performance.now() - activeStartedAt); + if (remaining > 0) await delay(remaining); + } + eventLoop.disable(); + const activeElapsedSeconds = (performance.now() - activeStartedAt) / 1000; + const activeCpu = process.cpuUsage(activeCpuStart); + const ordered = latencies.toSorted((left, right) => left - right); + + for (const document of reactiveDocuments) document.dispose(); + + return { + documentCount, + changedDocumentCount, + frames, + expectedPublishes: frames * changedDocumentCount, + actualPublishes: publishes, + payloadBytesPublished, + idleHeapGrowthBytes, + idleCpuMilliseconds: (idleCpu.user + idleCpu.system) / 1000, + activeHeapPeakGrowthBytes: activeHeapPeak - activeHeapStart, + activeCpuMilliseconds: (activeCpu.user + activeCpu.system) / 1000, + publishedPayloadBytesPerSecond: payloadBytesPublished / activeElapsedSeconds, + p50PublishLatencyMilliseconds: percentile(ordered, 0.50), + p95PublishLatencyMilliseconds: percentile(ordered, 0.95), + p99PublishLatencyMilliseconds: percentile(ordered, 0.99), + p99EventLoopDelayMilliseconds: eventLoop.percentile(99) / 1_000_000, + }; +} + +function percentile(ordered, ratio) { + if (ordered.length === 0) return 0; + return ordered[Math.max(0, Math.ceil(ordered.length * ratio) - 1)]; +} + +function collect() { + if (typeof globalThis.gc !== "function") { + throw new Error("Run this probe with node --expose-gc."); + } + globalThis.gc(); +} + +function delay(milliseconds) { + return new Promise(resolve => setTimeout(resolve, milliseconds)); +} diff --git a/scripts/measure-reactive-document-performance.ps1 b/scripts/measure-reactive-document-performance.ps1 new file mode 100644 index 00000000..c0f44eaf --- /dev/null +++ b/scripts/measure-reactive-document-performance.ps1 @@ -0,0 +1,88 @@ +param( + [switch] $Quick, + [switch] $SkipRuntimeBuild, + [string] $PythonPath = "", + [string] $NodePath = "" +) + +$ErrorActionPreference = "Stop" +$root = Split-Path -Parent $PSScriptRoot +$nodeExecutable = if ([string]::IsNullOrWhiteSpace($NodePath)) { + (Get-Command node -ErrorAction Stop).Source +} +else { + [IO.Path]::GetFullPath($NodePath) +} +$pythonExecutable = if ([string]::IsNullOrWhiteSpace($PythonPath)) { + (Get-Command python -ErrorAction Stop).Source +} +else { + [IO.Path]::GetFullPath($PythonPath) +} +foreach ($executable in @($nodeExecutable, $pythonExecutable)) { + if (-not (Test-Path -LiteralPath $executable -PathType Leaf)) { + throw "Performance probe runtime not found: $executable" + } +} + +if (-not $SkipRuntimeBuild) { + $typescript = Join-Path $root "node_modules\typescript\bin\tsc" + foreach ($dependency in @("cultcache-ts", "cultnet-ts", "cultmesh-ts")) { + $dependencyRoot = Join-Path $root "packages\$dependency" + if ($dependency -eq "cultnet-ts") { + & $nodeExecutable (Join-Path $dependencyRoot "tools\generate-swarm-contracts.mjs") + if ($LASTEXITCODE -ne 0) { throw "CultNet TypeScript contract generation failed." } + } + & $nodeExecutable $typescript -p (Join-Path $dependencyRoot "tsconfig.json") --pretty false + if ($LASTEXITCODE -ne 0) { throw "TypeScript performance dependency build failed: $dependency" } + } +} + +$project = Join-Path $root "tools\GameCult.Mesh.PerformanceProbe\GameCult.Mesh.PerformanceProbe.csproj" +$buildArguments = @( + "build", + $project, + "--configuration", "Release", + "--verbosity", "quiet", + "/p:NoWarn=1591%3BCS8632" +) +& dotnet @buildArguments +if ($LASTEXITCODE -ne 0) { + throw "CultMesh reactive document performance probe build failed." +} + +$arguments = @("run", "--project", $project, "--configuration", "Release", "--no-build") +if ($Quick) { + $arguments += @("--", "--quick") +} + +& dotnet @arguments +if ($LASTEXITCODE -ne 0) { + throw "CultMesh C# reactive document performance probe failed." +} + +$runtimeArguments = @() +if ($Quick) { $runtimeArguments += "--quick" } + +& $nodeExecutable --expose-gc (Join-Path $root "scripts\measure-reactive-document-performance.mjs") @runtimeArguments +if ($LASTEXITCODE -ne 0) { + throw "CultMesh TypeScript reactive document performance probe failed." +} + +$pythonSource = (@("cultcache-py", "cultnet-py", "cultmesh-py") | ForEach-Object { Join-Path $root "packages\$_\src" }) -join [IO.Path]::PathSeparator +$previousPythonPath = $env:PYTHONPATH +$env:PYTHONPATH = if ([string]::IsNullOrWhiteSpace($previousPythonPath)) { + $pythonSource +} +else { + $pythonSource + [IO.Path]::PathSeparator + $previousPythonPath +} +try { + & $pythonExecutable (Join-Path $root "packages\cultmesh-py\tools\reactive_performance_probe.py") @runtimeArguments + if ($LASTEXITCODE -ne 0) { + throw "CultMesh Python reactive document performance probe failed." + } +} +finally { + $env:PYTHONPATH = $previousPythonPath +} diff --git a/scripts/pack-nuget.ps1 b/scripts/pack-nuget.ps1 index b027c492..95e5cd2f 100644 --- a/scripts/pack-nuget.ps1 +++ b/scripts/pack-nuget.ps1 @@ -1,6 +1,7 @@ param( [string] $Configuration = "Release", [string] $OutputDirectory = "artifacts\nuget", + [string] $PackageVersion = "", [switch] $SkipConsumerSmoke ) @@ -15,9 +16,9 @@ $outputRoot = if ([IO.Path]::IsPathRooted($OutputDirectory)) { $projects = @( "src\GameCult.Logging\GameCult.Logging.csproj", "src\GameCult.Caching\GameCult.Caching.csproj", - "src\GameCult.Caching.MessagePack.Analyzers\GameCult.Caching.MessagePack.Analyzers.csproj", "src\GameCult.Caching.MessagePack\GameCult.Caching.MessagePack.csproj", "src\GameCult.Networking\GameCult.Networking.csproj", + "src\GameCult.Networking.WebSockets\GameCult.Networking.WebSockets.csproj", "src\GameCult.Mesh\GameCult.Mesh.csproj" ) @@ -27,7 +28,14 @@ if (Test-Path -LiteralPath $outputRoot) { New-Item -ItemType Directory -Force -Path $outputRoot | Out-Null foreach ($project in $projects) { - dotnet pack (Join-Path $repoRoot $project) -c $Configuration -o $outputRoot + $packArguments = @( + "pack", (Join-Path $repoRoot $project), "-c", $Configuration, "-o", $outputRoot, + "--nologo", "--verbosity", "quiet", "-p:NoWarn=1591%3BCS8632" + ) + if (-not [string]::IsNullOrWhiteSpace($PackageVersion)) { + $packArguments += "-p:CultLibPackageVersion=$PackageVersion" + } + & dotnet @packArguments if ($LASTEXITCODE -ne 0) { throw "NuGet pack failed for $project with exit code $LASTEXITCODE" } @@ -44,9 +52,9 @@ $version = [regex]::Match($meshPackage.Name, '^GameCult\.Mesh\.(.+)\.nupkg$').Gr $expectedPackages = @( "GameCult.Logging.$version.nupkg", "GameCult.Caching.$version.nupkg", - "GameCult.Caching.MessagePack.Analyzers.$version.nupkg", "GameCult.Caching.MessagePack.$version.nupkg", "GameCult.Networking.$version.nupkg", + "GameCult.Networking.WebSockets.$version.nupkg", "GameCult.Mesh.$version.nupkg" ) foreach ($package in $expectedPackages) { @@ -61,6 +69,7 @@ if (-not $SkipConsumerSmoke) { Remove-Item -LiteralPath $smokeRoot -Recurse -Force } New-Item -ItemType Directory -Force -Path $smokeRoot | Out-Null + $consumerPackages = Join-Path $smokeRoot "packages" $projectDocument = @" @@ -68,6 +77,8 @@ if (-not $SkipConsumerSmoke) { Exe enable $outputRoot;https://api.nuget.org/v3/index.json + $consumerPackages + true @@ -82,7 +93,8 @@ Console.WriteLine(verse.Context.VerseId); "@ [IO.File]::WriteAllText((Join-Path $smokeRoot "Consumer.csproj"), $projectDocument, [Text.UTF8Encoding]::new($false)) [IO.File]::WriteAllText((Join-Path $smokeRoot "Program.cs"), $program, [Text.UTF8Encoding]::new($false)) - dotnet run --project (Join-Path $smokeRoot "Consumer.csproj") -c Release + dotnet run --project (Join-Path $smokeRoot "Consumer.csproj") -c Release ` + --nologo --verbosity quiet if ($LASTEXITCODE -ne 0) { throw "GameCult.Mesh NuGet consumer smoke failed with exit code $LASTEXITCODE" } diff --git a/scripts/run-typescript-workspaces.mjs b/scripts/run-typescript-workspaces.mjs index f2cf74bc..933079da 100644 --- a/scripts/run-typescript-workspaces.mjs +++ b/scripts/run-typescript-workspaces.mjs @@ -6,7 +6,7 @@ if (!command) throw new Error("Usage: node scripts/run-typescript-workspaces.mjs const npmCli = process.env.npm_execpath ?? join(dirname(process.execPath), "node_modules", "npm", "bin", "npm-cli.js"); -const workspaces = ["cultcache-ts", "cultnet-ts", "cultmesh-ts"]; +const workspaces = ["cultcache-ts", "cultnet-ts", "cultmesh-ts", "cultmesh-browser"]; const childEnv = process.platform === "win32" ? { ...process.env, diff --git a/scripts/test-geometry-planetary-tile.ps1 b/scripts/test-geometry-planetary-tile.ps1 deleted file mode 100644 index 59cc859b..00000000 --- a/scripts/test-geometry-planetary-tile.ps1 +++ /dev/null @@ -1,9 +0,0 @@ -$ErrorActionPreference = "Stop" -$root = Split-Path -Parent $PSScriptRoot -$fixture = Join-Path $root ".tools\planetary-tile\fixture.cmpt" -dotnet run --project (Join-Path $root "tools\GameCult.Geometry.PlanetaryTileFixture\GameCult.Geometry.PlanetaryTileFixture.csproj") -- $fixture -if ($LASTEXITCODE -ne 0) { throw "Planetary tile fixture generation failed." } -node --no-warnings --experimental-strip-types (Join-Path $root "packages\gamecult-geometry-ts\test\planetary-tile.cross-runtime.test.mjs") $fixture -if ($LASTEXITCODE -ne 0) { throw "Planetary tile web decoding failed." } -node --no-warnings --experimental-strip-types (Join-Path $root "packages\gamecult-geometry-ts\test\planetary-tile.test.mjs") -if ($LASTEXITCODE -ne 0) { throw "Planetary tile decoder unit test failed." } diff --git a/scripts/test-typescript-package-closure.mjs b/scripts/test-typescript-package-closure.mjs index c56a13a6..5f00aa47 100644 --- a/scripts/test-typescript-package-closure.mjs +++ b/scripts/test-typescript-package-closure.mjs @@ -6,7 +6,7 @@ import { dirname, join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), ".."); -const workspacePackages = ["cultcache-ts", "cultnet-ts", "cultmesh-ts"]; +const workspacePackages = ["cultcache-ts", "cultnet-ts", "cultmesh-ts", "cultmesh-browser"]; const npmCli = process.env.npm_execpath ?? join(dirname(process.execPath), "node_modules", "npm", "bin", "npm-cli.js"); const childEnv = process.platform === "win32" @@ -58,7 +58,7 @@ try { writeFileSync(join(consumerRoot, "runtime-smoke.cjs"), ` const assert = require("node:assert/strict"); -const cache = require("cultcache-ts"); +const cache = require("@gamecult/cultcache-ts"); const net = require("cultnet-ts"); const mesh = require("cultmesh-ts"); assert.equal(typeof cache.defineDocumentType, "function"); @@ -69,8 +69,19 @@ assert.deepEqual(mesh.CultMesh.vec2(3, 4), { x: 3, y: 4 }); `); execFileSync(process.execPath, ["runtime-smoke.cjs"], { cwd: consumerRoot, stdio: "inherit" }); + writeFileSync(join(consumerRoot, "browser-runtime-smoke.mjs"), ` +import assert from "node:assert/strict"; +import { CultMeshBrowserClient, decodeCultNetPayload } from "cultmesh-browser"; +import { parseCultNetMessage } from "cultnet-ts/contracts"; +assert.equal(typeof CultMeshBrowserClient.connect, "function"); +assert.equal(typeof decodeCultNetPayload, "function"); +assert.equal(typeof parseCultNetMessage, "function"); +assert.equal("putDocument" in CultMeshBrowserClient.prototype, false); +`); + execFileSync(process.execPath, ["browser-runtime-smoke.mjs"], { cwd: consumerRoot, stdio: "inherit" }); + writeFileSync(join(consumerRoot, "types-smoke.ts"), ` -import { defineDocumentType } from "cultcache-ts"; +import { defineDocumentType } from "@gamecult/cultcache-ts"; import { CultNetPeer } from "cultnet-ts"; import { CultMesh, @@ -111,6 +122,37 @@ void providerTransport; "types-smoke.ts", ], { cwd: consumerRoot, stdio: "inherit" }); + writeFileSync(join(consumerRoot, "browser-types-smoke.mts"), ` +import { + CultMeshBrowserClient, + type CultMeshBrowserIdentity, + type CultMeshRawDocumentLeaseOptions, +} from "cultmesh-browser"; + +const identity: CultMeshBrowserIdentity = { + verseId: "smoke", + authorityRuntimeId: "smoke.authority", +}; +const lease: CultMeshRawDocumentLeaseOptions = { + schemaId: "smoke.document.v1", + recordKey: "smoke:main", +}; +const clientType: typeof CultMeshBrowserClient = CultMeshBrowserClient; +void identity; +void lease; +void clientType; +`); + execFileSync(process.execPath, [ + tsc, + "--noEmit", + "--strict", + "--target", "ES2022", + "--module", "NodeNext", + "--moduleResolution", "NodeNext", + "--skipLibCheck", + "browser-types-smoke.mts", + ], { cwd: consumerRoot, stdio: "inherit" }); + console.log("TypeScript package closure smoke: passed"); for (const tarball of tarballs) console.log(`Packed: ${tarball}`); } finally { diff --git a/scripts/verify-eve-browser-network.mjs b/scripts/verify-eve-browser-network.mjs new file mode 100644 index 00000000..c1388712 --- /dev/null +++ b/scripts/verify-eve-browser-network.mjs @@ -0,0 +1,339 @@ +import assert from "node:assert/strict"; +import { spawn } from "node:child_process"; +import { existsSync } from "node:fs"; +import { createServer as createHttpServer } from "node:http"; +import { mkdtemp, readFile, rm } from "node:fs/promises"; +import { createServer as createNetServer } from "node:net"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +import { build } from "esbuild"; +import { chromium } from "playwright-core"; + +const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), ".."); +const eveFlag = process.argv.indexOf("--eve-root"); +const eveRoot = resolve(eveFlag >= 0 ? process.argv[eveFlag + 1] : join(repoRoot, "..", "Eve")); +const packageFeedFlag = process.argv.indexOf("--package-feed"); +const packageFeed = packageFeedFlag >= 0 ? resolve(process.argv[packageFeedFlag + 1]) : undefined; +const cultLibVersionFlag = process.argv.indexOf("--cultlib-package-version"); +const cultLibPackageVersion = cultLibVersionFlag >= 0 ? process.argv[cultLibVersionFlag + 1] : undefined; +const eveSurfaceVersionFlag = process.argv.indexOf("--eve-surface-package-version"); +const eveSurfacePackageVersion = eveSurfaceVersionFlag >= 0 ? process.argv[eveSurfaceVersionFlag + 1] : undefined; +const nugetConfigFlag = process.argv.indexOf("--nuget-config"); +const nugetConfig = nugetConfigFlag >= 0 ? resolve(process.argv[nugetConfigFlag + 1]) : undefined; +const sampleRoot = join(repoRoot, "samples", "eve-browser-network"); +const workRoot = await mkdtemp(join(tmpdir(), "cultmesh-browser-network-")); +const statePath = join(workRoot, "counter.cc"); +const decoyStatePath = join(workRoot, "decoy-counter.cc"); +const bundlePath = join(workRoot, "bundle.js"); +const token = "sample-session"; +let provider; +let decoyProvider; +let replacementProvider; +let odin; +let headless; +let httpServer; +let browser; + +try { + const tsc = join(repoRoot, "node_modules", "typescript", "bin", "tsc"); + await run(process.execPath, [tsc, "-p", join(repoRoot, "packages", "cultnet-ts", "tsconfig.json"), "--pretty", "false"]); + await run(process.execPath, [tsc, "-p", join(repoRoot, "packages", "cultmesh-browser", "tsconfig.json"), "--pretty", "false"]); + await build({ + entryPoints: [join(sampleRoot, "browser", "main.ts")], + outfile: bundlePath, + bundle: true, + format: "esm", + platform: "browser", + target: "es2022", + sourcemap: true, + alias: { + "@gamecult/eve-contracts": join(eveRoot, "packages", "eve-contracts", "src", "index.ts"), + "@gamecult/eve-browser-lowering": join(eveRoot, "packages", "eve-browser-lowering", "src", "index.ts"), + "cultmesh-browser": join(repoRoot, "packages", "cultmesh-browser", "src", "index.ts"), + "cultnet-ts/contracts": join(repoRoot, "packages", "cultnet-ts", "src", "contracts.ts"), + }, + logLevel: "warning", + }); + const bundle = await readFile(bundlePath, "utf8"); + assert.doesNotMatch(bundle, /(?:from\s*["']node:|require\(["']node:)/u, "browser bundle contains a Node builtin import"); + + const dotnetBuildArguments = [ + "build", + join(sampleRoot, "EveBrowserNetworkSample.csproj"), + "-m:1", + "--verbosity", "quiet", + "-p:NoWarn=1591%3BCS8632", + "-p:NuGetAudit=false", + "-p:RestoreIgnoreFailedSources=true", + ]; + if (nugetConfig) dotnetBuildArguments.push(`-p:RestoreConfigFile=${nugetConfig}`); + if (packageFeed) { + if (!cultLibPackageVersion || !eveSurfacePackageVersion) { + throw new Error("Package artifact builds require both CultLib and Eve surface package versions."); + } + dotnetBuildArguments.push( + "-p:UsePackageArtifacts=true", + `-p:PackageFeed=${packageFeed}`, + `-p:CultLibPackageVersion=${cultLibPackageVersion}`, + `-p:EveSurfacePackageVersion=${eveSurfacePackageVersion}`, + ); + } else { + dotnetBuildArguments.push(`-p:EveRoot=${eveRoot}`, `-p:CultLibRoot=${repoRoot}`); + } + await run("dotnet", dotnetBuildArguments); + const providerPort = await freePort(); + const decoyProviderPort = await freePort(); + const replacementProviderPort = await freePort(); + const odinPort = await freePort(); + const httpPort = await freePort(); + provider = await startProvider(providerPort); + const endpoint = await provider.waitFor("PROVIDER_READY "); + decoyProvider = await startProvider(decoyProviderPort, { + authorityRuntimeId: "sample.decoy-daemon", + routeGeneration: "sample-decoy-route-1", + statePath: decoyStatePath, + }); + const decoyEndpoint = await decoyProvider.waitFor("PROVIDER_READY "); + odin = await startOdin(odinPort, endpoint, decoyEndpoint); + const odinEndpoint = await odin.waitFor("ODIN_READY "); + headless = startDotnet([ + "headless", "--odin", odinEndpoint, "--verse-id", "sample.counter", + "--expected-count", "2", "--token", token, + ]); + await headless.waitFor("HEADLESS_READY "); + httpServer = await serve(httpPort, sampleRoot, bundlePath); + + browser = await chromium.launch({ executablePath: resolveChromiumExecutable(), headless: true }); + let page = await browser.newPage(); + // A command the lowering refuses is caught inside the page; without this + // the only symptom is a 30 s timeout on the receipt. + page.on("console", message => console.log(`[browser:${message.type()}] ${message.text()}`)); + page.on("pageerror", error => console.log(`[browser:pageerror] ${error.message}`)); + const url = `http://127.0.0.1:${httpPort}/?odin=${encodeURIComponent(odinEndpoint)}&token=${token}`; + await page.goto(url); + await page.waitForFunction(() => window.__sampleReady || window.__sampleError); + assert.equal(await page.evaluate(() => window.__sampleError), undefined); + assert.equal(await page.evaluate(() => window.__sampleCount), 0); + await page.locator("button").click(); + await page.waitForFunction(() => window.__sampleReceipt?.count === 1 && window.__sampleCount === 1); + const firstReceipt = await page.evaluate(() => window.__sampleReceipt?.receiptId); + const firstHeadlessUpdate = JSON.parse(await headless.waitFor("HEADLESS_UPDATE_1 ")); + assert.equal(firstHeadlessUpdate.count, 1); + assert.ok(firstHeadlessUpdate.receiptIds.includes(firstReceipt)); + await page.locator("button").click(); + await page.waitForFunction(receipt => window.__sampleReceipt?.receiptId === receipt, firstReceipt); + assert.equal(await page.evaluate(() => window.__sampleCount), 1, "duplicate idempotency key changed canonical state"); + + replacementProvider = await startProvider(replacementProviderPort); + const restartedEndpoint = await replacementProvider.waitFor("PROVIDER_READY "); + assert.notEqual(restartedEndpoint, endpoint); + await stop(odin.process); + odin = await startOdin(odinPort, restartedEndpoint, decoyEndpoint); + assert.equal(await odin.waitFor("ODIN_READY "), odinEndpoint); + await stop(provider.process); + provider = replacementProvider; + replacementProvider = undefined; + await page.waitForFunction(() => + window.__sampleConnectionStates?.includes("reconnecting") + && window.__sampleConnectionStates.at(-1) === "connected"); + headless.sendLine("INVOKE headless-command-2"); + const headlessReceipt = JSON.parse(await headless.waitFor("HEADLESS_RECEIPT ")); + assert.equal(headlessReceipt.status, "accepted"); + assert.equal(headlessReceipt.count, 2); + await page.waitForFunction(() => window.__sampleCount === 2); + const secondReceipt = headlessReceipt.receiptId; + const secondHeadlessUpdate = JSON.parse(await headless.waitFor("HEADLESS_UPDATE_2 ")); + assert.equal(secondHeadlessUpdate.count, 2); + assert.ok(secondHeadlessUpdate.receiptIds.includes(firstReceipt)); + assert.ok(secondHeadlessUpdate.receiptIds.includes(secondReceipt)); + const networkBenchmark = JSON.parse(await headless.waitFor("HEADLESS_NETWORK_BENCHMARK ", 60_000)); + assert.equal(networkBenchmark.operations, 10_000); + assert.ok(networkBenchmark.p99Milliseconds < 250); + assert.ok(networkBenchmark.managedHeapGrowth <= 8 * 1024 * 1024); + const connectionStates = await page.evaluate(() => window.__sampleConnectionStates); + assert.equal(connectionStates[0], "connected"); + assert.equal(connectionStates.at(-1), "connected"); + assert.ok(connectionStates.includes("reconnecting")); + assert.ok(!connectionStates.includes("disposed")); + + console.log(JSON.stringify({ + providerEndpoint: endpoint, + replacementProviderEndpoint: restartedEndpoint, + rejectedWrongAuthorityEndpoint: decoyEndpoint, + odinEndpoint, + browserRuntime: "chromium", + headlessRuntime: "csharp", + receiptId: firstReceipt, + secondReceiptId: secondReceipt, + count: 2, + routeRotationCount: 1, + retainedHeadlessLease: true, + networkBenchmark, + })); +} finally { + if (browser) await browser.close().catch(() => undefined); + if (headless) await stop(headless.process); + if (provider) await stop(provider.process); + if (decoyProvider) await stop(decoyProvider.process); + if (replacementProvider) await stop(replacementProvider.process); + if (odin) await stop(odin.process); + if (httpServer) await new Promise(resolve => httpServer.close(resolve)); + await rm(workRoot, { recursive: true, force: true }); +} + +async function startProvider(port, options = {}) { + return startDotnet([ + "provider", "--port", String(port), + "--state", options.statePath ?? statePath, + "--authority-runtime-id", options.authorityRuntimeId ?? "sample.counter-daemon", + "--route-generation", options.routeGeneration ?? "sample-counter-route-1", + "--token", token, + ]); +} + +async function startOdin(port, providerEndpoint, decoyEndpoint) { + return startDotnet([ + "odin", + "--port", String(port), + "--provider-endpoint", providerEndpoint, + "--decoy-endpoint", decoyEndpoint, + "--decoy-authority-runtime-id", "sample.decoy-daemon", + "--decoy-route-generation", "sample-decoy-route-1", + "--token", token, + ]); +} + +function startDotnet(arguments_) { + const process = spawn("dotnet", [ + "run", "--no-build", "--project", join(sampleRoot, "EveBrowserNetworkSample.csproj"), "--", ...arguments_, + ], { cwd: repoRoot, stdio: ["pipe", "pipe", "pipe"] }); + let output = ""; + let error = ""; + const waiters = []; + process.stdout.setEncoding("utf8"); + process.stderr.setEncoding("utf8"); + process.stdout.on("data", chunk => { output += chunk; settle(); }); + process.stderr.on("data", chunk => { error += chunk; }); + process.on("exit", code => { + for (const waiter of waiters.splice(0)) { + clearTimeout(waiter.timer); + waiter.reject(new Error(`dotnet sample exited ${code}\n${output}\n${error}`)); + } + }); + function settle() { + for (let index = waiters.length - 1; index >= 0; index--) { + const waiter = waiters[index]; + const line = output.split(/\r?\n/u).find(value => value.startsWith(waiter.prefix)); + if (!line) continue; + waiters.splice(index, 1); + clearTimeout(waiter.timer); + waiter.resolve(line.slice(waiter.prefix.length)); + } + } + return { + process, + sendLine(line) { + process.stdin.write(`${line}\n`); + }, + waitFor(prefix, timeoutMs = 30_000) { + const existing = output.split(/\r?\n/u).find(value => value.startsWith(prefix)); + if (existing) return Promise.resolve(existing.slice(prefix.length)); + return new Promise((resolve, reject) => { + const waiter = { + prefix, + resolve, + reject, + timer: setTimeout(() => { + const index = waiters.indexOf(waiter); + if (index >= 0) waiters.splice(index, 1); + reject(new Error(`Timed out waiting for '${prefix}'.\n${output}\n${error}`)); + }, timeoutMs), + }; + waiters.push(waiter); + }); + }, + }; +} + +function run(command, arguments_) { + return new Promise((resolvePromise, reject) => { + const process = spawn(command, arguments_, { cwd: repoRoot, stdio: "inherit" }); + process.on("error", reject); + process.on("exit", code => code === 0 ? resolvePromise() : reject(new Error(`${command} exited ${code}`))); + }); +} + +async function serve(port, root, builtBundle) { + const index = await readFile(join(root, "browser", "index.html")); + const server = createHttpServer(async (request, response) => { + if (request.url?.startsWith("/bundle.js")) { + response.setHeader("Content-Type", "text/javascript; charset=utf-8"); + response.end(await readFile(builtBundle)); + return; + } + response.setHeader("Content-Type", "text/html; charset=utf-8"); + response.end(index); + }); + await new Promise((resolvePromise, reject) => { + server.once("error", reject); + server.listen(port, "127.0.0.1", resolvePromise); + }); + return server; +} + +async function freePort() { + const server = createNetServer(); + await new Promise((resolvePromise, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", resolvePromise); + }); + const address = server.address(); + const port = typeof address === "object" && address ? address.port : 0; + await new Promise(resolvePromise => server.close(resolvePromise)); + return port; +} + +async function stop(process) { + if (!process || process.exitCode !== null) return; + process.kill(); + await Promise.race([ + new Promise(resolvePromise => process.once("exit", resolvePromise)), + new Promise(resolvePromise => setTimeout(resolvePromise, 2_000)), + ]); + if (process.exitCode === null) process.kill("SIGKILL"); +} + +function resolveChromiumExecutable() { + const candidates = [process.env.CHROME_PATH]; + if (process.platform === "win32") { + for (const root of [process.env.ProgramFiles, process.env["ProgramFiles(x86)"], process.env.LOCALAPPDATA]) { + if (!root) continue; + candidates.push( + join(root, "Google", "Chrome", "Application", "chrome.exe"), + join(root, "Microsoft", "Edge", "Application", "msedge.exe"), + ); + } + } else if (process.platform === "darwin") { + candidates.push( + "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome", + "/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge", + ); + } else { + candidates.push( + "/usr/bin/google-chrome", + "/usr/bin/google-chrome-stable", + "/usr/bin/chromium", + "/usr/bin/chromium-browser", + "/opt/google/chrome/chrome", + ); + } + candidates.push(chromium.executablePath()); + const executablePath = candidates.find(candidate => candidate && existsSync(candidate)); + if (executablePath) return executablePath; + throw new Error( + "No Chromium-family browser was found. Set CHROME_PATH or run " + + "'node node_modules/playwright-core/cli.js install chromium'.", + ); +} diff --git a/scripts/verify-eve-getting-started.ps1 b/scripts/verify-eve-getting-started.ps1 new file mode 100644 index 00000000..4a87b19d --- /dev/null +++ b/scripts/verify-eve-getting-started.ps1 @@ -0,0 +1,86 @@ +param( + [string] $EveRoot = (Join-Path (Split-Path -Parent $PSScriptRoot) "..\Eve"), + [switch] $SkipDependencyInstall +) + +$ErrorActionPreference = "Stop" +$cultLibRoot = Split-Path -Parent $PSScriptRoot +$EveRoot = [IO.Path]::GetFullPath($EveRoot) +$networkSample = Join-Path $cultLibRoot "samples\eve-browser-network\Program.cs" +$networkSampleSource = Get-Content -LiteralPath $networkSample -Raw +if ($networkSampleSource -match 'OnCultNet\s*<\s*CultNetOperationRequestMessage' -or + $networkSampleSource -match 'Convert\.FromBase64String\s*\(\s*request\.Payload') { + throw "The getting-started provider has regressed to hand-written CultNet operation envelope dispatch." +} +foreach ($requiredPrimitive in @("CultNetOperationServer", "EveSurface.Create")) { + if ($networkSampleSource -notmatch [regex]::Escape($requiredPrimitive)) { + throw "The getting-started provider is missing the public primitive '$requiredPrimitive'." + } +} +if ($networkSampleSource -notmatch 'new\s+CultMeshSessionTarget\s*\(') { + throw "The getting-started client must address one explicit Verse/authority-runtime target." +} +$cultMeshClientSource = Get-Content -LiteralPath (Join-Path $cultLibRoot "src\GameCult.Mesh\CultMeshClient.cs") -Raw +if ($cultMeshClientSource -match 'ConnectAsync\s*\(\s*string\s+(endpointId|verseId)' -or + $cultMeshClientSource -match 'Lease(Document|Collection)Async[^\(]*\(\s*string\s+(endpointId|verseId)') { + throw "CultMeshClient must not expose an ambiguous one-string session identity." +} +$dotnetPackageRoot = Join-Path ([IO.Path]::GetTempPath()) ` + ("eve-getting-started-nuget-" + [guid]::NewGuid().ToString("N")) +$sourceRevision = (& git -C $cultLibRoot rev-parse --short=12 HEAD).Trim() +if ($LASTEXITCODE -ne 0 -or [string]::IsNullOrWhiteSpace($sourceRevision)) { + throw "Could not determine the CultLib source revision for local package identity." +} +$verificationNonce = [guid]::NewGuid().ToString("N").Substring(0, 8) +$cultLibLocalVersion = "1.0.35-local.$sourceRevision.$verificationNonce" +$surfaceLocalVersion = "0.3.3-local.$sourceRevision.$verificationNonce" +try { + & (Join-Path $EveRoot "scripts\pack-dotnet-surface.ps1") ` + -CultLibRoot $cultLibRoot ` + -OutputDirectory $dotnetPackageRoot ` + -CultLibPackageVersion $cultLibLocalVersion ` + -SurfacePackageVersion $surfaceLocalVersion + if ($LASTEXITCODE -ne 0) { + throw "The clean .NET package-consumer checkpoint failed." + } + + $cultLibPackage = Get-ChildItem -LiteralPath $dotnetPackageRoot -Filter "GameCult.Mesh.*.nupkg" | + Where-Object { $_.Name -notlike "*.snupkg" } | + Select-Object -First 1 + $surfacePackage = Get-ChildItem -LiteralPath $dotnetPackageRoot -Filter "GameCult.Eve.Surface.*.nupkg" | + Where-Object { $_.Name -notlike "*.snupkg" } | + Select-Object -First 1 + if (-not $cultLibPackage -or -not $surfacePackage) { + throw "The local package feed is missing CultMesh or Eve surface artifacts." + } + $cultLibVersion = [regex]::Match($cultLibPackage.Name, '^GameCult\.Mesh\.(.+)\.nupkg$').Groups[1].Value + $surfaceVersion = [regex]::Match($surfacePackage.Name, '^GameCult\.Eve\.Surface\.(.+)\.nupkg$').Groups[1].Value + + & (Join-Path $PSScriptRoot "verify-eve-two-runtime-sample.ps1") ` + -EveRoot $EveRoot ` + -SkipDependencyInstall:$SkipDependencyInstall + if ($LASTEXITCODE -ne 0) { + throw "The clean package-consumer checkpoint failed." + } + + $nodeCommand = Get-Command node -ErrorAction Stop + $playwrightCli = Join-Path $cultLibRoot "node_modules\playwright-core\cli.js" + if (-not (Test-Path -LiteralPath $playwrightCli -PathType Leaf)) { + throw "playwright-core is missing after dependency setup: $playwrightCli" + } + + & $nodeCommand.Source (Join-Path $PSScriptRoot "verify-eve-browser-network.mjs") ` + --eve-root $EveRoot ` + --package-feed $dotnetPackageRoot ` + --cultlib-package-version $cultLibVersion ` + --eve-surface-package-version $surfaceVersion + if ($LASTEXITCODE -ne 0) { + throw "The real browser/C# network checkpoint failed." + } + + Write-Host "Eve getting-started verification passed: clean .NET and TypeScript artifacts, DOM and headless convergence, real Chromium and C# clients, Odin discovery, authority-bound routes, receipts, persistence, and route replacement." +} finally { + if (Test-Path -LiteralPath $dotnetPackageRoot) { + Remove-Item -LiteralPath $dotnetPackageRoot -Recurse -Force + } +} diff --git a/scripts/verify-eve-two-runtime-sample.ps1 b/scripts/verify-eve-two-runtime-sample.ps1 new file mode 100644 index 00000000..e33a0622 --- /dev/null +++ b/scripts/verify-eve-two-runtime-sample.ps1 @@ -0,0 +1,149 @@ +param( + [string] $EveRoot = (Join-Path (Split-Path -Parent $PSScriptRoot) "..\Eve"), + [switch] $SkipDependencyInstall +) + +$ErrorActionPreference = "Stop" + +$cultLibRoot = Split-Path -Parent $PSScriptRoot +$samplePath = Join-Path $cultLibRoot "samples\eve-two-runtime\sample.mjs" +$eveBrowserPackage = Join-Path $EveRoot "packages\eve-browser-lowering" +$eveContractsPackage = Join-Path $EveRoot "packages\eve-contracts" +$packageRoots = @( + (Join-Path $cultLibRoot "packages\cultcache-ts"), + (Join-Path $cultLibRoot "packages\cultnet-ts"), + (Join-Path $cultLibRoot "packages\cultmesh-ts"), + $eveContractsPackage, + $eveBrowserPackage +) +$onWindows = [System.Environment]::OSVersion.Platform -eq [System.PlatformID]::Win32NT + +$nodeCommand = Get-Command node -ErrorAction SilentlyContinue +if (-not $nodeCommand) { + $standardNode = if ($onWindows -and $env:ProgramFiles) { + Join-Path $env:ProgramFiles "nodejs\node.exe" + } + else { + "" + } + if (Test-Path -LiteralPath $standardNode) { + $nodeCommand = Get-Item -LiteralPath $standardNode + } +} +if (-not $nodeCommand) { + throw "Node.js is required to verify the Eve two-runtime sample." +} +$nodePath = $nodeCommand.Source +$nodeDirectory = Split-Path -Parent $nodePath +$npmPath = if ($onWindows) { + Join-Path $nodeDirectory "npm.cmd" +} +else { + (Get-Command npm -ErrorAction Stop).Source +} +if (-not (Test-Path -LiteralPath $npmPath)) { + throw "npm was not found beside Node.js: $npmPath" +} +$env:PATH = $nodeDirectory + [IO.Path]::PathSeparator + $env:PATH +if (-not (Test-Path -LiteralPath $samplePath)) { + throw "Sample not found: $samplePath" +} +foreach ($packageRoot in $packageRoots) { + if (-not (Test-Path -LiteralPath (Join-Path $packageRoot "package.json"))) { + throw "Package root not found: $packageRoot" + } +} + +if (-not $SkipDependencyInstall) { + $dependencyRoots = @($cultLibRoot, $eveContractsPackage, $eveBrowserPackage) + foreach ($dependencyRoot in $dependencyRoots) { + $typescript = Join-Path $dependencyRoot "node_modules\typescript\bin\tsc" + if (Test-Path -LiteralPath $typescript -PathType Leaf) { + continue + } + Push-Location $dependencyRoot + try { + & $npmPath ci --no-audit --no-fund + if ($LASTEXITCODE -ne 0) { + throw "Dependency installation failed: $dependencyRoot" + } + } + finally { + Pop-Location + } + } +} + +$tempBase = [IO.Path]::GetFullPath([IO.Path]::GetTempPath()) +$workRoot = Join-Path $tempBase ("gamecult-eve-two-runtime-" + [guid]::NewGuid().ToString("N")) +$artifactRoot = Join-Path $workRoot "artifacts" +$consumerRoot = Join-Path $workRoot "consumer" +New-Item -ItemType Directory -Path $artifactRoot, $consumerRoot -Force | Out-Null + +try { + foreach ($packageRoot in $packageRoots) { + Push-Location $packageRoot + try { + if ((Split-Path -Leaf $packageRoot) -eq "cultnet-ts") { + & $nodePath (Join-Path $packageRoot "tools\generate-swarm-contracts.mjs") + if ($LASTEXITCODE -ne 0) { + throw "CultNet TypeScript contract generation failed: $packageRoot" + } + } + $typescriptPath = if ($packageRoot -eq $eveBrowserPackage -or $packageRoot -eq $eveContractsPackage) { + Join-Path $packageRoot "node_modules\typescript\bin\tsc" + } + else { + Join-Path $cultLibRoot "node_modules\typescript\bin\tsc" + } + & $nodePath $typescriptPath -p (Join-Path $packageRoot "tsconfig.json") --pretty false + if ($LASTEXITCODE -ne 0) { + throw "Package build failed: $packageRoot" + } + & $npmPath pack --ignore-scripts --silent --pack-destination $artifactRoot | Out-Null + if ($LASTEXITCODE -ne 0) { + throw "Package pack failed: $packageRoot" + } + } + finally { + Pop-Location + } + } + + Push-Location $consumerRoot + try { + & $npmPath init --yes --silent | Out-Null + $tarballs = Get-ChildItem -LiteralPath $artifactRoot -Filter "*.tgz" | + Sort-Object Name | + ForEach-Object FullName + $installArguments = @( + "install", + "--silent", + "--no-audit", + "--no-fund", + "@msgpack/msgpack@3", + "ajv@8", + "jsdom@30", + "zod@3" + ) + @($tarballs) + & $npmPath @installArguments + if ($LASTEXITCODE -ne 0) { + throw "Clean consumer package installation failed." + } + Copy-Item -LiteralPath $samplePath -Destination (Join-Path $consumerRoot "sample.mjs") + & $nodePath (Join-Path $consumerRoot "sample.mjs") + if ($LASTEXITCODE -ne 0) { + throw "Eve two-runtime sample failed." + } + } + finally { + Pop-Location + } +} +finally { + $resolvedWorkRoot = [IO.Path]::GetFullPath($workRoot) + if ($resolvedWorkRoot.StartsWith($tempBase, [StringComparison]::OrdinalIgnoreCase) -and + (Test-Path -LiteralPath $resolvedWorkRoot)) { + Remove-Item -LiteralPath $resolvedWorkRoot -Recurse -Force + } +} diff --git a/scripts/verify-reactive-document-scaling.ps1 b/scripts/verify-reactive-document-scaling.ps1 new file mode 100644 index 00000000..523b761f --- /dev/null +++ b/scripts/verify-reactive-document-scaling.ps1 @@ -0,0 +1,105 @@ +param( + [string] $PythonPath = "", + [string] $NodePath = "" +) + +$ErrorActionPreference = "Stop" + +$root = Split-Path -Parent $PSScriptRoot +$dotnet = Get-Command dotnet -ErrorAction Stop +$nodeExecutable = if ([string]::IsNullOrWhiteSpace($NodePath)) { + $node = Get-Command node -ErrorAction SilentlyContinue + if ($node) { $node.Source } else { Join-Path $env:ProgramFiles "nodejs\node.exe" } +} +else { + [IO.Path]::GetFullPath($NodePath) +} +if (-not (Test-Path -LiteralPath $nodeExecutable -PathType Leaf)) { + throw "Node.js executable not found: $nodeExecutable" +} +$pythonExecutable = if ([string]::IsNullOrWhiteSpace($PythonPath)) { + (Get-Command python -ErrorAction Stop).Source +} +else { + [IO.Path]::GetFullPath($PythonPath) +} +if (-not (Test-Path -LiteralPath $pythonExecutable)) { + throw "Python executable not found: $pythonExecutable" +} +$typescript = Join-Path $root "node_modules\typescript\bin\tsc" +if (-not (Test-Path -LiteralPath $typescript)) { + throw "TypeScript compiler not found: $typescript" +} + +foreach ($dependency in @("cultcache-ts", "cultnet-ts")) { + $dependencyRoot = Join-Path $root "packages\$dependency" + Push-Location $dependencyRoot + try { + if ($dependency -eq "cultnet-ts") { + & $nodeExecutable (Join-Path $dependencyRoot "tools\generate-swarm-contracts.mjs") + if ($LASTEXITCODE -ne 0) { + throw "CultNet TypeScript contract generation failed." + } + } + & $nodeExecutable $typescript -p tsconfig.json --pretty false + if ($LASTEXITCODE -ne 0) { + throw "TypeScript dependency build failed: $dependency" + } + } + finally { + Pop-Location + } +} + +& $dotnet.Source test (Join-Path $root "tests\GameCult.Mesh.Tests\GameCult.Mesh.Tests.csproj") ` + --filter "FullyQualifiedName~ReactiveDocument_SchedulingScalesWithChangedDocumentsOnly" ` + --nologo ` + --verbosity minimal +if ($LASTEXITCODE -ne 0) { + throw "CultMesh C# reactive scaling verification failed." +} + +Push-Location (Join-Path $root "packages\cultmesh-ts") +try { + & $nodeExecutable $typescript -p tsconfig.json --pretty false + if ($LASTEXITCODE -ne 0) { + throw "CultMesh TypeScript source build failed." + } + & $nodeExecutable $typescript -p tsconfig.test.json --pretty false + if ($LASTEXITCODE -ne 0) { + throw "CultMesh TypeScript test build failed." + } + $testFiles = Get-ChildItem -LiteralPath .\dist-test\test -Filter "*.test.js" | + Sort-Object Name | + ForEach-Object FullName + & $nodeExecutable --test @testFiles + if ($LASTEXITCODE -ne 0) { + throw "CultMesh TypeScript reactive scaling verification failed." + } +} +finally { + Pop-Location +} + +$pythonPath = (@("cultcache-py", "cultnet-py", "cultmesh-py") | ForEach-Object { Join-Path $root "packages\$_\src" }) -join [IO.Path]::PathSeparator +$previousPythonPath = $env:PYTHONPATH +$env:PYTHONPATH = if ([string]::IsNullOrWhiteSpace($previousPythonPath)) { + $pythonPath +} +else { + $pythonPath + [IO.Path]::PathSeparator + $previousPythonPath +} +try { + & $pythonExecutable -m unittest discover ` + -s (Join-Path $root "packages\cultmesh-py\tests") ` + -p "test_cultmesh.py" ` + -k "reactive_scheduling_scales_with_changed_documents_only" + if ($LASTEXITCODE -ne 0) { + throw "CultMesh Python reactive scaling verification failed." + } +} +finally { + $env:PYTHONPATH = $previousPythonPath +} + +Write-Output '{"runtimes":["csharp","typescript","python"],"documentCounts":[1,100,1000],"changedFraction":0.01,"idleSchedules":0,"invariant":"scheduling is proportional to explicitly changed documents"}' diff --git a/src/GameCult.Caching.MessagePack.Analyzers/Class1.cs b/src/GameCult.Caching.MessagePack.Analyzers/Class1.cs deleted file mode 100644 index 345e38e6..00000000 --- a/src/GameCult.Caching.MessagePack.Analyzers/Class1.cs +++ /dev/null @@ -1,11 +0,0 @@ -using System; - -namespace GameCult.Caching.MessagePack.Analyzer -{ - /// - /// Placeholder type for the analyzer packaging project. - /// - public class Class1 - { - } -} diff --git a/src/GameCult.Caching.MessagePack.Analyzers/GameCult.Caching.MessagePack.Analyzers.csproj b/src/GameCult.Caching.MessagePack.Analyzers/GameCult.Caching.MessagePack.Analyzers.csproj deleted file mode 100644 index 12c4e684..00000000 --- a/src/GameCult.Caching.MessagePack.Analyzers/GameCult.Caching.MessagePack.Analyzers.csproj +++ /dev/null @@ -1,33 +0,0 @@ - - - - - Source generator that emits MessagePack formatters for DatabaseEntry subclasses - MessagePack;SourceGenerator;DatabaseEntry;Analyzer - - false - $(TargetsForTfmSpecificContentInPackage);PackBuildOutputs - true - false - true - - - - - - - - - - - - - - - - - - - - diff --git a/src/GameCult.Caching.MessagePack.Analyzers/README.md b/src/GameCult.Caching.MessagePack.Analyzers/README.md deleted file mode 100644 index e4b25bd4..00000000 --- a/src/GameCult.Caching.MessagePack.Analyzers/README.md +++ /dev/null @@ -1,25 +0,0 @@ -# GameCult.Caching.MessagePack.Analyzers - -`GameCult.Caching.MessagePack.Analyzers` is the delivery mechanism for the MessagePack source generator used by the cache libraries. - -## Scope - -This project is primarily packaging infrastructure. It exists so consuming projects can receive generator output at compile time without treating the generator assembly as a normal runtime dependency. - -## What It Does - -- references the generator project -- arranges for the generator assembly to be packed under `analyzers/dotnet/cs` -- keeps generator assets out of the normal compile/runtime reference flow - -## Practical Meaning - -When a consuming project adds the analyzer package: - -- the generator runs during compilation -- concrete `DatabaseEntry` types receive generated MessagePack formatters -- `GameCult.Caching.MessagePack` can then serialize those types without handwritten formatter code - -## Current State - -This project is a packaging host. It does not currently expose custom analyzer diagnostics of its own. diff --git a/src/GameCult.Caching.MessagePack.Generator/CultDocumentMessagePackGenerator.cs b/src/GameCult.Caching.MessagePack.Generator/CultDocumentMessagePackGenerator.cs deleted file mode 100644 index a58918d5..00000000 --- a/src/GameCult.Caching.MessagePack.Generator/CultDocumentMessagePackGenerator.cs +++ /dev/null @@ -1,636 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Collections.Immutable; -using System.Linq; -using System.Text; -using Microsoft.CodeAnalysis; -using Microsoft.CodeAnalysis.CSharp.Syntax; -using Microsoft.CodeAnalysis.Text; - -namespace GameCult.Caching.MessagePack.Generator -{ -/// -/// Emits MessagePack payload serializers for CultCache document types. -/// -[Generator] -public sealed class CultDocumentMessagePackGenerator : IIncrementalGenerator -{ - /// - /// Registers the generator pipeline. - /// - public void Initialize(IncrementalGeneratorInitializationContext context) - { - var documentShapes = context.SyntaxProvider - .CreateSyntaxProvider( - static (node, _) => node is ClassDeclarationSyntax cds && cds.AttributeLists.Count > 0, - static (ctx, _) => BuildDocumentShape(ctx)) - .Where(static shape => shape != null) - .Select(static (shape, _) => shape!); - - var compilationAndShapes = context.CompilationProvider.Combine(documentShapes.Collect()); - context.RegisterSourceOutput( - compilationAndShapes, - static (spc, pair) => EmitProvider(spc, pair.Left, pair.Right)); - } - - private static DocumentShape? BuildDocumentShape(GeneratorSyntaxContext context) - { - if (context.SemanticModel.GetDeclaredSymbol(context.Node) is not INamedTypeSymbol typeSymbol) - { - return null; - } - - var documentAttribute = GetAttribute(typeSymbol, "GameCult.Caching.CultDocumentAttribute"); - if (documentAttribute == null || - documentAttribute.ConstructorArguments.Length < 2 || - documentAttribute.ConstructorArguments[0].Value is not string schemaName || - documentAttribute.ConstructorArguments[1].Value is not string schemaVersion) - { - return null; - } - - var members = DiscoverMembers(typeSymbol); - var nameMember = members.FirstOrDefault(member => member.IsName); - var canConstructForDeserialization = typeSymbol.TypeKind == TypeKind.Struct || - typeSymbol.InstanceConstructors.Any(constructor => - constructor.Parameters.Length == 0 && - constructor.DeclaredAccessibility is Accessibility.Public or Accessibility.Internal); - var hasDenseSlots = members - .OrderBy(member => member.Slot) - .Select((member, index) => member.Slot == index) - .All(matches => matches); - return new DocumentShape( - typeSymbol.ToDisplayString(SymbolDisplayFormat.FullyQualifiedFormat), - schemaName, - schemaVersion, - GetAttribute(typeSymbol, "GameCult.Caching.CultGlobalAttribute") != null, - nameMember?.Name, - nameMember?.AccessorMethodName, - members.Where(member => member.IndexAlias != null) - .Select(member => new IndexAccessorShape(member.IndexAlias!, member.IndexAccessorMethodName!)) - .ToImmutableArray(), - members.ToImmutableArray(), - canConstructForDeserialization, - hasDenseSlots); - } - - private static ImmutableArray DiscoverMembers(INamedTypeSymbol typeSymbol) - { - var candidates = new List(); - foreach (var member in typeSymbol.GetMembers()) - { - if (member is IFieldSymbol field) - { - if (field.IsStatic || field.DeclaredAccessibility != Accessibility.Public || IsIgnored(field)) - { - continue; - } - - candidates.Add(MemberShapeSeed.From(field, field.Type)); - continue; - } - - if (member is IPropertySymbol property) - { - if (property.IsStatic || - property.IsIndexer || - property.DeclaredAccessibility != Accessibility.Public || - property.GetMethod == null || - property.SetMethod == null || - IsIgnored(property)) - { - continue; - } - - candidates.Add(MemberShapeSeed.From(property, property.Type)); - } - } - - var explicitMembers = candidates - .Where(member => member.ExplicitSlot.HasValue) - .OrderBy(member => member.ExplicitSlot!.Value) - .ThenBy(member => member.SortKey, StringComparer.Ordinal) - .ToArray(); - var implicitMembers = candidates - .Where(member => !member.ExplicitSlot.HasValue) - .OrderBy(member => member.SortKey, StringComparer.Ordinal) - .ToArray(); - var nextSlot = explicitMembers.Length == 0 ? 0 : explicitMembers.Max(member => member.ExplicitSlot!.Value) + 1; - var shaped = new List(candidates.Count); - - foreach (var member in explicitMembers) - { - shaped.Add(member.WithSlot(member.ExplicitSlot!.Value)); - } - - foreach (var member in implicitMembers) - { - shaped.Add(member.WithSlot(nextSlot++)); - } - - return shaped - .OrderBy(member => member.Slot) - .ToImmutableArray(); - } - - private static bool IsIgnored(ISymbol member) - { - return GetAttribute(member, "MessagePack.IgnoreMemberAttribute") != null; - } - - private static AttributeData? GetAttribute(ISymbol symbol, string fullyQualifiedName) - { - return symbol.GetAttributes().FirstOrDefault(attribute => - attribute.AttributeClass?.ToDisplayString() == fullyQualifiedName); - } - - private static int? GetExplicitSlot(ISymbol member) - { - var keyAttribute = GetAttribute(member, "MessagePack.KeyAttribute"); - if (keyAttribute == null || keyAttribute.ConstructorArguments.Length == 0) - { - return null; - } - - var argument = keyAttribute.ConstructorArguments[0]; - return argument.Kind == TypedConstantKind.Primitive && argument.Value is int slot - ? slot - : null; - } - - private static void EmitProvider(SourceProductionContext context, Compilation compilation, ImmutableArray shapes) - { - var documents = shapes - .Where(shape => shape != null) - .GroupBy(shape => shape.DocumentTypeName, StringComparer.Ordinal) - .Select(group => group.First()) - .OrderBy(shape => shape.DocumentTypeName, StringComparer.Ordinal) - .ToArray(); - - if (documents.Length == 0) - { - return; - } - - var canEmitPayloadCodecs = - compilation.GetTypeByMetadataName("MessagePack.MessagePackWriter") != null && - compilation.GetTypeByMetadataName("GameCult.Caching.MessagePack.CultDocumentMessagePackSerialization") != null; - - var providerName = $"GeneratedCultDocumentMetadataProvider_{SanitizeIdentifier(compilation.AssemblyName ?? "GameCult")}"; - var builder = new StringBuilder(); - builder.AppendLine("// "); - builder.AppendLine("#nullable enable"); - builder.AppendLine($"[assembly: global::GameCult.Caching.CultGeneratedDocumentMetadataProviderAttribute(typeof(global::GameCult.Caching.Generated.{providerName}))]"); - builder.AppendLine("namespace GameCult.Caching.Generated"); - builder.AppendLine("{"); - builder.AppendLine($" internal sealed class {providerName} : global::GameCult.Caching.ICultGeneratedDocumentMetadataProvider"); - builder.AppendLine(" {"); - builder.AppendLine(" private static readonly global::GameCult.Caching.CultGeneratedDocumentDefinition[] s_documents ="); - builder.AppendLine(" {"); - - foreach (var document in documents) - { - builder.AppendLine(" new global::GameCult.Caching.CultGeneratedDocumentDefinition("); - builder.AppendLine($" typeof({document.DocumentTypeName}),"); - builder.AppendLine($" \"{Escape(document.SchemaName)}\","); - builder.AppendLine($" \"{Escape(document.SchemaVersion)}\","); - builder.AppendLine($" {(document.IsGlobal ? "true" : "false")},"); - builder.AppendLine(document.NameMember == null - ? " null," - : $" \"{Escape(document.NameMember)}\","); - builder.AppendLine(document.NameAccessorMethodName == null - ? " null," - : $" {document.NameAccessorMethodName},"); - builder.AppendLine(canEmitPayloadCodecs && document.HasDenseSlots - ? $" {document.PayloadSerializerMethodName}," - : " null,"); - builder.AppendLine(canEmitPayloadCodecs && document.HasDenseSlots && document.CanConstructForDeserialization - ? $" {document.PayloadDeserializerMethodName}," - : " null,"); - builder.AppendLine(" new global::GameCult.Caching.CultGeneratedDocumentIndexAccessor[]"); - builder.AppendLine(" {"); - foreach (var indexAccessor in document.IndexAccessors) - { - builder.AppendLine($" new global::GameCult.Caching.CultGeneratedDocumentIndexAccessor(\"{Escape(indexAccessor.Alias)}\", {indexAccessor.AccessorMethodName}),"); - } - builder.AppendLine(" },"); - builder.AppendLine(" new global::GameCult.Caching.CultGeneratedDocumentMemberDefinition[]"); - builder.AppendLine(" {"); - foreach (var member in document.Members.OrderBy(member => member.Slot)) - { - builder.AppendLine(" new global::GameCult.Caching.CultGeneratedDocumentMemberDefinition("); - builder.AppendLine($" \"{Escape(member.Name)}\","); - builder.AppendLine($" {member.Slot},"); - builder.AppendLine($" \"{Escape(member.SchemaTypeName)}\","); - builder.AppendLine($" {(member.IsReference ? "true" : "false")},"); - builder.AppendLine($" {(member.IsMany ? "true" : "false")},"); - builder.AppendLine(member.TargetSchemaName == null - ? " null," - : $" \"{Escape(member.TargetSchemaName)}\","); - builder.AppendLine($" {(member.IsName ? "true" : "false")},"); - builder.AppendLine(member.IndexAlias == null - ? " null)," - : $" \"{Escape(member.IndexAlias)}\"),"); - } - builder.AppendLine(" }),"); - } - - builder.AppendLine(" };"); - builder.AppendLine(); - builder.AppendLine(" public global::System.Collections.Generic.IEnumerable GetDocumentDefinitions()"); - builder.AppendLine(" {"); - builder.AppendLine(" return s_documents;"); - builder.AppendLine(" }"); - builder.AppendLine(); - - foreach (var document in documents) - { - if (canEmitPayloadCodecs && document.HasDenseSlots) - { - EmitPayloadSerializer(builder, document); - if (document.CanConstructForDeserialization) - { - EmitPayloadDeserializer(builder, document); - } - } - - foreach (var member in document.Members.Where(member => member.AccessorMethodName != null)) - { - builder.AppendLine($" private static string? {member.AccessorMethodName}(object document)"); - builder.AppendLine(" {"); - builder.AppendLine(member.CanBeNull - ? $" return (({document.DocumentTypeName})document).{member.Name}?.ToString();" - : $" return (({document.DocumentTypeName})document).{member.Name}.ToString();"); - builder.AppendLine(" }"); - builder.AppendLine(); - } - - foreach (var member in document.Members.Where(member => member.IndexAccessorMethodName != null)) - { - builder.AppendLine($" private static string {member.IndexAccessorMethodName}(object document)"); - builder.AppendLine(" {"); - builder.AppendLine(member.CanBeNull - ? $" return (({document.DocumentTypeName})document).{member.Name}?.ToString() ?? string.Empty;" - : $" return (({document.DocumentTypeName})document).{member.Name}.ToString();"); - builder.AppendLine(" }"); - builder.AppendLine(); - } - } - - builder.AppendLine(" }"); - builder.AppendLine("}"); - context.AddSource($"{providerName}.g.cs", SourceText.From(builder.ToString(), Encoding.UTF8)); - } - - private static void EmitPayloadSerializer(StringBuilder builder, DocumentShape document) - { - builder.AppendLine($" private static byte[] {document.PayloadSerializerMethodName}(object document)"); - builder.AppendLine(" {"); - builder.AppendLine($" var typed = ({document.DocumentTypeName})document;"); - builder.AppendLine($" var options = global::GameCult.Caching.MessagePack.CultDocumentMessagePackSerialization.OptionsFor(typeof({document.DocumentTypeName}));"); - builder.AppendLine(" var buffer = new global::System.Buffers.ArrayBufferWriter();"); - builder.AppendLine(" var writer = new global::MessagePack.MessagePackWriter(buffer);"); - builder.AppendLine($" writer.WriteArrayHeader({document.Members.Length});"); - foreach (var member in document.Members.OrderBy(member => member.Slot)) - { - builder.AppendLine($" global::MessagePack.FormatterResolverExtensions.GetFormatterWithVerify<{member.TypeSyntaxName}>(options.Resolver).Serialize(ref writer, typed.{member.Name}, options);"); - } - builder.AppendLine(" writer.Flush();"); - builder.AppendLine(" return buffer.WrittenSpan.ToArray();"); - builder.AppendLine(" }"); - builder.AppendLine(); - } - - private static void EmitPayloadDeserializer(StringBuilder builder, DocumentShape document) - { - builder.AppendLine($" private static object {document.PayloadDeserializerMethodName}(byte[] payload)"); - builder.AppendLine(" {"); - builder.AppendLine($" var options = global::GameCult.Caching.MessagePack.CultDocumentMessagePackSerialization.OptionsFor(typeof({document.DocumentTypeName}));"); - builder.AppendLine(" var reader = new global::MessagePack.MessagePackReader(payload);"); - builder.AppendLine(" var count = reader.ReadArrayHeader();"); - builder.AppendLine($" var value = new {document.DocumentTypeName}();"); - foreach (var member in document.Members.OrderBy(member => member.Slot)) - { - builder.AppendLine($" if (count > {member.Slot})"); - builder.AppendLine(" {"); - builder.AppendLine($" value.{member.Name} = global::MessagePack.FormatterResolverExtensions.GetFormatterWithVerify<{member.TypeSyntaxName}>(options.Resolver).Deserialize(ref reader, options);"); - builder.AppendLine(" }"); - } - builder.AppendLine($" for (var index = {document.Members.Length}; index < count; index++)"); - builder.AppendLine(" {"); - builder.AppendLine(" reader.Skip();"); - builder.AppendLine(" }"); - builder.AppendLine(" return value;"); - builder.AppendLine(" }"); - builder.AppendLine(); - } - - private static string Escape(string value) - { - return value - .Replace("\\", "\\\\") - .Replace("\"", "\\\"") - .Replace("\r", "\\r") - .Replace("\n", "\\n") - .Replace("\t", "\\t"); - } - - private static string SanitizeIdentifier(string value) - { - var builder = new StringBuilder(value.Length); - foreach (var character in value) - { - builder.Append(char.IsLetterOrDigit(character) ? character : '_'); - } - - if (builder.Length == 0 || !char.IsLetter(builder[0]) && builder[0] != '_') - { - builder.Insert(0, '_'); - } - - return builder.ToString(); - } - - private static string GetSchemaTypeName(ITypeSymbol typeSymbol) - { - if (typeSymbol is IArrayTypeSymbol arrayType) - { - return GetSchemaTypeName(arrayType.ElementType) + "[]"; - } - - if (typeSymbol is INamedTypeSymbol namedType && namedType.IsGenericType) - { - var baseName = GetNamedTypeBaseName(namedType.ConstructedFrom); - var arguments = string.Join(", ", namedType.TypeArguments.Select(GetSchemaTypeName)); - return $"{baseName}<{arguments}>"; - } - - return typeSymbol.SpecialType switch - { - SpecialType.System_Boolean => "System.Boolean", - SpecialType.System_Byte => "System.Byte", - SpecialType.System_Char => "System.Char", - SpecialType.System_Decimal => "System.Decimal", - SpecialType.System_Double => "System.Double", - SpecialType.System_Int16 => "System.Int16", - SpecialType.System_Int32 => "System.Int32", - SpecialType.System_Int64 => "System.Int64", - SpecialType.System_Object => "System.Object", - SpecialType.System_SByte => "System.SByte", - SpecialType.System_Single => "System.Single", - SpecialType.System_String => "System.String", - SpecialType.System_UInt16 => "System.UInt16", - SpecialType.System_UInt32 => "System.UInt32", - SpecialType.System_UInt64 => "System.UInt64", - _ => GetNamedTypeBaseName((INamedTypeSymbol)typeSymbol) - }; - } - - private static string GetNamedTypeBaseName(INamedTypeSymbol typeSymbol) - { - var parts = new Stack(); - var current = typeSymbol; - while (current != null) - { - parts.Push(current.Name); - current = current.ContainingType; - } - - var namespaceName = typeSymbol.ContainingNamespace?.ToDisplayString(); - return string.IsNullOrWhiteSpace(namespaceName) - ? string.Join(".", parts) - : namespaceName + "." + string.Join(".", parts); - } - - private static bool CanBeNull(ITypeSymbol typeSymbol) - { - if (typeSymbol.NullableAnnotation == NullableAnnotation.Annotated) - { - return true; - } - - if (typeSymbol.IsReferenceType) - { - return true; - } - - return typeSymbol is INamedTypeSymbol namedType && - namedType.IsGenericType && - namedType.ConstructedFrom.SpecialType == SpecialType.System_Nullable_T; - } - - private sealed class MemberShapeSeed - { - private MemberShapeSeed( - string name, - string schemaTypeName, - string typeSyntaxName, - int? explicitSlot, - string sortKey, - bool isName, - string? indexAlias, - bool isReference, - bool isMany, - bool canBeNull, - string? targetSchemaName) - { - Name = name; - SchemaTypeName = schemaTypeName; - TypeSyntaxName = typeSyntaxName; - ExplicitSlot = explicitSlot; - SortKey = sortKey; - IsName = isName; - IndexAlias = indexAlias; - IsReference = isReference; - IsMany = isMany; - CanBeNull = canBeNull; - TargetSchemaName = targetSchemaName; - } - - public string Name { get; } - public string SchemaTypeName { get; } - public string TypeSyntaxName { get; } - public int? ExplicitSlot { get; } - public string SortKey { get; } - public bool IsName { get; } - public string? IndexAlias { get; } - public bool IsReference { get; } - public bool IsMany { get; } - public bool CanBeNull { get; } - public string? TargetSchemaName { get; } - - public MemberShape WithSlot(int slot) - { - var accessorStem = SanitizeIdentifier(SortKey); - return new MemberShape( - Name, - slot, - SchemaTypeName, - TypeSyntaxName, - IsReference, - IsMany, - TargetSchemaName, - IsName, - IndexAlias, - CanBeNull, - IsName ? $"Access_{accessorStem}_Name" : null, - IndexAlias != null ? $"Access_{accessorStem}_Index" : null); - } - - public static MemberShapeSeed From(ISymbol member, ITypeSymbol memberType) - { - var referenceAttribute = GetAttribute(member, "GameCult.Caching.CultReferenceAttribute"); - var explicitTarget = referenceAttribute?.ConstructorArguments.Length > 0 - ? referenceAttribute.ConstructorArguments[0].Value as INamedTypeSymbol - : null; - var many = referenceAttribute?.ConstructorArguments.Length > 1 && - referenceAttribute.ConstructorArguments[1].Value is bool manyValue && - manyValue; - var targetType = ResolveReferenceTarget(memberType, explicitTarget); - var targetSchemaName = targetType == null - ? null - : GetAttribute(targetType, "GameCult.Caching.CultDocumentAttribute")?.ConstructorArguments[0].Value as string; - var indexAttribute = GetAttribute(member, "GameCult.Caching.CultIndexAttribute"); - var indexAlias = indexAttribute == null - ? null - : indexAttribute.ConstructorArguments.Length == 0 || indexAttribute.ConstructorArguments[0].Value is not string alias || string.IsNullOrWhiteSpace(alias) - ? member.Name - : alias; - - return new MemberShapeSeed( - member.Name, - GetSchemaTypeName(memberType), - memberType.ToDisplayString(SymbolDisplayFormat.FullyQualifiedFormat), - GetExplicitSlot(member), - BuildAccessorStem(member), - GetAttribute(member, "GameCult.Caching.CultNameAttribute") != null, - indexAlias, - targetType != null || referenceAttribute != null, - many, - CanBeNull(memberType), - targetSchemaName); - } - - private static INamedTypeSymbol? ResolveReferenceTarget(ITypeSymbol memberType, INamedTypeSymbol? explicitTarget) - { - if (explicitTarget != null) - { - return explicitTarget; - } - - return memberType is INamedTypeSymbol namedType && - namedType.IsGenericType && - namedType.Name == "CultRecordRef" && - namedType.ContainingNamespace.ToDisplayString() == "GameCult.Caching" - ? namedType.TypeArguments[0] as INamedTypeSymbol - : null; - } - - private static string BuildAccessorStem(ISymbol member) - { - return member.ContainingType == null - ? member.Name - : member.ContainingType.Name + "_" + member.Name; - } - } - - private sealed class DocumentShape - { - public DocumentShape( - string documentTypeName, - string schemaName, - string schemaVersion, - bool isGlobal, - string? nameMember, - string? nameAccessorMethodName, - ImmutableArray indexAccessors, - ImmutableArray members, - bool canConstructForDeserialization, - bool hasDenseSlots) - { - DocumentTypeName = documentTypeName; - SchemaName = schemaName; - SchemaVersion = schemaVersion; - IsGlobal = isGlobal; - NameMember = nameMember; - NameAccessorMethodName = nameAccessorMethodName; - var safeStem = SanitizeIdentifier(documentTypeName.Replace("global::", string.Empty).Replace(".", "_")); - PayloadSerializerMethodName = $"Serialize_{safeStem}_Payload"; - PayloadDeserializerMethodName = $"Deserialize_{safeStem}_Payload"; - IndexAccessors = indexAccessors; - Members = members; - CanConstructForDeserialization = canConstructForDeserialization; - HasDenseSlots = hasDenseSlots; - } - - public string DocumentTypeName { get; } - public string SchemaName { get; } - public string SchemaVersion { get; } - public bool IsGlobal { get; } - public string? NameMember { get; } - public string? NameAccessorMethodName { get; } - public string PayloadSerializerMethodName { get; } - public string PayloadDeserializerMethodName { get; } - public ImmutableArray IndexAccessors { get; } - public ImmutableArray Members { get; } - public bool CanConstructForDeserialization { get; } - public bool HasDenseSlots { get; } - } - - private sealed class IndexAccessorShape - { - public IndexAccessorShape(string alias, string accessorMethodName) - { - Alias = alias; - AccessorMethodName = accessorMethodName; - } - - public string Alias { get; } - public string AccessorMethodName { get; } - } - - private sealed class MemberShape - { - public MemberShape( - string name, - int slot, - string schemaTypeName, - string typeSyntaxName, - bool isReference, - bool isMany, - string? targetSchemaName, - bool isName, - string? indexAlias, - bool canBeNull, - string? accessorMethodName, - string? indexAccessorMethodName) - { - Name = name; - Slot = slot; - SchemaTypeName = schemaTypeName; - TypeSyntaxName = typeSyntaxName; - IsReference = isReference; - IsMany = isMany; - TargetSchemaName = targetSchemaName; - IsName = isName; - IndexAlias = indexAlias; - CanBeNull = canBeNull; - AccessorMethodName = accessorMethodName; - IndexAccessorMethodName = indexAccessorMethodName; - } - - public string Name { get; } - public int Slot { get; } - public string SchemaTypeName { get; } - public string TypeSyntaxName { get; } - public bool IsReference { get; } - public bool IsMany { get; } - public string? TargetSchemaName { get; } - public bool IsName { get; } - public string? IndexAlias { get; } - public bool CanBeNull { get; } - public string? AccessorMethodName { get; } - public string? IndexAccessorMethodName { get; } - } - } -} diff --git a/src/GameCult.Caching.MessagePack.Generator/GameCult.Caching.MessagePack.Generator.csproj b/src/GameCult.Caching.MessagePack.Generator/GameCult.Caching.MessagePack.Generator.csproj deleted file mode 100644 index 1a6e372e..00000000 --- a/src/GameCult.Caching.MessagePack.Generator/GameCult.Caching.MessagePack.Generator.csproj +++ /dev/null @@ -1,16 +0,0 @@ - - - - true - enable - false - netstandard2.0 - true - - - - - - - - diff --git a/src/GameCult.Caching.MessagePack.Generator/README.md b/src/GameCult.Caching.MessagePack.Generator/README.md deleted file mode 100644 index 79bfd29a..00000000 --- a/src/GameCult.Caching.MessagePack.Generator/README.md +++ /dev/null @@ -1,73 +0,0 @@ -# GameCult.Caching.MessagePack.Generator - -`GameCult.Caching.MessagePack.Generator` is a Roslyn incremental source generator that emits MessagePack formatters for concrete `DatabaseEntry` subclasses. - -## Scope - -This project is infrastructure for the caching stack. Application code usually does not reference it directly. - -Its job is to: - -- find non-abstract subclasses of `DatabaseEntry` -- generate concrete `IMessagePackFormatter` implementations -- generate resolver mappings consumed by `DatabaseEntryResolver` - -## What It Generates - -For each discovered concrete `DatabaseEntry` subtype, the generator emits: - -- a formatter type -- a resolver mapping entry - -Member slot assignment is: - -1. `ID` in slot `0` -2. members with `[Key(...)]` in their exact key slot -3. remaining non-ignored instance fields and properties appended after the - highest occupied slot as legacy convenience - -The generated formatter writes a MessagePack array sized to the highest occupied -slot and writes nil for unused gaps. Deserialization tolerates shorter older -arrays by leaving missing members at their constructor/field-initializer values, -and skips unknown extra slots. That makes `[Key(n)]` a durable wire contract: -deleted fields can leave their slot reserved and new fields can use new keys -without a store migration. - -For schema-stable cache entries, annotate every persisted member. Unkeyed -members are supported for older code, but they are not a good long-term wire -contract because source member order is not a schema. - -## Example - -Given: - -```csharp -[MessagePackObject] -public class PlayerData : DatabaseEntry, INamedEntry -{ - [Key(1)] public string Email = string.Empty; - [Key(2)] public string PasswordHash = string.Empty; - [Key(3)] public string Username = string.Empty; - - [IgnoreMember] - public string EntryName - { - get => Username; - set => Username = value; - } -} -``` - -the generator emits the formatter plumbing needed for MessagePack serialization of `PlayerData` without manually implementing an `IMessagePackFormatter`. - -## When You Need To Think About This Package - -Usually only when: - -- formatter generation is not happening -- a cache-entry type is not serializing as expected -- you are debugging build-time source generation behavior - -## Distribution - -The generator is packaged for consumers through [../GameCult.Caching.MessagePack.Analyzers/README.md](../GameCult.Caching.MessagePack.Analyzers/README.md). diff --git a/src/GameCult.Caching.MessagePack/CultCacheFormatterResolverAttribute.cs b/src/GameCult.Caching.MessagePack/CultCacheFormatterResolverAttribute.cs new file mode 100644 index 00000000..038b65f6 --- /dev/null +++ b/src/GameCult.Caching.MessagePack/CultCacheFormatterResolverAttribute.cs @@ -0,0 +1,15 @@ +using System; + +namespace GameCult.Caching.MessagePack; + +// The resolver type exposes a public static Instance of IFormatterResolver or a public parameterless constructor. +[AttributeUsage(AttributeTargets.Assembly, AllowMultiple = true)] +public sealed class CultCacheFormatterResolverAttribute : Attribute +{ + public CultCacheFormatterResolverAttribute(Type resolverType) + { + ResolverType = resolverType ?? throw new ArgumentNullException(nameof(resolverType)); + } + + public Type ResolverType { get; } +} diff --git a/src/GameCult.Caching.MessagePack/CultCacheMessagePack.cs b/src/GameCult.Caching.MessagePack/CultCacheMessagePack.cs index 16bbd6c9..e1a68c12 100644 --- a/src/GameCult.Caching.MessagePack/CultCacheMessagePack.cs +++ b/src/GameCult.Caching.MessagePack/CultCacheMessagePack.cs @@ -4,131 +4,55 @@ using System.Threading.Tasks; namespace GameCult.Caching.MessagePack { - /// - /// Options for opening a CultCache over the canonical single-file MessagePack backing store. - /// public sealed class CultCacheOpenOptions { - /// - /// Gets or sets the document registry to use. When omitted, the shared registry is used. - /// public CultDocumentRegistry? Registry { get; set; } - /// - /// Gets or sets whether the cache should pull the existing snapshot during open. - /// - public bool PullOnOpen { get; set; } = true; + public bool ReadOnly { get; set; } - /// - /// Gets or sets whether the cache should flush attached stores when disposed. - /// public bool FlushOnDispose { get; set; } - /// - /// Gets or sets whether the backing store should flush when disposed. - /// public bool StoreFlushOnDispose { get; set; } - /// - /// Gets or sets an optional callback used to customize the cache before opening. - /// - public Action? ConfigureCache { get; set; } - - /// - /// Gets or sets an optional callback used to customize the backing store before opening. - /// - public Action? ConfigureStore { get; set; } - - /// - /// Gets or sets whether the cache should use a paged directory store instead of one whole-file snapshot. - /// public bool UseDirectoryStore { get; set; } - - /// - /// Gets or sets the directory used for paged records. When omitted, the store uses the file path plus ".records". - /// - public string? DirectoryStorePath { get; set; } - - /// - /// Gets or sets the records to hydrate when opening a paged directory store. - /// The directory manifest is always read; rejected record payloads are never opened. - /// - public Func? DirectoryStoreHydrationFilter { get; set; } - - /// - /// Gets or sets an optional callback used to customize the directory backing store before opening. - /// - public Action? ConfigureDirectoryStore { get; set; } } - /// - /// Friendly MessagePack entrypoints for opening a CultCache with the canonical backing store. - /// public static class CultCacheMessagePack { - /// - /// Creates a cache with the canonical single-file MessagePack backing store attached. - /// + // The inspection model over this codec. Values serialize without the document guard but with their owning document's + // options, resolvers included, so dictionary keys compare by the bytes the store writes; clones deserialize as + // registered documents. + public static CultInspectorModel CreateInspectorModel(CultDocumentRegistry registry) => new CultInspectorModel( + registry, + (value, type, document) => global::MessagePack.MessagePackSerializer.Serialize(type, value, CultDocumentMessagePackSerialization.OptionsFor(document.Assembly)), + (type, bytes) => CultDocumentMessagePackSerialization.DeserializeUntyped(type, bytes, registry)); + + // Opening hydrates. A consumer that wants a fresh store removes the records it does not keep, upserts, and + // flushes; the atomic replace does the rest. public static CultCache Create(string filePath, CultCacheOpenOptions? options = null) - { - options ??= new CultCacheOpenOptions(); - return CreateCore(filePath, options, initializeGlobals: true); - } - - /// - /// Creates a cache with the canonical single-file MessagePack backing store attached and optionally pulls the on-disk snapshot. - /// - public static async Task OpenAsync(string filePath, CultCacheOpenOptions? options = null) - { - options ??= new CultCacheOpenOptions(); - var cache = CreateCore(filePath, options, initializeGlobals: false); - if (options.PullOnOpen) - { - await cache.PullAllBackingStoresAsync().ConfigureAwait(false); - } - - cache.MaterializeMissingGlobals(); - - return cache; - } - - private static CultCache CreateCore( - string filePath, - CultCacheOpenOptions options, - bool initializeGlobals) { if (string.IsNullOrWhiteSpace(filePath)) { throw new ArgumentException("File path must be non-empty.", nameof(filePath)); } - var cache = new CultCache(options.Registry, initializeGlobals) + options ??= new CultCacheOpenOptions(); + var cache = new CultCache(options.Registry) { FlushAttachedStoresOnDispose = options.FlushOnDispose }; - options.ConfigureCache?.Invoke(cache); - - if (options.UseDirectoryStore || Directory.Exists(options.DirectoryStorePath ?? DirectoryMessagePackBackingStore.DefaultRecordDirectoryPath(filePath))) - { - var directoryStore = new DirectoryMessagePackBackingStore(filePath, options.DirectoryStorePath) - { - FlushOnDispose = options.StoreFlushOnDispose, - HydrationFilter = options.DirectoryStoreHydrationFilter - }; - options.ConfigureDirectoryStore?.Invoke(directoryStore); - cache.AddBackingStore(directoryStore); - } - else - { - var store = new SingleFileMessagePackBackingStore(filePath) - { - FlushOnDispose = options.StoreFlushOnDispose - }; - options.ConfigureStore?.Invoke(store); - cache.AddBackingStore(store); - } + CacheBackingStore store = options.UseDirectoryStore || Directory.Exists(DirectoryMessagePackBackingStore.DefaultRecordDirectoryPath(filePath)) + ? new DirectoryMessagePackBackingStore(filePath, readOnly: options.ReadOnly) + : new SingleFileMessagePackBackingStore(filePath, options.ReadOnly); + store.FlushOnDispose = options.StoreFlushOnDispose; + cache.AddBackingStore(store); return cache; } + + public static Task OpenAsync(string filePath, CultCacheOpenOptions? options = null) + { + return Task.FromResult(Create(filePath, options)); + } } } diff --git a/src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs b/src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs index dcf0da0f..1e99125e 100644 --- a/src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs +++ b/src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs @@ -10,41 +10,11 @@ using MessagePack.Resolvers; namespace GameCult.Caching.MessagePack; -/// -/// Declares MessagePack resolvers owned by the assembly that defines a Cult document. -/// -[AttributeUsage(AttributeTargets.Assembly, AllowMultiple = true)] -public sealed class CultDocumentMessagePackResolversAttribute : Attribute -{ - /// Creates a resolver declaration in precedence order. - public CultDocumentMessagePackResolversAttribute(params Type[] resolverTypes) - { - if (resolverTypes == null || resolverTypes.Length == 0) - { - throw new ArgumentException("At least one resolver type is required.", nameof(resolverTypes)); - } - - ResolverTypes = resolverTypes; - } - - /// Gets the declared resolver types in precedence order. - public Type[] ResolverTypes { get; } -} - -/// -/// MessagePack resolver for CultCache-specific value types. -/// public sealed class CultDocumentResolver : IFormatterResolver { - /// - /// Gets the shared resolver instance. - /// public static readonly CultDocumentResolver Instance = new(); private CultDocumentResolver() { } - /// - /// Gets a formatter for the requested type, when this resolver owns it. - /// public IMessagePackFormatter? GetFormatter() { var type = typeof(T); @@ -58,146 +28,72 @@ public sealed class CultDocumentResolver : IFormatterResolver } } -/// -/// MessagePack serialization helpers for CultCache documents and backing stores. -/// public static class CultDocumentMessagePackSerialization { private const int PersistedRecordFieldCount = 4; private const int SchemaCatalogEntryFieldCount = 7; private const int SchemaCatalogMemberFieldCount = 8; private const int StoreSnapshotFieldCount = 3; - private static readonly ConcurrentDictionary DocumentAssemblyOptions = new(); - /// - /// Gets the shared MessagePack serializer options for CultCache payloads. - /// - public static readonly MessagePackSerializerOptions Options = - MessagePackSerializerOptions.Standard - .WithResolver(CompositeResolver.Create( - CultDocumentResolver.Instance, - StandardResolver.Instance)) - .WithSecurity(MessagePackSecurity.UntrustedData); + public static readonly MessagePackSerializerOptions Options = Compose(Array.Empty()); + + private static readonly ConcurrentDictionary AssemblyOptions = new(); - /// - /// Gets serializer options for documents owned by an assembly. CultCache formatters - /// take precedence, followed by assembly-declared resolvers and the standard resolver. - /// public static MessagePackSerializerOptions OptionsFor(Assembly documentAssembly) { if (documentAssembly == null) throw new ArgumentNullException(nameof(documentAssembly)); - return DocumentAssemblyOptions.GetOrAdd(documentAssembly, CreateDocumentAssemblyOptions); + return AssemblyOptions.GetOrAdd(documentAssembly, static assembly => + { + var resolvers = assembly.GetCustomAttributes() + .Select(attribute => CreateResolver(attribute.ResolverType)) + .ToArray(); + return resolvers.Length == 0 ? Options : Compose(resolvers); + }); } - /// Gets serializer options for the assembly that owns a document type. - public static MessagePackSerializerOptions OptionsFor(Type documentType) + private static MessagePackSerializerOptions Compose(IFormatterResolver[] consumerResolvers) { - if (documentType == null) throw new ArgumentNullException(nameof(documentType)); - return OptionsFor(documentType.Assembly); + return MessagePackSerializerOptions.Standard + .WithResolver(CompositeResolver.Create( + consumerResolvers.Append(CultDocumentResolver.Instance).Append(StandardResolver.Instance).ToArray())) + .WithSecurity(CultMessagePackSecurity.Instance); } - /// - /// Serializes a typed value with the CultCache MessagePack options. - /// - public static byte[] Serialize(T value) + private static IFormatterResolver CreateResolver(Type resolverType) { - return MessagePackSerializer.Serialize(value, OptionsFor(typeof(T))); + var instance = resolverType.GetField("Instance", BindingFlags.Public | BindingFlags.Static)?.GetValue(null) + ?? resolverType.GetProperty("Instance", BindingFlags.Public | BindingFlags.Static)?.GetValue(null) + ?? (resolverType.GetConstructor(Type.EmptyTypes) != null ? Activator.CreateInstance(resolverType) : null); + return instance as IFormatterResolver + ?? throw new InvalidOperationException( + $"Formatter resolver {resolverType.FullName} needs a public static Instance or a public parameterless constructor and must implement {nameof(IFormatterResolver)}."); } - /// - /// Deserializes a typed value with the CultCache MessagePack options. - /// - public static T Deserialize(byte[] payload) - { - return MessagePackSerializer.Deserialize(payload, OptionsFor(typeof(T))); - } - - /// - /// Serializes a value whose document type is known at runtime. - /// public static byte[] SerializeUntyped(object value, Type type) { return SerializeUntyped(value, type, CultDocumentRegistry.Shared); } - /// - /// Serializes a value through an explicitly owned document registry. - /// public static byte[] SerializeUntyped(object value, Type type, CultDocumentRegistry registry) { if (registry == null) throw new ArgumentNullException(nameof(registry)); - if (value != null) - { - var descriptor = registry.GetRequired(type); - if (descriptor.GeneratedPayloadSerializer != null) - { - return descriptor.GeneratedPayloadSerializer(value); - } - } - - return MessagePackSerializer.Serialize(type, value, OptionsFor(type)); + if (value != null) registry.GetRequired(type); + return MessagePackSerializer.Serialize(type, value, OptionsFor(type.Assembly)); } - /// - /// Deserializes a value whose document type is known at runtime. - /// public static object DeserializeUntyped(Type type, byte[] payload) { return DeserializeUntyped(type, payload, CultDocumentRegistry.Shared); } - /// - /// Deserializes a value through an explicitly owned document registry. - /// public static object DeserializeUntyped(Type type, byte[] payload, CultDocumentRegistry registry) { if (registry == null) throw new ArgumentNullException(nameof(registry)); - var descriptor = registry.GetRequired(type); - if (descriptor.GeneratedPayloadDeserializer != null) - { - return descriptor.GeneratedPayloadDeserializer(payload); - } - - return MessagePackSerializer.Deserialize(type, payload, OptionsFor(type)) + registry.GetRequired(type); + return MessagePackSerializer.Deserialize(type, payload, OptionsFor(type.Assembly)) ?? throw new InvalidOperationException($"MessagePack returned null for Cult document type {type.FullName}."); } - private static MessagePackSerializerOptions CreateDocumentAssemblyOptions(Assembly documentAssembly) - { - var ownerResolvers = documentAssembly - .GetCustomAttributes() - .SelectMany(attribute => attribute.ResolverTypes) - .Select(CreateOwnerResolver) - .ToArray(); - - if (ownerResolvers.Length == 0) - { - return Options; - } - - var resolvers = new IFormatterResolver[ownerResolvers.Length + 2]; - resolvers[0] = CultDocumentResolver.Instance; - Array.Copy(ownerResolvers, 0, resolvers, 1, ownerResolvers.Length); - resolvers[resolvers.Length - 1] = StandardResolver.Instance; - return Options.WithResolver(CompositeResolver.Create(resolvers)); - } - - private static IFormatterResolver CreateOwnerResolver(Type resolverType) - { - if (resolverType == null || !typeof(IFormatterResolver).IsAssignableFrom(resolverType)) - { - throw new InvalidOperationException( - $"Cult document MessagePack resolver '{resolverType?.FullName ?? ""}' must implement {typeof(IFormatterResolver).FullName}."); - } - - return Activator.CreateInstance(resolverType) as IFormatterResolver - ?? throw new InvalidOperationException( - $"Cult document MessagePack resolver '{resolverType.FullName}' must have a public parameterless constructor."); - } - - /// - /// Serializes one persisted store record. - /// public static byte[] SerializePersistedRecord(CultPersistedRecord record) { var buffer = new global::System.Buffers.ArrayBufferWriter(); @@ -207,51 +103,12 @@ public static class CultDocumentMessagePackSerialization return buffer.WrittenSpan.ToArray(); } - /// - /// Deserializes one persisted store record. - /// public static CultPersistedRecord DeserializePersistedRecord(byte[] payload) { var reader = new MessagePackReader(payload); return ReadPersistedRecord(ref reader); } - /// - /// Serializes a schema catalog. - /// - public static byte[] SerializeSchemaCatalog(CultSchemaCatalogEntry[] catalog) - { - var buffer = new global::System.Buffers.ArrayBufferWriter(); - var writer = new MessagePackWriter(buffer); - writer.WriteArrayHeader(catalog.Length); - foreach (var entry in catalog) - { - WriteSchemaCatalogEntry(ref writer, entry); - } - - writer.Flush(); - return buffer.WrittenSpan.ToArray(); - } - - /// - /// Deserializes a schema catalog. - /// - public static CultSchemaCatalogEntry[] DeserializeSchemaCatalog(byte[] payload) - { - var reader = new MessagePackReader(payload); - var count = reader.ReadArrayHeader(); - var catalog = new CultSchemaCatalogEntry[count]; - for (var index = 0; index < count; index++) - { - catalog[index] = ReadSchemaCatalogEntry(ref reader); - } - - return catalog; - } - - /// - /// Serializes a complete persisted store snapshot. - /// public static byte[] SerializeSnapshot(CultPersistedStoreSnapshot snapshot) { var buffer = new global::System.Buffers.ArrayBufferWriter(); @@ -274,9 +131,6 @@ public static class CultDocumentMessagePackSerialization return buffer.WrittenSpan.ToArray(); } - /// - /// Deserializes a complete persisted store snapshot. - /// public static CultPersistedStoreSnapshot DeserializeSnapshot(byte[] payload) { var reader = new MessagePackReader(payload); @@ -504,45 +358,27 @@ public static class CultDocumentMessagePackSerialization } } -/// -/// Single-file CultCache backing store that persists snapshots as MessagePack. -/// public class SingleFileMessagePackBackingStore : SingleFileBackingStore { - /// - /// Creates a MessagePack single-file backing store. - /// - public SingleFileMessagePackBackingStore(string filePath) : base(filePath) + public SingleFileMessagePackBackingStore(string filePath, bool readOnly = false) : base(filePath, readOnly) { } - /// - /// Serializes a store snapshot. - /// protected override byte[] SerializeSnapshot(CultPersistedStoreSnapshot snapshot) { return CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot); } - /// - /// Deserializes a store snapshot. - /// protected override CultPersistedStoreSnapshot DeserializeSnapshot(byte[] data) { return CultDocumentMessagePackSerialization.DeserializeSnapshot(data); } - /// - /// Serializes one document payload. - /// protected override byte[] SerializePayload(object document) { return CultDocumentMessagePackSerialization.SerializeUntyped(document, document.GetType(), Registry); } - /// - /// Deserializes one document payload. - /// protected override object DeserializePayload(Type documentType, byte[] payload) { return CultDocumentMessagePackSerialization.DeserializeUntyped(documentType, payload, Registry); diff --git a/src/GameCult.Caching.MessagePack/CultMessagePackSecurity.cs b/src/GameCult.Caching.MessagePack/CultMessagePackSecurity.cs new file mode 100644 index 00000000..64b237d5 --- /dev/null +++ b/src/GameCult.Caching.MessagePack/CultMessagePackSecurity.cs @@ -0,0 +1,35 @@ +using System; +using System.Collections.Generic; +using MessagePack; + +namespace GameCult.Caching.MessagePack; + +// UntrustedData, plus a collision-resistant comparer for CultRecordRef dictionary keys over the key string. +public sealed class CultMessagePackSecurity : MessagePackSecurity +{ + public static readonly CultMessagePackSecurity Instance = new(UntrustedData); + + private CultMessagePackSecurity(MessagePackSecurity copyFrom) : base(copyFrom) { } + + protected override IEqualityComparer GetHashCollisionResistantEqualityComparer() + { + var type = typeof(T); + if (!type.IsGenericType || type.GetGenericTypeDefinition() != typeof(CultRecordRef<>)) + return base.GetHashCollisionResistantEqualityComparer(); + var comparerType = typeof(RecordRefComparer<>).MakeGenericType(type.GetGenericArguments()[0]); + return (IEqualityComparer)Activator.CreateInstance(comparerType, GetEqualityComparer())!; + } + + protected override MessagePackSecurity Clone() => new CultMessagePackSecurity(this); + + private sealed class RecordRefComparer : IEqualityComparer> + { + private readonly IEqualityComparer _keys; + + public RecordRefComparer(IEqualityComparer keys) => _keys = keys; + + public bool Equals(CultRecordRef x, CultRecordRef y) => _keys.Equals(x.Key.Value, y.Key.Value); + + public int GetHashCode(CultRecordRef value) => _keys.GetHashCode(value.Key.Value); + } +} diff --git a/src/GameCult.Caching.MessagePack/CultRecordRefFormatter.cs b/src/GameCult.Caching.MessagePack/CultRecordRefFormatter.cs index 3348c1c3..a5f984fc 100644 --- a/src/GameCult.Caching.MessagePack/CultRecordRefFormatter.cs +++ b/src/GameCult.Caching.MessagePack/CultRecordRefFormatter.cs @@ -3,22 +3,13 @@ using MessagePack.Formatters; namespace GameCult.Caching; -/// -/// Serializes typed CultCache record references as their persisted key strings. -/// public sealed class CultRecordRefFormatter : IMessagePackFormatter> { - /// - /// Writes a typed record reference to MessagePack. - /// public void Serialize(ref MessagePackWriter writer, CultRecordRef value, MessagePackSerializerOptions options) { options.Resolver.GetFormatterWithVerify().Serialize(ref writer, value.Key.Value, options); } - /// - /// Reads a typed record reference from MessagePack. - /// public CultRecordRef Deserialize(ref MessagePackReader reader, MessagePackSerializerOptions options) { var key = options.Resolver.GetFormatterWithVerify().Deserialize(ref reader, options) ?? string.Empty; diff --git a/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs b/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs index 191febef..9e185202 100644 --- a/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs +++ b/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs @@ -4,40 +4,29 @@ using System.Collections.Generic; using System.Diagnostics; using System.IO; using System.Linq; +using System.Runtime.ExceptionServices; using System.Security.Cryptography; -using System.Text; using System.Threading; using System.Threading.Tasks; using MessagePack; namespace GameCult.Caching.MessagePack; -/// -/// MessagePack backing store that keeps a small schema manifest hot and stores each record as its own cold file. -/// +// A small hot manifest indexes the store; each record lives in one cold content-addressed page. public sealed class DirectoryMessagePackBackingStore : CacheBackingStore { - private const string IndexedFormatVersion = "cultcache.store.v2.directory-indexed"; + private const string IndexedFormatVersion = "cultcache.store.v4.directory-content-addressed-pages"; + // An unleased load that has not settled after this many attempts throws. + private const int UnleasedLoadAttempts = 5; private readonly FileInfo _manifestFile; private readonly DirectoryInfo _recordDirectory; private readonly ConcurrentDictionary _dirtyKeys = new(StringComparer.Ordinal); private readonly ConcurrentDictionary _deletedKeys = new(StringComparer.Ordinal); private readonly HashSet _hydratedKeys = new(StringComparer.Ordinal); - private Dictionary _durableIndex = new(StringComparer.Ordinal); - private Dictionary _legacyInlineRecords = new(StringComparer.Ordinal); private CultSchemaCatalogEntry[] _durableCatalog = Array.Empty(); - private bool _needsIndexUpgrade; - /// - /// Gets or sets the predicate that selects indexed record payloads for hydration. - /// A null predicate hydrates every record. - /// - public Func? HydrationFilter { get; set; } - - /// - /// Creates a paged MessagePack backing store. - /// - public DirectoryMessagePackBackingStore(string manifestPath, string? recordDirectoryPath = null) + public DirectoryMessagePackBackingStore(string manifestPath, string? recordDirectoryPath = null, bool readOnly = false) + : base(readOnly) { if (string.IsNullOrWhiteSpace(manifestPath)) { @@ -48,13 +37,16 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore _recordDirectory = new DirectoryInfo(recordDirectoryPath ?? DefaultRecordDirectoryPath(manifestPath)); } - /// - /// Gets the default record directory path for a manifest path. - /// public static string DefaultRecordDirectoryPath(string manifestPath) => manifestPath + ".records"; - /// - public override void PullAll() + // Test seam: runs between an unleased load's manifest read and its page reads. + internal Action? UnleasedManifestRead { get; set; } + + public override string ToString() => _manifestFile.FullName; + + public override void PullAll() => Held(PullAllCore); + + private void PullAllCore() { var traceStartup = string.Equals( Environment.GetEnvironmentVariable("CULTCACHE_TRACE_STARTUP_PHASES"), @@ -68,73 +60,73 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore startupPhase.Restart(); } - _recordDirectory.Refresh(); - if (!File.Exists(_manifestFile.FullName) && !_recordDirectory.Exists) - { - SetLastSchemaMigrationReports(Array.Empty()); - IsDirty = false; - return; - } - - var manifest = ReadManifest(); - Trace($"manifest records={manifest.Records.Length}"); - _durableCatalog = manifest.SchemaCatalog; var reports = new List(); var loaded = new Dictionary(StringComparer.Ordinal); - var indexed = string.Equals(manifest.FormatVersion, IndexedFormatVersion, StringComparison.Ordinal); - if (indexed) + CultSchemaCatalogEntry[] catalog; + for (var attempt = 1; ; attempt++) { - _needsIndexUpgrade = false; - _durableIndex = manifest.Records - .Where(record => !string.IsNullOrWhiteSpace(record.Key)) - .GroupBy(record => record.Key, StringComparer.Ordinal) - .ToDictionary(group => group.Key, group => group.Last(), StringComparer.Ordinal); - - LoadIndexedRecords(manifest, loaded, reports); - Trace($"indexed-pages loaded={loaded.Count}"); - } - else - { - // V1 manifests have no page index. Hydrate once to discover the durable keys; - // the next flush replaces this compatibility path with the indexed manifest. - _needsIndexUpgrade = true; - var legacyRecords = LoadLegacyRecords(manifest, loaded, reports); - _durableIndex = legacyRecords - .Where(record => !string.IsNullOrWhiteSpace(record.Key)) - .GroupBy(record => record.Key, StringComparer.Ordinal) - .ToDictionary(group => group.Key, group => ToIndexRecord(group.Last()), StringComparer.Ordinal); - Trace($"legacy-pages loaded={loaded.Count}"); - } - - foreach (var pair in loaded) - { - if (_dirtyKeys.ContainsKey(pair.Key) || _deletedKeys.ContainsKey(pair.Key)) - continue; - - if (Entries.TryGetValue(pair.Key, out var existing) && - string.Equals(existing.StoredAt, pair.Value.StoredAt, StringComparison.Ordinal) && - string.Equals(existing.Descriptor.SchemaId, pair.Value.Descriptor.SchemaId, StringComparison.Ordinal)) + loaded.Clear(); + reports.Clear(); + // Only pages named by the manifest read under the lease are loaded; orphaned pages are never loaded. With no + // lock to lease (none created yet, or a store copied without it) a writer may commit mid-load, so an unleased + // load stands only if its manifest is still current; if the manifest moved it reloads, under the lease the + // writer created. A page that fails under an unmoved manifest is corruption, not a race. + using var lease = AcquireCommitLease(wait: true, create: false); + var manifestBytes = ReadManifestBytes(); + var manifest = ParseManifest(manifestBytes); + Trace($"manifest records={manifest.Records.Length}"); + try { + if (lease == null) + UnleasedManifestRead?.Invoke(); + LoadRecordPages( + manifest.Records.OrderBy(record => record.Key, StringComparer.Ordinal).ToArray(), + manifest.SchemaCatalog, + loaded, + reports); + if (lease != null || !ManifestMoved(manifestBytes)) + { + catalog = manifest.SchemaCatalog; + Trace($"indexed-pages loaded={loaded.Count}"); + break; + } + } + catch (Exception exception) when (lease == null && IsTornGeneration(exception)) + { + // The re-read stays out of the filter: an exception thrown inside a filter is swallowed as false. + if (!ManifestMoved(manifestBytes)) + throw; + if (attempt == UnleasedLoadAttempts) + throw Unsettled(exception); continue; } - Entries[pair.Key] = pair.Value; - if (existing == null) - EntryAdded.OnNext(pair.Value); - else - EntryUpdated.OnNext(pair.Value); + if (attempt == UnleasedLoadAttempts) + throw Unsettled(null); } - foreach (var removedKey in _hydratedKeys.Where(key => !loaded.ContainsKey(key)).ToArray()) - { - if (_dirtyKeys.ContainsKey(removedKey) || _deletedKeys.ContainsKey(removedKey)) - continue; - - if (Entries.TryRemove(removedKey, out var removed)) - EntryDeleted.OnNext(removed); - _hydratedKeys.Remove(removedKey); - } + bool Staged(string key) => _dirtyKeys.ContainsKey(key) || _deletedKeys.ContainsKey(key); + var arrived = loaded + .Where(pair => !Staged(pair.Key) && + (!Entries.TryGetValue(pair.Key, out var existing) || + existing.StoredAt != pair.Value.StoredAt || + existing.Descriptor.SchemaId != pair.Value.Descriptor.SchemaId)) + .Select(pair => pair.Value) + .ToArray(); + var departed = _hydratedKeys + .Where(key => !loaded.ContainsKey(key) && !Staged(key) && Entries.ContainsKey(key)) + .Select(key => Entries[key]) + .ToArray(); + if (arrived.Length > 0 || departed.Length > 0) + Loaded?.Invoke(arrived, departed); + // Adopted only once the cache admitted the load; a refused load leaves the store's durable view untouched. + _durableCatalog = catalog; + foreach (var stored in departed) + Entries.TryRemove(stored.Key.Value, out _); + _hydratedKeys.RemoveWhere(key => !loaded.ContainsKey(key) && !Staged(key)); + foreach (var stored in arrived) + Entries[stored.Key.Value] = stored; foreach (var key in loaded.Keys) _hydratedKeys.Add(key); @@ -143,77 +135,101 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore Trace("publish"); } - /// - public override void PullSelected(Func selector) - { - if (selector == null) throw new ArgumentNullException(nameof(selector)); - var manifest = ReadManifest(); - if (!string.Equals(manifest.FormatVersion, IndexedFormatVersion, StringComparison.Ordinal)) - { - PullAll(); - return; - } - - _durableCatalog = manifest.SchemaCatalog; - _durableIndex = manifest.Records - .Where(record => !string.IsNullOrWhiteSpace(record.Key)) - .GroupBy(record => record.Key, StringComparer.Ordinal) - .ToDictionary(group => group.Key, group => group.Last(), StringComparer.Ordinal); - var selected = manifest.Records - .Where(record => selector(ToMetadata(record))) - .OrderBy(record => record.Key, StringComparer.Ordinal) - .ToArray(); - var loaded = new Dictionary(StringComparer.Ordinal); - var reports = new List(); - LoadRecordPages(selected, manifest.SchemaCatalog, loaded, reports); - PublishSelected(loaded, selected.Select(record => record.Key).ToHashSet(StringComparer.Ordinal)); - SetLastSchemaMigrationReports(reports); - IsDirty = !_dirtyKeys.IsEmpty || !_deletedKeys.IsEmpty; - } - - /// - public override bool ContainsDurableRecord(CultRecordKey key) - { - return _durableIndex.ContainsKey(key.Value) || base.ContainsDurableRecord(key); - } - - /// public override void Push(CultStoredDocument entry) { - Entries[entry.Key.Value] = entry; - _dirtyKeys[entry.Key.Value] = true; - _deletedKeys.TryRemove(entry.Key.Value, out _); - IsDirty = true; + ThrowIfReadOnly(); + Held(() => + { + Entries[entry.Key.Value] = entry; + _dirtyKeys[entry.Key.Value] = true; + _deletedKeys.TryRemove(entry.Key.Value, out _); + IsDirty = true; + }); } - /// public override void Delete(CultStoredDocument entry) { - Entries.TryRemove(entry.Key.Value, out _); - _dirtyKeys.TryRemove(entry.Key.Value, out _); - _deletedKeys[entry.Key.Value] = true; - IsDirty = true; + ThrowIfReadOnly(); + Held(() => + { + Entries.TryRemove(entry.Key.Value, out _); + _dirtyKeys.TryRemove(entry.Key.Value, out _); + _deletedKeys[entry.Key.Value] = true; + IsDirty = true; + }); } - /// - public override void PushAll(bool soft = false) + public override CultCommitOutcome CommitBatch(CultCommitRequest request, bool wait) { - if (!IsDirty && !_needsIndexUpgrade) + ThrowIfReadOnly(); + return Held(() => { - return; - } + Directory.CreateDirectory(_manifestFile.DirectoryName!); + using var commitLease = AcquireCommitLease(wait, create: true); + if (commitLease == null) + return CultCommitOutcome.Contended; + var manifest = ReadManifest(); + if (!request.ConditionsHold(manifest.Records, Entries.Values)) + return CultCommitOutcome.Mismatch; - Directory.CreateDirectory(_manifestFile.DirectoryName!); + var previousEntries = Entries.ToDictionary(pair => pair.Key, pair => pair.Value, StringComparer.Ordinal); + var previousDirtyKeys = _dirtyKeys.Keys.ToArray(); + var previousDeletedKeys = _deletedKeys.Keys.ToArray(); + var wasDirty = IsDirty; + try + { + foreach (var entry in request.Deletes) + { + Entries.TryRemove(entry.Key.Value, out _); + _dirtyKeys.TryRemove(entry.Key.Value, out _); + _deletedKeys[entry.Key.Value] = true; + } + foreach (var entry in request.Upserts) + { + Entries[entry.Key.Value] = entry; + _dirtyKeys[entry.Key.Value] = true; + _deletedKeys.TryRemove(entry.Key.Value, out _); + } + IsDirty = true; + WriteGeneration(manifest); + return CultCommitOutcome.Committed; + } + catch + { + Entries.Clear(); + foreach (var pair in previousEntries) + Entries[pair.Key] = pair.Value; + _dirtyKeys.Clear(); + foreach (var key in previousDirtyKeys) + _dirtyKeys[key] = true; + _deletedKeys.Clear(); + foreach (var key in previousDeletedKeys) + _deletedKeys[key] = true; + IsDirty = wasDirty; + throw; + } + }); + } + + public override void PushAll() + { + ThrowIfReadOnly(); + Held(() => + { + if (!IsDirty) + return; + + Directory.CreateDirectory(_manifestFile.DirectoryName!); + using var commitLease = AcquireCommitLease(wait: true, create: true); + WriteGeneration(ReadManifest()); + }); + } + + // Runs under the commit lease: pages first, then the manifest that names them. + private void WriteGeneration(CultPersistedStoreSnapshot currentManifest) + { Directory.CreateDirectory(_recordDirectory.FullName); - - var currentManifest = ReadManifest(); - var currentIndex = string.Equals(currentManifest.FormatVersion, IndexedFormatVersion, StringComparison.Ordinal) - ? currentManifest.Records.ToDictionary(record => record.Key, record => record, StringComparer.Ordinal) - : new Dictionary(_durableIndex, StringComparer.Ordinal); - var precommitIndex = currentIndex.Values - .Select(ToIndexRecord) - .OrderBy(record => record.Key, StringComparer.Ordinal) - .ToArray(); + var currentIndex = currentManifest.Records.ToDictionary(record => record.Key, record => record, StringComparer.Ordinal); foreach (var key in _deletedKeys.Keys) currentIndex.Remove(key); foreach (var key in _dirtyKeys.Keys) @@ -233,16 +249,7 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore .Where(entry => usedSchemaIds.Contains(entry.SchemaId)) .OrderBy(entry => entry.SchemaName, StringComparer.Ordinal) .ToArray(); - var precommitCatalog = currentManifest.SchemaCatalog - .Concat(targetCatalog) - .GroupBy(entry => entry.SchemaId, StringComparer.Ordinal) - .Select(group => group.Last()) - .OrderBy(entry => entry.SchemaName, StringComparer.Ordinal) - .ToArray(); - - WriteManifest(precommitCatalog, precommitIndex); - - var keysToWrite = (_needsIndexUpgrade ? Entries.Keys : _dirtyKeys.Keys) + var keysToWrite = _dirtyKeys.Keys .OrderBy(value => value, StringComparer.Ordinal) .ToArray(); foreach (var key in keysToWrite) @@ -257,35 +264,21 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore document, stored.Descriptor.DocumentType, Registry)); - WriteFileAtomically(RecordPath(key), CultDocumentMessagePackSerialization.SerializePersistedRecord(record)); - } - - foreach (var pair in _legacyInlineRecords.OrderBy(pair => pair.Key, StringComparer.Ordinal)) - { - if (_deletedKeys.ContainsKey(pair.Key) || _dirtyKeys.ContainsKey(pair.Key)) - continue; + var pagePayload = CultDocumentMessagePackSerialization.SerializePersistedRecord(record); + var indexRecord = ToContentAddressedIndexRecord(record, pagePayload); + currentIndex[key] = indexRecord; WriteFileAtomically( - RecordPath(pair.Key), - CultDocumentMessagePackSerialization.SerializePersistedRecord(pair.Value)); - } - - foreach (var key in _deletedKeys.Keys.OrderBy(value => value, StringComparer.Ordinal)) - { - var path = RecordPath(key); - if (File.Exists(path)) - { - File.Delete(path); - } + ContentAddressedRecordPath(indexRecord), + pagePayload); } WriteManifest(targetCatalog, currentIndex.Values .OrderBy(record => record.Key, StringComparer.Ordinal) .ToArray()); + DeleteUnreferencedRecordPages(currentIndex.Values); + _durableCatalog = targetCatalog; - _durableIndex = currentIndex; - _legacyInlineRecords.Clear(); - _needsIndexUpgrade = false; _dirtyKeys.Clear(); _deletedKeys.Clear(); MarkFlushSucceeded(); @@ -302,49 +295,6 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore WriteFileAtomically(_manifestFile.FullName, CultDocumentMessagePackSerialization.SerializeSnapshot(manifest)); } - private void LoadIndexedRecords( - CultPersistedStoreSnapshot manifest, - Dictionary loaded, - List reports) - { - var selected = manifest.Records - .Where(ShouldHydrate) - .OrderBy(record => record.Key, StringComparer.Ordinal) - .ToArray(); - LoadRecordPages(selected, manifest.SchemaCatalog, loaded, reports); - } - - private CultPersistedRecord[] LoadLegacyRecords( - CultPersistedStoreSnapshot manifest, - Dictionary loaded, - List reports) - { - _legacyInlineRecords = manifest.Records - .Where(record => !string.IsNullOrWhiteSpace(record.Key)) - .GroupBy(record => record.Key, StringComparer.Ordinal) - .ToDictionary(group => group.Key, group => group.Last(), StringComparer.Ordinal); - foreach (var record in manifest.Records.Where(ShouldHydrate)) - { - var catalog = ResolveLegacyUncataloguedRecordCatalog(record, manifest.SchemaCatalog); - reports.Add(Registry.ResolvePersistedSchemaReport(record.SchemaId, catalog)); - var stored = ToStoredDocument( - record, - catalog, - (type, payload) => CultDocumentMessagePackSerialization.DeserializeUntyped(type, payload, Registry)); - loaded[stored.Key.Value] = stored; - } - - if (!_recordDirectory.Exists) - return manifest.Records; - - var pages = _recordDirectory.EnumerateFiles("*.msgpack") - .OrderBy(file => file.Name, StringComparer.Ordinal) - .Select(file => CultDocumentMessagePackSerialization.DeserializePersistedRecord(ReadAllBytesShared(file.FullName))) - .ToArray(); - LoadPersistedRecords(pages.Where(ShouldHydrate).ToArray(), manifest.SchemaCatalog, loaded, reports); - return manifest.Records.Concat(pages).ToArray(); - } - private void LoadRecordPages( CultPersistedRecord[] records, IReadOnlyCollection catalogEntries, @@ -359,7 +309,17 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore var storedRecords = new CultStoredDocument?[records.Length]; var pageBytes = tracePages ? new long[records.Length] : Array.Empty(); var pageElapsedTicks = tracePages ? new long[records.Length] : Array.Empty(); - Parallel.For( + try + { + ReadPages(); + } + catch (AggregateException aggregate) + { + // The page's own failure propagates, not Parallel.For's wrapper. + ExceptionDispatchInfo.Capture(aggregate.Flatten().InnerExceptions[0]).Throw(); + } + + void ReadPages() => Parallel.For( 0, records.Length, new ParallelOptions { MaxDegreeOfParallelism = Math.Min(Environment.ProcessorCount, 8) }, @@ -367,18 +327,11 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore { var started = tracePages ? Stopwatch.GetTimestamp() : 0L; var metadata = records[index]; - var path = RecordPath(metadata.Key); - if (!File.Exists(path)) - return; - var pagePayload = ReadAllBytesShared(path); - var record = CultDocumentMessagePackSerialization.DeserializePersistedRecord(pagePayload); - if (!string.Equals(record.Key, metadata.Key, StringComparison.Ordinal)) - throw new InvalidDataException($"Record page '{path}' contains key '{record.Key}', expected '{metadata.Key}'."); - var catalog = ResolveLegacyUncataloguedRecordCatalog(record, catalogEntries); - recordReports[index] = Registry.ResolvePersistedSchemaReport(record.SchemaId, catalog); + var record = ReadPersistedRecordPage(metadata, out var pagePayload); + recordReports[index] = Registry.ResolvePersistedSchemaReport(record.SchemaId, catalogEntries); storedRecords[index] = ToStoredDocument( record, - catalog, + catalogEntries, (type, payload) => CultDocumentMessagePackSerialization.DeserializeUntyped(type, payload, Registry)); if (tracePages) { @@ -422,77 +375,6 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore } } - private void LoadPersistedRecords( - CultPersistedRecord[] records, - IReadOnlyCollection catalogEntries, - Dictionary loaded, - List reports) - { - var recordReports = new CultSchemaMigrationReport[records.Length]; - var storedRecords = new CultStoredDocument[records.Length]; - Parallel.For( - 0, - records.Length, - new ParallelOptions { MaxDegreeOfParallelism = Math.Min(Environment.ProcessorCount, 8) }, - index => - { - var record = records[index]; - var catalog = ResolveLegacyUncataloguedRecordCatalog(record, catalogEntries); - recordReports[index] = Registry.ResolvePersistedSchemaReport(record.SchemaId, catalog); - storedRecords[index] = ToStoredDocument( - record, - catalog, - (type, payload) => CultDocumentMessagePackSerialization.DeserializeUntyped(type, payload, Registry)); - }); - - for (var index = 0; index < storedRecords.Length; index++) - { - reports.Add(recordReports[index]); - var stored = storedRecords[index]; - loaded[stored.Key.Value] = stored; - } - } - - private bool ShouldHydrate(CultPersistedRecord record) => - HydrationFilter?.Invoke(ToMetadata(record)) ?? true; - - private static CultPersistedRecordMetadata ToMetadata(CultPersistedRecord record) => - new(record.Key, record.SchemaId, record.StoredAt); - - private void PublishSelected( - IReadOnlyDictionary loaded, - HashSet selectedKeys) - { - foreach (var pair in loaded) - { - if (_dirtyKeys.ContainsKey(pair.Key) || _deletedKeys.ContainsKey(pair.Key)) - continue; - - if (Entries.TryGetValue(pair.Key, out var existing) && - string.Equals(existing.StoredAt, pair.Value.StoredAt, StringComparison.Ordinal) && - string.Equals(existing.Descriptor.SchemaId, pair.Value.Descriptor.SchemaId, StringComparison.Ordinal)) - continue; - - Entries[pair.Key] = pair.Value; - if (existing == null) - EntryAdded.OnNext(pair.Value); - else - EntryUpdated.OnNext(pair.Value); - } - - foreach (var missingKey in selectedKeys.Where(key => !loaded.ContainsKey(key)).ToArray()) - { - if (_dirtyKeys.ContainsKey(missingKey) || _deletedKeys.ContainsKey(missingKey)) - continue; - if (Entries.TryRemove(missingKey, out var removed)) - EntryDeleted.OnNext(removed); - _hydratedKeys.Remove(missingKey); - } - - foreach (var key in loaded.Keys) - _hydratedKeys.Add(key); - } - private static CultPersistedRecord ToIndexRecord(CultStoredDocument stored) => new() { Key = stored.Key.Value, @@ -501,119 +383,115 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore Payload = Array.Empty() }; - private static CultPersistedRecord ToIndexRecord(CultPersistedRecord record) => new() + private static CultPersistedRecord ToContentAddressedIndexRecord( + CultPersistedRecord record, + byte[] pagePayload) => new() { Key = record.Key, SchemaId = record.SchemaId, StoredAt = record.StoredAt, - Payload = Array.Empty() + Payload = HashPayload(pagePayload) }; - private CultPersistedStoreSnapshot ReadManifest() + private CultPersistedStoreSnapshot ReadManifest() => ParseManifest(ReadManifestBytes()); + + private byte[]? ReadManifestBytes() => File.Exists(_manifestFile.FullName) ? ReadAllBytesShared(_manifestFile.FullName) : null; + + // A manifest that cannot be re-read (a writer mid-replace) has moved. + private bool ManifestMoved(byte[]? read) { - if (File.Exists(_manifestFile.FullName)) - { - var snapshot = CultDocumentMessagePackSerialization.DeserializeSnapshot(ReadAllBytesShared(_manifestFile.FullName)); - if (snapshot.SchemaCatalog.Length > 0 || snapshot.Records.Length == 0) - { - return snapshot; - } - } - - if (File.Exists(_manifestFile.FullName)) - { - return CultDocumentMessagePackSerialization.DeserializeSnapshot(ReadAllBytesShared(_manifestFile.FullName)); - } - - return new CultPersistedStoreSnapshot - { - FormatVersion = "cultcache.store.v1.directory", - SchemaCatalog = Array.Empty(), - Records = Array.Empty() - }; - } - - // Explicit compatibility path for records written before catalog precommit existed. - private IReadOnlyCollection ResolveLegacyUncataloguedRecordCatalog( - CultPersistedRecord record, - IReadOnlyCollection manifestCatalog) - { - if (manifestCatalog.Any(entry => string.Equals(entry.SchemaId, record.SchemaId, StringComparison.Ordinal))) - { - return manifestCatalog; - } - - var schemaVersion = TryReadPayloadSchemaVersion(record.Payload); - if (string.IsNullOrWhiteSpace(schemaVersion)) - { - return manifestCatalog; - } - - var schemaName = InferSchemaName(schemaVersion); - if (string.IsNullOrWhiteSpace(schemaName)) - { - return manifestCatalog; - } - - return manifestCatalog - .Append(new CultSchemaCatalogEntry - { - SchemaId = record.SchemaId, - SchemaName = schemaName, - SchemaVersion = schemaVersion, - ContentHash = record.SchemaId, - CanonicalSchemaJson = "", - CompatibleSchemaIds = Array.Empty(), - Members = Array.Empty() - }) - .ToArray(); - } - - private static string TryReadPayloadSchemaVersion(byte[] payload) - { - if (payload == null || payload.Length == 0) - { - return ""; - } - try { - var reader = new MessagePackReader(payload); - if (reader.NextMessagePackType != MessagePackType.Array) - { - return ""; - } - - if (reader.ReadArrayHeader() <= 0) - { - return ""; - } - - return reader.NextMessagePackType == MessagePackType.String - ? reader.ReadString() ?? "" - : ""; + var current = ReadManifestBytes(); + return read == null ? current != null : current == null || !read.SequenceEqual(current); } - catch + catch (Exception) { - return ""; + return true; } } - private static string InferSchemaName(string schemaVersion) + // A page the manifest named that vanished or was replaced. It is a writer committing mid-load only when the manifest + // moved too; under an unchanged manifest the same failure is corruption and is thrown as it is. + private static bool IsTornGeneration(Exception exception) => + exception is InvalidDataException or FileNotFoundException or DirectoryNotFoundException; + + private InvalidOperationException Unsettled(Exception? last) => new( + $"Directory store {_manifestFile.FullName} changed under every one of {UnleasedLoadAttempts} unlocked loads; it did not settle.", last); + + // A missing manifest is an empty store. An existing manifest in any other format is refused. + private CultPersistedStoreSnapshot ParseManifest(byte[]? manifestBytes) { - var versionMarker = schemaVersion.LastIndexOf(".v", StringComparison.Ordinal); - return versionMarker > 0 && versionMarker + 2 < schemaVersion.Length && char.IsDigit(schemaVersion[versionMarker + 2]) - ? schemaVersion.Substring(0, versionMarker) - : ""; + if (manifestBytes == null) + { + return new CultPersistedStoreSnapshot + { + FormatVersion = IndexedFormatVersion, + SchemaCatalog = Array.Empty(), + Records = Array.Empty() + }; + } + + var snapshot = CultDocumentMessagePackSerialization.DeserializeSnapshot(manifestBytes); + if (!string.Equals(snapshot.FormatVersion, IndexedFormatVersion, StringComparison.Ordinal)) + { + throw new InvalidOperationException( + $"Directory store {_manifestFile.FullName} is {snapshot.FormatVersion}; only {IndexedFormatVersion} is readable."); + } + + return snapshot; } - private string RecordPath(string key) => Path.Combine(_recordDirectory.FullName, $"{HashKey(key)}.msgpack"); + private string ContentAddressedRecordPath(CultPersistedRecord metadata) + { + if (metadata.Payload == null || metadata.Payload.Length != 32) + throw new InvalidDataException($"Record '{metadata.Key}' has no committed SHA-256 page identity."); + var hash = BitConverter.ToString(metadata.Payload).Replace("-", string.Empty).ToLowerInvariant(); + return Path.Combine(_recordDirectory.FullName, $"{hash}.msgpack"); + } - private static string HashKey(string key) + private static byte[] HashPayload(byte[] payload) { using var sha256 = SHA256.Create(); - var hash = sha256.ComputeHash(Encoding.UTF8.GetBytes(key)); - return BitConverter.ToString(hash).Replace("-", string.Empty).ToLowerInvariant(); + return sha256.ComputeHash(payload); + } + + private void DeleteUnreferencedRecordPages(IEnumerable durableRecords) + { + var referenced = durableRecords + .Select(ContentAddressedRecordPath) + .ToHashSet(StringComparer.OrdinalIgnoreCase); + foreach (var file in _recordDirectory.EnumerateFiles("*.msgpack")) + { + if (referenced.Contains(file.FullName)) + continue; + try + { + File.Delete(file.FullName); + } + catch (IOException) + { + // An orphan is harmless: no committed manifest references it. + } + catch (UnauthorizedAccessException) + { + // Cleanup is best effort and never changes the committed generation. + } + } + } + + private CultPersistedRecord ReadPersistedRecordPage(CultPersistedRecord metadata, out byte[] pagePayload) + { + var path = ContentAddressedRecordPath(metadata); + if (!File.Exists(path)) + throw new InvalidDataException($"Committed record page '{path}' is missing from the selected manifest generation."); + pagePayload = ReadAllBytesShared(path); + if (!HashPayload(pagePayload).SequenceEqual(metadata.Payload)) + throw new InvalidDataException($"Record page '{path}' does not match its committed content hash."); + var record = CultDocumentMessagePackSerialization.DeserializePersistedRecord(pagePayload); + if (!string.Equals(record.Key, metadata.Key, StringComparison.Ordinal)) + throw new InvalidDataException($"Record page '{path}' contains key '{record.Key}', expected '{metadata.Key}'."); + return record; } private static void WriteFileAtomically(string path, byte[] payload) @@ -662,6 +540,46 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore return buffer.ToArray(); } + // Only a writer creates the records folder and the lock, so opening the store creates nothing. A reader opens an + // existing lock and returns null when there is none; that does not mean no writer committed (a store may be copied + // without its lock, and a first writer may create it mid-load), so an unleased load checks that it settled. + private FileStream? AcquireCommitLease(bool wait, bool create) + { + if (create) + Directory.CreateDirectory(_recordDirectory.FullName); + var lockPath = Path.Combine(_recordDirectory.FullName, ".commit.lock"); + var started = Stopwatch.StartNew(); + while (true) + { + try + { + return new FileStream( + lockPath, + create ? FileMode.OpenOrCreate : FileMode.Open, + FileAccess.ReadWrite, + FileShare.None, + bufferSize: 1, + FileOptions.WriteThrough); + } + catch (FileNotFoundException) when (!create) + { + return null; + } + catch (DirectoryNotFoundException) when (!create) + { + return null; + } + catch (IOException) when (!wait) + { + return null; + } + catch (IOException) when (started.Elapsed < TimeSpan.FromSeconds(30)) + { + Thread.Sleep(10); + } + } + } + private static void ReplaceExistingFile(string sourcePath, string destinationPath) { for (var attempt = 0; attempt < 3; attempt++) @@ -681,6 +599,6 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore } } - File.Copy(sourcePath, destinationPath, overwrite: true); + File.Replace(sourcePath, destinationPath, null); } } diff --git a/src/GameCult.Caching.MessagePack/GameCult.Caching.MessagePack.csproj b/src/GameCult.Caching.MessagePack/GameCult.Caching.MessagePack.csproj index 71fcc969..c2bd1c1b 100644 --- a/src/GameCult.Caching.MessagePack/GameCult.Caching.MessagePack.csproj +++ b/src/GameCult.Caching.MessagePack/GameCult.Caching.MessagePack.csproj @@ -1,13 +1,13 @@ File-based MessagePack backing store for CultCache + $(NoWarn);CS1591 - - + + diff --git a/src/GameCult.Caching.MessagePack/README.md b/src/GameCult.Caching.MessagePack/README.md index 660da700..c69ab887 100644 --- a/src/GameCult.Caching.MessagePack/README.md +++ b/src/GameCult.Caching.MessagePack/README.md @@ -10,16 +10,16 @@ format for the attribute-first `GameCult.Caching` stack. - `CultDocumentResolver` - `CultRecordRefFormatter` -When a project declares `[CultDocument]` models, wire in the -`GameCult.Caching.MessagePack.Generator` analyzer there as well. It emits -assembly-local metadata providers for the cache registry. Cult documents then -serialize through explicit generated slot codecs, while the store snapshot -format is written by hand against `MessagePackWriter`/`MessagePackReader`. +Cult document payloads serialize through `MessagePackSerializer` with the +options of the document's assembly (`CultDocumentMessagePackSerialization.OptionsFor`), +after the reflective `CultDocumentRegistry` has accepted the document shape. The +store snapshot format is written by hand against +`MessagePackWriter`/`MessagePackReader`. ## What It Does - stores whole CultCache snapshots in MessagePack -- serializes plain attributed document payloads through explicit generated slot codecs +- serializes attributed document payloads as MessagePack `[Key(n)]` arrays - keeps explicit `CultRecordRef` values compact on disk/wire - preserves the cache/store split where metadata lives outside the domain model @@ -54,7 +54,5 @@ await cache.PullAllBackingStoresAsync(); - Payload bytes are MessagePack. Store metadata and schema catalogs are also persisted through hand-written MessagePack array layouts in this package. -- The generator package now targets attributed document types instead of - `DatabaseEntry` subclasses. - The raw CultNet document lane should treat these payload bytes as already blessed, not decode and re-encode them for sport. diff --git a/src/GameCult.Caching/Contracts/cultcache-persistence-format.md b/src/GameCult.Caching/Contracts/cultcache-persistence-format.md index 66d0e135..e8231c10 100644 --- a/src/GameCult.Caching/Contracts/cultcache-persistence-format.md +++ b/src/GameCult.Caching/Contracts/cultcache-persistence-format.md @@ -39,6 +39,36 @@ These are **not** domain fields and should not be exposed on app-facing types: Those belong to the CultCache persistence layer. +## Transaction Visibility + +A multi-record commit is an explicit batch of records that resolve to one home +store. The store commits the batch as one durable step. Nothing in the batch is +visible, to the committing flow or to observers, until the store has accepted +it; a failed commit changes nothing on disk or in memory. + +A batch may carry conditions: per-record `(schemaId, storedAt)` identity, or +the whole store unchanged since it was last loaded. Conditions are evaluated +under the store's exclusive lock against what is durably on disk. A failed +condition is a lost race, not an error: the commit reports it and writes +nothing. + +**Conditional commit is the only safe multi-process write.** A plain flush and +an unconditional commit are last-writer-wins and write the same bytes: a single +file is replaced by the writer's whole view, a directory manifest takes the +writer's staged keys over the current manifest. An unconditional commit also +persists writes staged earlier and leaves the store clean. Processes sharing a +store must all use conditional commit. + +A store that fails to hydrate on open (corrupt bytes, unresolvable schema) +makes the open throw and is left byte-identical; it is never silently +overwritten. Within one process the lock order is the cache's gate, then the +store's file lock; a store's I/O blocks its cache's readers. + +C# implements batch and conditional commit (`CultCache.Commit`, `TryCommit`, +`CultCacheBatch`). Rust implements both. TypeScript and Python implement +neither yet. Routing, batches and conditions are specified in +`cultcache-store-composition.md`. + ## Canonical Store Shape A persisted CultCache file should contain: @@ -335,12 +365,14 @@ The v1 concurrent single-file policy is: lock. 2. Writers take an exclusive sidecar lock derived from the `.cc` path. 3. Writers re-read the current snapshot after taking the lock. -4. Writers merge or reject local staged changes against the latest generation. +4. Writers using conditional commit evaluate their conditions against the + latest snapshot and write nothing if one fails. A plain flush and an + unconditional commit compare nothing and are last-writer-wins. 5. Writers write a temp file, flush it, atomically replace the `.cc` file, and release the lock. -This preserves the key invariant: every reader sees a complete snapshot, and no -writer commits over unseen data without passing through the merge/reject point. +This preserves the key invariant: every reader sees a complete snapshot. Only a +conditional commit is protected against committing over unseen data. ## Live Change Observation diff --git a/src/GameCult.Caching/Contracts/cultcache-schema-compatibility.md b/src/GameCult.Caching/Contracts/cultcache-schema-compatibility.md index 26e9e205..297c3184 100644 --- a/src/GameCult.Caching/Contracts/cultcache-schema-compatibility.md +++ b/src/GameCult.Caching/Contracts/cultcache-schema-compatibility.md @@ -26,6 +26,53 @@ The canonical fixtures currently exercised in C# are: See `GameCult.Caching.Tests/BackingStoreTests.cs` for the canonical fixture documents and the expected receipts. +## Slots + +A member's slot is its MessagePack `[Key(n)]` integer. MessagePack's `[Key]` and +`[IgnoreMember]` are the single slot authority, so `GameCult.Caching` depends on +`MessagePack.Annotations`. Payloads are written by `MessagePackSerializer`, so +the registry refuses a document type without `[MessagePackObject]` and a +non-public document type without `[MessagePackObject(AllowPrivate = true)]`. +A member's type name is the CLR full name; a nested type is written `Outer+Inner`. +The registry also refuses a public member without `[Key]` or `[IgnoreMember]` +(get-only properties included; MessagePack throws on them), a string `[Key]`, +two members sharing a slot (including `new`-hidden members), a hidden persisted +member, and an override whose `[Key]` or `[IgnoreMember]` differs from its base +declaration's (MessagePack reads the base declaration). + +Visibility follows MessagePack's rule: the registry accepts what MessagePack +round-trips and refuses what it would silently lose. The registry persists +every keyed public field and property, get-only ones included, because +MessagePack writes them all. A `[Key]` on a non-public member without +`AllowPrivate` is refused because MessagePack skips that member. With +`AllowPrivate`, MessagePack also reads non-public members, so every non-public +instance field or property must be `[IgnoreMember]`. + +A class needs a constructor MessagePack can call: the `[SerializationConstructor]` +if one is marked, otherwise the longest constructor (public unless +`AllowPrivate`) whose every parameter takes the keyed member at its position +(`[Key(0)]`, `[Key(1)]`, ...). A parameterless constructor always qualifies; a +primary constructor usually does not. MessagePack reads a member back through +its setter, or through that constructor when the member's slot is below the +constructor's parameter count. A setter counts when it is public, or any setter +(private, internal, init-only) under `AllowPrivate`; a readonly field counts as +writable under `AllowPrivate` only. A keyed member MessagePack cannot read back +either way is refused: a get-only property, a readonly field without +`AllowPrivate`, or a property whose setter is non-public without +`AllowPrivate`, when the constructor does not fill it. + +## Which schema a record carries + +A record is written under the schema of its runtime type. Writing a derived +document through a base-typed handle or generic parameter (`UpsertAsync` +with a `Weapon`) persists the `Weapon` schema, and the record reloads as a +`Weapon`. A runtime type without `[CultDocument]` is refused. + +Typed lookups and watches (`Get`, `GetAll`, `GetByName`, +`GetByIndex`, `GetGlobal`, `Watch`) match every record whose runtime +type is assignable to `T`. A single-result lookup with several candidates +throws. + ## Soft-migratable drift CultCache accepts compatible drift only when the local reader can still map the diff --git a/src/GameCult.Caching/Contracts/cultcache-store-composition.md b/src/GameCult.Caching/Contracts/cultcache-store-composition.md new file mode 100644 index 00000000..8c4fc529 --- /dev/null +++ b/src/GameCult.Caching/Contracts/cultcache-store-composition.md @@ -0,0 +1,273 @@ +# CultCache Store Composition + +How one cache composes its backing stores: where a record lives, when a store +is read or written, what a global is, how a multi-record commit lands, and who +owns serialization options. `GameCult.Caching` in C# is the reference; the +runtime table at the end says which runtime implements what. + +None of this changes bytes on disk. Every store file is a complete +`cultcache.store.v1` snapshot (single file) or a v4 directory manifest, exactly +as `cultcache-persistence-format.md` describes. No file records which cache or +which route wrote it. + +## One home store per document type + +- A document type has exactly one home store. A cache attaches stores with an + optional list of home types: `AddBackingStore(store, params Type[] homes)`. +- The home of a document is chosen from its **runtime type** + (`document.GetType()`), never from a generic type parameter at the call site. + Among routed stores whose home types are assignable from the runtime type, the + most derived home type wins; otherwise the untyped store; otherwise none. +- C# routes by assignable CLR type. TypeScript, Rust and Python route by the + exact type string they already use. +- A second untyped store is a registration error. A home type already claimed by + another store, by exact type equality, is a registration error. Routes to a + base type and a derived type may coexist; the most derived wins. +- A write whose document has no home, once any store is attached, is an error. + A cache with zero stores is an in-memory cache and needs no home. +- Nothing is mirrored or replicated. A write touches exactly one store. +- This is the contract in every runtime. A runtime that still mirrors is + non-conforming until it deletes its mirrors. + +## Attachment is hydration + +- Attaching a store reads it. There is no interval in which a store is attached + but unread, and no push of existing records into a newly attached store. +- A store must be clean (`IsDirty` false) when attached. +- A record's home cannot change after it is admitted. Attaching a store that + would move the home of a record already in the cache is an error. The + consequence is the attach order: routed stores first, the untyped store last. +- A record loaded from a store that is not its home is refused at load, naming + the key and both stores. +- Pulling again after attach is a re-pull. It does not erase mutations staged in + a dirty store. A dirty single-file store skips the re-pull entirely; a missing + single file keeps staged writes dirty and does not drop loaded records. +- A load publishes changes to `Watch` subscribers and fires `OnUpdate`. A local + write or commit publishes to `Watch` only; `OnUpdate` fires for loads alone. +- Changes published through `Watch`/`WatchRecord` carry a `Sequence`: a + per-cache, in-memory number assigned when the cache admits the change, under + its gate. It increases in admission order and is not persisted. + `OnUpdate(previous, current)` carries none. Streams CultNet derives from + `OnUpdate` (`CultNetDatabase.WatchAllChanges`, the subscription server, + database-backed Mesh handles) have no ordering guarantee and no stale + protection. +- `GetWithSequence(key)` returns the document and the cache's current + `Sequence`, read together under the gate: every change with a `Sequence` at + or below it is reflected in the document. +- Adding the optional `sequence` constructor argument to + `CultCacheDocumentChange` changed its binary signature; no external + constructor is known. +- Observers and `OnUpdate` run after the cache releases its gate, never under + it: an observer may read or write the cache from any thread. Each call + publishes exactly its own changes, on its own thread, after releasing the gate + and before it returns; no call publishes another call's changes or waits on + another call's delivery. A write made by an observer is its own call and + publishes its own changes before that write returns. A direct `PullAll` on an + attached store is such a call. +- Cross-thread delivery order is not guaranteed. A consumer that keeps a latest + value per record ignores a change whose `Sequence` is not above the one it + already applied for that record. It subscribes first, then takes + `GetWithSequence` and adopts that `Sequence` as applied; a change delivered + after subscribing with a `Sequence` at or below the read is already in the + document and is dropped. CultMesh cache-record mirrors (`ObserveAsync`, + `ReactiveAsync`, and `RefreshAsync` on both) do this. +- Not stale-protected: schema-alias handles (`AsSchemaAlias`) get no + `Sequence`; removals do not advance a mirror's applied `Sequence`, and the + mirrors do not surface removals. +- An `OnUpdate` handler exception is rethrown to that caller after all of that + call's changes are delivered (an `AggregateException` if several threw). If + the call itself failed, its own exception is rethrown and handler exceptions + are dropped. A `Watch` subscriber exception is not rethrown; it follows R3's + unhandled-exception handling. A throwing handler cannot undo the store's + adoption of a load; the store and cache already agree when publication starts. +- Pulling all stores pulls every attached store even if a handler throws during + one store's load, then rethrows. +- Hydration failure on open is loud: a corrupt store file, or a record whose + schema the registry cannot resolve, makes the open throw and leaves the file + byte-identical. Consumers never delete and rewrite a store they failed to + open. + +## Locking + +- One lock order: the cache's gate, then the store's lock. A store attached to a + cache takes that cache's gate as its own lock, so a store's load callback into + the cache cannot take the two out of order. A store must not call `Loaded` + while holding its own lock outside the cache's hold. +- A thread holds one cache's gate at a time: entering a cache's hold inside + another cache's hold throws. +- A store doing I/O on behalf of its cache (pull, flush, commit) holds the gate, + so that cache's readers wait for the I/O. The file lock (`.lock` or the + directory commit lease) is always taken inside the gate and released before a + load is handed to the cache. +- Direct calls on an attached store (`Push`, `Delete`, `PushAll`, `CommitBatch`, + `PullAll`) take the same gate, so they wait for that cache's pull, flush or + commit, and the cache's flush on dispose runs under it. +- In every runtime, a store call must not wait for a write to its own cache to + finish: that write queues behind the operation that is waiting for it, a + cycle. A handler or subscriber that writes back without the store waiting for + it is fine; its write queues and lands after the operation. Python refuses a + mutation started on a thread already inside one of that cache's mutations + with an error; no runtime detects a wait that escapes to another thread or + async context. + +## Dirtiness: loading never writes + +- A store's `IsDirty` derives only from its own writes and deletes. Pulling sets + it false. +- A cache's `IsDirty` is whether any attached store is dirty, or, with no stores, + whether it has been mutated in memory. +- Opening a store and flushing without a mutation leaves every file + byte-identical. +- A mutation reaches the home store before the cache's in-memory view changes. + If the store refuses the write, the cache holds nothing new and publishes no + change. + +## Read-only stores + +- A store can be opened read-only. Writes, deletes and full pushes on it throw + and name the store. +- A write whose home is a read-only store throws before anything changes. +- Flushing skips read-only stores. + +## Lookups and watches honor inheritance + +`Get`, `GetAll`, `GetByName`, `GetByIndex`, `GetGlobal` and +`Watch` match every document whose runtime type is assignable to `T`. A +lookup that returns one document and finds several candidates throws, naming +them. Indexes stay keyed by concrete type; lookups enumerate assignable keys. + +## Globals + +- `[CultGlobal]` means: at most one record of the type per cache, `GetGlobal()` + returns it or `null`, and a write without a handle keys it + `global:{schemaId}`. +- The cache never creates a global. Constructing a cache, attaching a store and + pulling produce no record the store did not contain. The code path that begins + a global's lifecycle creates it with an ordinary upsert. +- A second record of a global type, through a write or a load, is refused + before any store is touched. +- TypeScript and Python key the single record `__global__`; C# keys it + `global:{schemaId}`. The divergence is recorded, not changed, because changing + it changes bytes. Rust has no global concept. + +## Atomic commit: an explicit batch + +- A multi-record commit is an explicit batch value staged by the caller and + committed with `Commit`. Every record in the batch must resolve to one home + store; a batch spanning two homes throws before any store is touched. +- The store commits the batch as one durable step: one atomic replace for a + single file, pages then manifest for a directory store. A failed commit + restores the store's staging and changes nothing in memory. +- Nothing in a batch is visible, to the staging code or to observers, until the + store has accepted it. Reads during staging see committed state. There is no + ambient transaction and no in-flight overlay. +- After the store accepts, each record is admitted through the same path as a + single write (home, read-only and global checks included) and one change is + published per record. +- A cache with no stores admits a batch in memory. + +## Conditional commit (compare-exchange) + +A batch may carry conditions, evaluated under the home store's exclusive lock +against what is durably on disk at that moment: + +- `Expect(key, current)`: the record at `key` must still be the one this cache + observed, or, with `current` null, must be absent. Only named keys are + constrained; unrelated concurrent writes do not fail the commit. Passing any + instance other than the one the cache holds throws. +- `ExpectUnchanged()`: the store's persisted record set must equal what this + cache last loaded from it, compared as the ordered list of + `(key, schemaId, storedAt)`. Any insert, delete or replace fails it. + +Record identity for conditions is `(schemaId, storedAt)`. It is sound because +every write to a key mints a `storedAt` strictly later than the record it +replaces; writers bump a minted timestamp by one tick when it is not later. + +- A failed condition is a lost race, not an error: `Commit` returns false and + nothing is written, changed in memory, or published. `TryCommit` makes one + non-blocking lock attempt and reports `Contended` instead of waiting. On a + directory store the lock is the commit lease that pulls also take, so + `Contended` can mean another cache or process is reading or committing. The + attempt itself runs inside the cache gate, so `TryCommit` may first block + behind a pull or flush of the same cache, for up to that operation's lease + wait bound (30 seconds). +- A conditional commit on a store with staged single-record writes throws: + flush first. +- Single-file stores lock a sidecar `.lock` opened exclusively. Directory + stores use their existing commit lease. +- A directory store with no `.commit.lock` (no writer has created it yet, or + the store was copied without it) gives a pull no lease, and opening it + creates none. Such a pull re-reads the manifest after loading its pages and + reloads when the manifest changed or a page it named was missing or did not + match its content address, at most five times; a pull that has not settled + by then throws. + +**A plain flush and an unconditional commit are last-writer-wins: per file for +single-file stores, per key for directory stores.** Both run +under the same lock, so two writers never interleave bytes, but they compare +nothing. Only a conditional commit (`Expect` or `ExpectUnchanged`) protects +against another writer; processes sharing a store must all use it. + +- A single-file store writes this cache's whole view of the file: a record + another writer added since this cache last pulled is gone. +- A directory store writes this cache's staged keys onto the current manifest: + another writer's unrelated keys survive, a key both wrote holds the last + write. +- An unconditional commit writes exactly what a flush of the same staged state + plus the batch would write. It also persists any single writes staged earlier + in that store, and leaves the store clean. +- A conditional commit lands the batch onto the file as it is under the lock. + +A store is written by one runtime at a time. Whether a Rust `fs2` lock and a C# +exclusive open of the same lock file exclude each other is not established, and +another runtime need not obey the `storedAt` rule. + +## Serialization options + +- Options belong to the assembly that declares a document. That assembly names + its formatter resolvers with `[assembly: CultCacheFormatterResolver(typeof(R))]`; + the payload serializer resolves options from the document's assembly. There is no mutable static, no registration call, and no load-order + rule. +- A generic document's options come from the generic definition's assembly: + `Doc` uses the resolvers `Doc`'s assembly declares, never `X`'s. A formatter + for `X` that `Doc` needs is declared in `Doc`'s assembly. +- An assembly that declares nothing, and the store envelope, use the base + options. +- Deserialization runs under untrusted-data security. `CultRecordRef` is a + permitted dictionary key, compared by its key string. + +## Value-type encoding + +Vector-like value types (Unity.Mathematics, consumer structs) encode as +**fixed-length positional arrays of primitive components**: a two-component +float vector is `[f32, f32]`, a three-component one `[f32, f32, f32]`, an +integer pair `[int, int]`. No ext types, no maps, no names. The persisted member +type name is the CLR full name, the same rule as `System.Int32`. A reader in +another runtime needs only the component count and primitive type. CultMath has +no serialization; a CultMath encoding, if added, uses this shape. + +## Runtime status + +| Rule | C# | Rust | TypeScript | Python | +|---|---|---|---|---| +| One home store, no mirrors | implemented | implemented (exact type ids; `add_backing_store` returns `Err` on a second generic store or a claimed type; an empty type list is the generic store) | implemented (exact type ids; `addBackingStore` throws on a second generic store or a claimed type; no types is the generic store) | implemented (exact type ids; `add_backing_store`/`add_generic_store` raise on a claimed type or a second generic store; an empty type list is the generic store) | +| Zero stores is in memory | implemented | implemented | implemented | implemented | +| Home cannot move after admission | implemented | implemented (`add_backing_store` returns `Err` and attaches nothing) | implemented (`addBackingStore` rejects and attaches nothing; it runs in the cache's serial queue, so it cannot move a write that is in flight) | implemented (`add_backing_store`/`add_generic_store` raise and attach nothing; they hold the cache's lock, so they cannot move a write that is in flight) | +| Refused at load when not home | implemented | implemented (`pull_all_backing_stores` returns `Err` naming key and both stores; `load_soa` returns `Err` for a type with no home once a store is attached; both admit nothing and leave the prior view intact) | implemented (`pullAllBackingStores` throws naming key and both stores; admits nothing) | implemented (`pull_all_backing_stores` raises naming key and both stores; admits nothing) | +| Refused write changes neither store nor cache | implemented | implemented (`put`, `put_envelope`, `put_raw_envelope`, `put_prepared_batch` and `delete` validate before the store call and apply infallibly after it; there are no user accessors) | implemented (home, `__global__` key and name and index accessors run before the store call; overwrite and delete use lookup values stored at admission; attach, pull, registration, writes and deletes run through one per-cache promise queue; one of those started from inside an operation queues behind it, so a store that waits for it never finishes (see Locking)) | implemented (same validation order and stored lookup keys as TypeScript; attach, pull, registration, writes, deletes, name and index resolution and snapshots hold one lock per cache; a mutation started on a thread already inside one of that cache's mutations raises, reads stay reentrant; a wait on another thread is not detected (see Locking)) | +| Attach hydrates; loading never writes | implemented | attach does not read (call `pull_all_backing_stores`); pull loads without writing | attach does not read (call `pullAllBackingStores`); pull loads without writing | attach does not read (call `pull_all_backing_stores`); pull loads without writing | +| Runtime type decides schema | implemented | n/a (explicit type ids) | n/a | n/a | +| Assignable lookups and watches | implemented | exact type ids | exact type ids | exact type ids | +| Globals never invented, singleton | implemented | no globals | single `__global__`; a write under any other key is refused before any store is touched; replacing writes `__global__` in place; a second persisted global (any keys) is refused on pull; invents nothing; load compatibility: a pull adopts a global stored under another key as `__global__` without writing, and the first write of it pushes `__global__`, then deletes the legacy record, deleting `__global__` again if that delete fails (a delete of it removes only the legacy record). A crash between the two steps, or a failed compensation, leaves both records on disk, and the next pull refuses them loudly as a second global. Removed once no store holds a non-`__global__` global | single `__global__`; `put`, `put_envelope` and `put_envelopes` refuse any other key before any store is touched; a second persisted global (any keys) is refused on pull; invents nothing; load compatibility: a pull adopts a global stored under another key as `__global__` without writing, and the first write of it pushes `__global__`, then deletes the legacy record, deleting `__global__` again if that delete fails (a delete of it removes only the legacy record). A crash between the two steps, or a failed compensation, leaves both records on disk, and the next pull refuses them loudly as a second global. Removed once no store holds a non-`__global__` global | +| Explicit batch commit | implemented | `put_prepared_batch`: staged records must share one home; that store receives one `push_all` of its own held records plus the batch; empty key or `stored_at` refused first; all-or-nothing on stores whose `push_all` is atomic, which the trait requires of every store: `push_all` has no default, and the bundled single-file and redb stores replace in one step | not implemented | `put_envelopes`, one type per call; every record validated before one `push_all`, applied only after the store accepts; atomicity on disk is the store's `push_all` | +| Conditional commit | implemented | store-level `compare_exchange` family | not implemented | not implemented | +| Per-assembly options, value-type arrays | contract | n/a | n/a | n/a | + +A "contract" cell means this document is ahead of the C# code. + +Opening a store always hydrates it. A consumer that wants a store to hold only +what it writes removes the records it does not keep, upserts, and flushes. On a +single-file store the flush replaces the file in one atomic step. On a directory +store the flush writes pages, then the manifest, under the commit lease, merging +its staged keys and removals onto the current manifest; a key another writer +added since the pull survives. There is no open-without-reading option. diff --git a/src/GameCult.Caching/CultCache.cs b/src/GameCult.Caching/CultCache.cs index d3da3096..5e8c8f6d 100644 --- a/src/GameCult.Caching/CultCache.cs +++ b/src/GameCult.Caching/CultCache.cs @@ -1,217 +1,98 @@ using System; using System.Collections.Concurrent; using System.Collections.Generic; +using System.Diagnostics; +using System.Globalization; using System.IO; using System.Linq; using System.Reflection; using System.Runtime.CompilerServices; +using System.Runtime.ExceptionServices; using System.Security.Cryptography; using System.Text; +using System.Threading; using System.Threading.Tasks; -using GameCult.Logging; +using MessagePack; using R3; namespace GameCult.Caching { - /// - /// Persisted schema metadata embedded in a CultCache backing store. - /// public sealed class CultSchemaCatalogEntry { - /// - /// Gets or sets the content-derived schema identifier. - /// public string SchemaId { get; set; } = string.Empty; - /// - /// Gets or sets the stable schema name. - /// public string SchemaName { get; set; } = string.Empty; - /// - /// Gets or sets the schema version string. - /// public string SchemaVersion { get; set; } = string.Empty; - /// - /// Gets or sets the canonical schema content hash. - /// public string ContentHash { get; set; } = string.Empty; - /// - /// Gets or sets the canonical schema description. - /// public string CanonicalSchemaJson { get; set; } = string.Empty; - /// - /// Gets or sets schema identifiers compatible with this entry. - /// public string[] CompatibleSchemaIds { get; set; } = Array.Empty(); - /// - /// Gets or sets the persisted member descriptors used for compatibility checks. - /// public CultSchemaMemberCatalogEntry[] Members { get; set; } = Array.Empty(); } - /// - /// Persisted catalog entry describing one schema member. - /// public sealed class CultSchemaMemberCatalogEntry { - /// - /// Gets or sets the persisted slot number. - /// public int Slot { get; set; } - /// - /// Gets or sets the member name. - /// public string MemberName { get; set; } = string.Empty; - /// - /// Gets or sets the persisted type name. - /// public string TypeName { get; set; } = string.Empty; - /// - /// Gets or sets whether the member is a reference. - /// public bool IsReference { get; set; } - /// - /// Gets or sets whether the member stores many values. - /// public bool IsMany { get; set; } - /// - /// Gets or sets the referenced schema name, when the member is a reference. - /// public string? TargetSchemaName { get; set; } - /// - /// Gets or sets whether the member provides the name lookup. - /// public bool IsName { get; set; } - /// - /// Gets or sets the index alias, when the member participates in lookups. - /// public string? IndexAlias { get; set; } } - /// - /// Persisted serialized document record. - /// public sealed class CultPersistedRecord { - /// - /// Gets or sets the persisted record key. - /// public string Key { get; set; } = string.Empty; - /// - /// Gets or sets the schema identifier used to deserialize the payload. - /// public string SchemaId { get; set; } = string.Empty; - /// - /// Gets or sets the storage timestamp. - /// public string StoredAt { get; set; } = string.Empty; - /// - /// Gets or sets the serialized document payload. - /// public byte[] Payload { get; set; } = Array.Empty(); } - /// - /// Complete snapshot persisted by a single-file CultCache backing store. - /// public sealed class CultPersistedStoreSnapshot { - /// - /// Gets or sets the backing store format version. - /// public string FormatVersion { get; set; } = "cultcache.store.v1"; - /// - /// Gets or sets the embedded schema catalog. - /// public CultSchemaCatalogEntry[] SchemaCatalog { get; set; } = Array.Empty(); - /// - /// Gets or sets the persisted records. - /// public CultPersistedRecord[] Records { get; set; } = Array.Empty(); } - /// - /// Classifies how a persisted schema mapped onto the local document registry. - /// public enum CultSchemaMigrationKind { - /// - /// The persisted schema matched the local schema exactly. - /// Exact, - /// - /// The persisted schema required a compatible soft-migration path. - /// CompatibleDrift } - /// - /// Describes one schema-resolution warning emitted during load. - /// public sealed class CultSchemaMigrationWarning { - /// - /// Gets or sets the stable warning code. - /// public string Code { get; set; } = string.Empty; - /// - /// Gets or sets the human-readable warning text. - /// public string Message { get; set; } = string.Empty; } - /// - /// Reports how one persisted schema entry resolved against the local schema catalog. - /// public sealed class CultSchemaMigrationReport { - /// - /// Gets or sets the persisted schema identifier. - /// public string PersistedSchemaId { get; set; } = string.Empty; - /// - /// Gets or sets the local schema identifier selected for reading. - /// public string LocalSchemaId { get; set; } = string.Empty; - /// - /// Gets or sets the persisted schema name. - /// public string PersistedSchemaName { get; set; } = string.Empty; - /// - /// Gets or sets the local schema name. - /// public string LocalSchemaName { get; set; } = string.Empty; - /// - /// Gets or sets how the persisted schema matched locally. - /// public CultSchemaMigrationKind Kind { get; set; } - /// - /// Gets or sets the persisted member slots ignored by the local reader. - /// public int[] IgnoredExtraSlots { get; set; } = Array.Empty(); - /// - /// Gets or sets the local member slots defaulted because the persisted record did not contain them. - /// public int[] DefaultedMissingSlots { get; set; } = Array.Empty(); - /// - /// Gets or sets emitted migration warnings. - /// public CultSchemaMigrationWarning[] Warnings { get; set; } = Array.Empty(); } @@ -228,9 +109,6 @@ namespace GameCult.Caching public CultSchemaMigrationReport Report { get; } } - /// - /// Runtime descriptor for a CultCache document type. - /// public sealed class CultDocumentDescriptor { internal CultDocumentDescriptor( @@ -243,8 +121,6 @@ namespace GameCult.Caching bool isGlobal, string? nameMember, Func? nameAccessor, - Func? generatedPayloadSerializer, - Func? generatedPayloadDeserializer, IReadOnlyDictionary> indexAccessors, IReadOnlyList members) { @@ -257,59 +133,22 @@ namespace GameCult.Caching IsGlobal = isGlobal; NameMember = nameMember; NameAccessor = nameAccessor; - GeneratedPayloadSerializer = generatedPayloadSerializer; - GeneratedPayloadDeserializer = generatedPayloadDeserializer; IndexAccessors = indexAccessors; Members = members; } - /// - /// Gets the CLR document type. - /// public Type DocumentType { get; } - /// - /// Gets the stable schema name. - /// public string SchemaName { get; } - /// - /// Gets the schema version string. - /// public string SchemaVersion { get; } - /// - /// Gets the content-derived schema identifier. - /// public string SchemaId { get; } - /// - /// Gets the canonical schema content hash. - /// public string ContentHash { get; } - /// - /// Gets the canonical schema description. - /// public string CanonicalSchemaJson { get; } - /// - /// Gets whether this document type stores one global record. - /// public bool IsGlobal { get; } - /// - /// Gets the document name member, if any. - /// public string? NameMember { get; } internal Func? NameAccessor { get; } - /// - /// Gets the generated payload serializer, if one is available. - /// - public Func? GeneratedPayloadSerializer { get; } - /// - /// Gets the generated payload deserializer, if one is available. - /// - public Func? GeneratedPayloadDeserializer { get; } internal IReadOnlyDictionary> IndexAccessors { get; } internal IReadOnlyList Members { get; } - /// - /// Converts this descriptor into a persisted schema catalog entry. - /// public CultSchemaCatalogEntry ToCatalogEntry() { return new CultSchemaCatalogEntry @@ -358,14 +197,8 @@ namespace GameCult.Caching public string? IndexAlias { get; set; } } - /// - /// Stores a resolved document together with its descriptor and key. - /// public sealed class CultStoredDocument { - /// - /// Creates a stored document wrapper. - /// public CultStoredDocument( CultRecordKey key, string storedAt, @@ -378,27 +211,12 @@ namespace GameCult.Caching Document = document; } - /// - /// Gets the record key. - /// public CultRecordKey Key { get; } - /// - /// Gets the storage timestamp. - /// public string StoredAt { get; } - /// - /// Gets the document descriptor. - /// public CultDocumentDescriptor Descriptor { get; } - /// - /// Gets the document instance. - /// public object Document { get; } } - /// - /// Discovers, indexes, and resolves CultCache document descriptors. - /// public sealed class CultDocumentRegistry { private static readonly Lazy SharedRegistry = @@ -407,14 +225,8 @@ namespace GameCult.Caching private volatile RegistryIndexes _indexes = new(); private readonly object _registrationGate = new(); - /// - /// Gets the shared process-wide document registry. - /// public static CultDocumentRegistry Shared => SharedRegistry.Value; - /// - /// Creates a registry and discovers currently loaded document metadata. - /// public CultDocumentRegistry() { Refresh(); @@ -426,9 +238,6 @@ namespace GameCult.Caching Refresh(); } - /// - /// Creates a registry containing exactly the requested document types without scanning loaded assemblies. - /// public static CultDocumentRegistry ForTypes(IEnumerable documentTypes) { if (documentTypes == null) throw new ArgumentNullException(nameof(documentTypes)); @@ -442,29 +251,15 @@ namespace GameCult.Caching return registry; } - /// - /// Gets all known document descriptors. - /// public IEnumerable AllDescriptors => _indexes.ByType.Values.OrderBy(d => d.SchemaName, StringComparer.Ordinal); - /// - /// Rebuilds the registry from generated metadata and reflected document attributes. - /// public void Refresh() { lock (_registrationGate) { var rebuilt = new RegistryIndexes(); - var generatedTypes = new HashSet(); - foreach (var definition in CultGeneratedDocumentMetadataLoader.LoadDefinitions()) - { - var descriptor = BuildDescriptor(definition); - RegisterDescriptor(rebuilt, descriptor); - generatedTypes.Add(descriptor.DocumentType); - } - - foreach (var type in ReflectionExtensions.GetAttributedDocumentTypes().Where(type => !generatedTypes.Contains(type))) + foreach (var type in ReflectionExtensions.GetAttributedDocumentTypes()) { RegisterDescriptor(rebuilt, BuildDescriptor(type)); } @@ -473,9 +268,6 @@ namespace GameCult.Caching } } - /// - /// Gets the descriptor for a document type, building it when needed. - /// public CultDocumentDescriptor GetRequired(Type type) { if (_indexes.ByType.TryGetValue(type, out var descriptor)) @@ -483,12 +275,6 @@ namespace GameCult.Caching return descriptor; } - descriptor = TryBuildGeneratedDescriptor(type); - if (descriptor != null) - { - return RegisterDescriptor(descriptor); - } - var attribute = type.GetCustomAttribute(); if (attribute == null) { @@ -500,17 +286,11 @@ namespace GameCult.Caching return RegisterDescriptor(descriptor); } - /// - /// Gets the descriptor for a document type. - /// public CultDocumentDescriptor GetRequired() where T : class { return GetRequired(typeof(T)); } - /// - /// Gets a descriptor by its schema identifier. - /// public CultDocumentDescriptor GetRequiredBySchemaId(string schemaId) { if (_indexes.BySchemaId.TryGetValue(schemaId, out var descriptor)) @@ -521,17 +301,11 @@ namespace GameCult.Caching throw new InvalidOperationException($"Unknown CultCache schema id '{schemaId}'."); } - /// - /// Resolves a persisted schema identifier against the local registry and embedded catalog. - /// public CultDocumentDescriptor ResolvePersistedSchema(string schemaId, IReadOnlyCollection catalog) { return ResolvePersistedSchemaDetailed(schemaId, catalog).Descriptor; } - /// - /// Resolves a persisted schema identifier and returns the selected descriptor with migration diagnostics. - /// public CultSchemaMigrationReport ResolvePersistedSchemaReport(string schemaId, IReadOnlyCollection catalog) { return ResolvePersistedSchemaDetailed(schemaId, catalog).Report; @@ -679,15 +453,6 @@ namespace GameCult.Caching public Dictionary BySchemaName { get; } } - private static CultDocumentDescriptor? TryBuildGeneratedDescriptor(Type type) - { - var definition = CultGeneratedDocumentMetadataLoader.LoadDefinitions(type.Assembly) - .FirstOrDefault(candidate => candidate.DocumentType == type); - return definition == null - ? null - : BuildDescriptor(definition); - } - private static CultDocumentDescriptor BuildDescriptor(Type type) { var attribute = type.GetCustomAttribute() @@ -729,49 +494,10 @@ namespace GameCult.Caching type.GetCustomAttribute() != null, nameMember?.Member.Name, nameMember?.GetterNullable, - null, - null, indexAccessors, descriptorMembers); } - private static CultDocumentDescriptor BuildDescriptor(CultGeneratedDocumentDefinition definition) - { - var descriptorMembers = definition.Members - .OrderBy(member => member.Slot) - .Select(member => new CultDocumentMemberDescriptor - { - MemberName = member.MemberName, - Slot = member.Slot, - TypeName = member.TypeName, - IsReference = member.IsReference, - IsMany = member.IsMany, - TargetSchemaName = member.TargetSchemaName, - IsName = member.IsName, - IndexAlias = member.IndexAlias - }) - .ToArray(); - var schemaJson = BuildCanonicalSchemaJson(definition.SchemaName, definition.SchemaVersion, descriptorMembers); - var contentHash = Sha256(schemaJson); - var semanticFingerprint = BuildSemanticFingerprint(definition.SchemaName, definition.SchemaVersion, descriptorMembers); - var schemaId = Sha256(semanticFingerprint); - - return new CultDocumentDescriptor( - definition.DocumentType, - definition.SchemaName, - definition.SchemaVersion, - schemaId, - contentHash, - schemaJson, - definition.IsGlobal, - definition.NameMember, - definition.NameAccessor, - definition.SerializePayload, - definition.DeserializePayload, - definition.IndexAccessors.ToDictionary(accessor => accessor.Alias, accessor => accessor.Accessor, StringComparer.Ordinal), - descriptorMembers); - } - private static string BuildSemanticFingerprint( string schemaName, string schemaVersion, @@ -1107,81 +833,211 @@ namespace GameCult.Caching public int[] DefaultedMissingSlots { get; } } + // Public instance fields and properties (get-only included: MessagePack writes every keyed public member), most-derived + // declaration first; GetCustomAttributes inherits a property's attributes up its override chain, as MessagePack's resolver reads them. private static IReadOnlyList DiscoverMembers(Type type) { - var members = new List(); - - foreach (var field in type.GetFields(BindingFlags.Instance | BindingFlags.Public)) + const BindingFlags declared = BindingFlags.DeclaredOnly | BindingFlags.Instance | BindingFlags.Public; + var messagePackObject = type.GetCustomAttribute(true); + var allowPrivate = messagePackObject?.AllowPrivate == true; + // Writable: MessagePack can assign the member after construction. A member that is not must be filled by the constructor. + var candidates = new List<(MemberInfo Member, Type MemberType, int Depth, bool Writable, PropertyInfo? DivergentRoot)>(); + var seenRoots = new HashSet<(Type, string)>(); + var depth = 0; + for (var current = type; current != null && current != typeof(object); current = current.BaseType, depth++) { - if (IsIgnored(field)) + foreach (var field in current.GetFields(declared)) { - continue; + // MessagePack's rule: a readonly field is writable under AllowPrivate only. + if (!IsIgnored(field)) + candidates.Add((field, field.FieldType, depth, !field.IsInitOnly || allowPrivate, null)); } - members.Add(PersistedMember.FromMember(field, field.FieldType, value => field.GetValue(value), GetKeyValue(field))); - } - - foreach (var property in type.GetProperties(BindingFlags.Instance | BindingFlags.Public)) - { - if (property.GetMethod == null || property.SetMethod == null || IsIgnored(property)) + foreach (var property in current.GetProperties(declared)) { - continue; + if (property.GetIndexParameters().Length > 0 || !seenRoots.Add(RootOf(property))) + continue; + var chain = OverrideChain(property).ToArray(); + var setter = chain.Select(declaration => declaration.SetMethod).FirstOrDefault(method => method != null); + if (chain.All(declaration => declaration.GetMethod == null)) + continue; + // MessagePack's resolver reads the base declaration's [Key]/[IgnoreMember]; an override that changes them is refused. + var root = chain[chain.Length - 1]; + var divergentRoot = chain.Any(declaration => + !Equals(GetKeyValue(declaration), GetKeyValue(root)) || IsIgnored(declaration) != IsIgnored(root)) + ? root + : null; + if (divergentRoot == null && IsIgnored(property)) + continue; + // MessagePack's own rule: it calls a setter that is public, or any setter (init-only included) under AllowPrivate. + candidates.Add((property, property.PropertyType, depth, setter != null && (setter.IsPublic || allowPrivate), divergentRoot)); } - - members.Add(PersistedMember.FromMember(property, property.PropertyType, value => property.GetValue(value), GetKeyValue(property))); } - var explicitMembers = members.Where(member => member.ExplicitSlot.HasValue).OrderBy(member => member.ExplicitSlot.GetValueOrDefault()).ToArray(); - var implicitMembers = members.Where(member => !member.ExplicitSlot.HasValue).OrderBy(member => member.MetadataToken).ToArray(); - var assigned = new List(members.Count); - var nextSlot = explicitMembers.Length == 0 ? 0 : explicitMembers.Max(member => member.ExplicitSlot.GetValueOrDefault()) + 1; - - foreach (var member in explicitMembers) + var rejections = new List(); + if (messagePackObject == null) + rejections.Add(NotMessagePackObjectMessage(type.Name)); + else if (!type.IsVisible && !allowPrivate) + rejections.Add(NotVisibleMessage(type.Name)); + else { - member.Slot = member.ExplicitSlot.GetValueOrDefault(); - assigned.Add(member); - } - - foreach (var member in implicitMembers) - { - member.Slot = nextSlot++; - assigned.Add(member); - } - - return assigned.OrderBy(member => member.Slot).ToArray(); - } - - private static bool IsIgnored(MemberInfo member) - { - return member.GetCustomAttributes().Any(attribute => - { - var name = attribute.GetType().FullName; - return name == "MessagePack.IgnoreMemberAttribute"; - }); - } - - private static int? GetKeyValue(MemberInfo member) - { - foreach (var attribute in member.GetCustomAttributes()) - { - var name = attribute.GetType().FullName; - if (name == "MessagePack.KeyAttribute") + // AllowPrivate makes MessagePack read non-public members, which the registry does not persist; without it MessagePack + // silently skips a keyed non-public member. + const BindingFlags nonPublic = BindingFlags.DeclaredOnly | BindingFlags.Instance | BindingFlags.NonPublic; + for (var current = type; current != null && current != typeof(object); current = current.BaseType) { - var property = attribute.GetType().GetProperty("IntKey"); - if (property?.GetValue(attribute) is int intKey) + foreach (var member in current.GetFields(nonPublic).Cast().Concat(current.GetProperties(nonPublic))) { - return intKey; + if (allowPrivate && !member.IsDefined(typeof(CompilerGeneratedAttribute)) && !IsIgnored(member)) + rejections.Add(UnmarkedNonPublicMemberMessage(type.Name, Qualified(member))); + else if (!allowPrivate && GetKeyValue(member) != null && !IsIgnored(member)) + rejections.Add(KeyedNonPublicMemberMessage(type.Name, Qualified(member))); } + } + } + var keyed = new List<(MemberInfo Member, Type MemberType, int Slot, bool Writable)>(); + foreach (var candidate in candidates.OrderBy(candidate => candidate.Depth).ThenBy(candidate => candidate.Member.Name, StringComparer.Ordinal)) + { + var key = GetKeyValue(candidate.Member); + if (candidate.DivergentRoot != null) + rejections.Add(DivergentOverrideMessage(type.Name, Qualified(candidate.Member), Qualified(candidate.DivergentRoot))); + else if (key is string) + rejections.Add(StringKeyMessage(type.Name, candidate.Member.Name)); + else if (key is not int slot) + rejections.Add(UnkeyedMemberMessage(type.Name, candidate.Member.Name)); + else + keyed.Add((candidate.Member, candidate.MemberType, slot, candidate.Writable)); + } - var ctorArg = attribute.GetType().GetProperty("StringKey"); - if (ctorArg?.GetValue(attribute) is string) + if (messagePackObject != null) + { + // A member MessagePack cannot assign is read back only through the picked constructor's parameter at its slot. + var arity = ConstructorArity(type, allowPrivate, keyed.Select(entry => (entry.MemberType, entry.Slot))); + if (arity == null) + rejections.Add(NoConstructorMessage(type.Name)); + else + { + foreach (var entry in keyed.Where(entry => !entry.Writable && entry.Slot >= arity)) { - return null; + rejections.Add(entry.Member switch + { + FieldInfo => ReadonlyUnfilledMessage(type.Name, entry.Member.Name, entry.Slot), + PropertyInfo property when property.SetMethod == null && OverrideChain(property).All(declaration => declaration.SetMethod == null) => + GetOnlyUnfilledMessage(type.Name, entry.Member.Name, entry.Slot), + _ => NonPublicSetterMessage(type.Name, entry.Member.Name) + }); } } } - return null; + foreach (var group in keyed.GroupBy(entry => entry.Slot).Where(group => group.Count() > 1).OrderBy(group => group.Key)) + { + var pair = group.Take(2).ToArray(); + rejections.Add(DuplicateSlotMessage(type.Name, Qualified(pair[0].Member), Qualified(pair[1].Member), group.Key)); + } + + foreach (var group in keyed.GroupBy(entry => entry.Member.Name) + .Where(group => group.Select(entry => entry.Slot).Distinct().Count() > 1) + .OrderBy(group => group.Key, StringComparer.Ordinal)) + { + var pair = group.Take(2).ToArray(); + rejections.Add(HiddenMemberMessage(type.Name, Qualified(pair[0].Member), Qualified(pair[1].Member))); + } + + if (rejections.Count > 0) + throw new InvalidOperationException(rejections[0]); + + return keyed + .Select(entry => PersistedMember.FromMember( + entry.Member, + entry.MemberType, + entry.Member is FieldInfo field ? field.GetValue : ((PropertyInfo)entry.Member).GetValue, + entry.Slot)) + .OrderBy(member => member.Slot) + .ToArray(); + } + + private static (Type, string) RootOf(PropertyInfo property) => + ((property.GetMethod ?? property.SetMethod)!.GetBaseDefinition().DeclaringType!, property.Name); + + private static IEnumerable OverrideChain(PropertyInfo property) + { + var root = RootOf(property); + for (var current = property.DeclaringType; current != null; current = current.BaseType) + { + var declaration = current.GetProperty(property.Name, BindingFlags.DeclaredOnly | BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic); + if (declaration != null && declaration.GetIndexParameters().Length == 0 && RootOf(declaration) == root) + yield return declaration; + } + } + + // MessagePack's DynamicObjectResolver picks the [SerializationConstructor] if one is marked, otherwise the longest public constructor + // (any constructor under AllowPrivate) whose every parameter takes the keyed member at its position (slot == parameter index). + // Returns that constructor's parameter count, 0 for a struct with none, or null when MessagePack finds no constructor. + private static int? ConstructorArity(Type type, bool allowPrivate, IEnumerable<(Type MemberType, int Slot)> members) + { + var slots = new Dictionary(); + foreach (var member in members) + slots.TryAdd(member.Slot, member.MemberType); + var constructors = type.GetConstructors(BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic); + var marked = constructors.Where(constructor => constructor.IsDefined(typeof(SerializationConstructorAttribute))).ToArray(); + var arities = (marked.Length > 0 ? marked : constructors.Where(constructor => allowPrivate || constructor.IsPublic)) + .Select(constructor => constructor.GetParameters()) + .Where(parameters => parameters.All(parameter => + slots.TryGetValue(parameter.Position, out var memberType) && parameter.ParameterType.IsAssignableFrom(memberType))) + .Select(parameters => (int?)parameters.Length) + .ToArray(); + return arities.Length > 0 ? arities.Max() : type.IsValueType && marked.Length == 0 ? 0 : null; + } + + private static string Qualified(MemberInfo member) => member.DeclaringType!.Name.Split('`')[0] + "." + member.Name; + + private static string NotMessagePackObjectMessage(string documentTypeName) => + $"Cult document {documentTypeName} is not a MessagePack object; add [MessagePackObject] so MessagePack serializes its [Key(n)] members."; + + private static string NotVisibleMessage(string documentTypeName) => + $"Cult document {documentTypeName} is not public; MessagePack serializes a non-public type only with [MessagePackObject(AllowPrivate = true)], so make it public or set AllowPrivate."; + + private static string UnmarkedNonPublicMemberMessage(string documentTypeName, string memberName) => + $"Cult document {documentTypeName} member {memberName} is non-public; [MessagePackObject(AllowPrivate = true)] makes MessagePack read it, so mark it [IgnoreMember]."; + + internal static string UnkeyedMemberMessage(string documentTypeName, string memberName) => + $"Cult document {documentTypeName} member {memberName} has no [Key]; MessagePack requires [Key(n)] or [IgnoreMember] on every public member, so mark it one or the other."; + + private static string StringKeyMessage(string documentTypeName, string memberName) => + $"Cult document {documentTypeName} member {memberName} has a string [Key]; string keys are not supported; use integer [Key(n)]."; + + private static string ReadonlyUnfilledMessage(string documentTypeName, string memberName, int slot) => + $"Cult document {documentTypeName} member {memberName} is a readonly field that no constructor fills; MessagePack writes it but never reads it back, so give the constructor MessagePack calls a parameter at position {slot}, drop readonly, or add [MessagePackObject(AllowPrivate = true)]."; + + private static string GetOnlyUnfilledMessage(string documentTypeName, string memberName, int slot) => + $"Cult document {documentTypeName} member {memberName} is a get-only property that no constructor fills; MessagePack writes it but never reads it back, so give the constructor MessagePack calls a parameter at position {slot}, add a setter, or mark it [IgnoreMember]."; + + private static string NonPublicSetterMessage(string documentTypeName, string memberName) => + $"Cult document {documentTypeName} member {memberName} has a non-public set accessor; MessagePack writes it but never reads it back without AllowPrivate, so make the setter public or add [MessagePackObject(AllowPrivate = true)]."; + + private static string KeyedNonPublicMemberMessage(string documentTypeName, string memberName) => + $"Cult document {documentTypeName} member {memberName} is non-public; MessagePack skips it silently, so [Key] on a non-public member requires [MessagePackObject(AllowPrivate = true)]."; + + private static string NoConstructorMessage(string documentTypeName) => + $"Cult document {documentTypeName} has no constructor MessagePack can call; add a parameterless constructor (public unless AllowPrivate), or one whose parameters take the [Key(0)], [Key(1)], ... members in order."; + + private static string DuplicateSlotMessage(string documentTypeName, string first, string second, int slot) => + $"Cult document {documentTypeName} members {first} and {second} share [Key({slot})]; every persisted member needs a distinct [Key(n)]."; + + private static string DivergentOverrideMessage(string documentTypeName, string member, string root) => + $"Cult document {documentTypeName} member {member} overrides {root} with a different [Key] or [IgnoreMember]; MessagePack reads the base declaration's, so an override must repeat or omit them."; + + private static string HiddenMemberMessage(string documentTypeName, string first, string second) => + $"Cult document {documentTypeName} member {first} hides persisted member {second}; persisted member names must be unique."; + + private static bool IsIgnored(MemberInfo member) => + member.GetCustomAttribute() != null; + + private static object? GetKeyValue(MemberInfo member) + { + var key = member.GetCustomAttribute(); + return key?.IntKey is int slot ? slot : key?.StringKey; } private static string Sha256(string input) @@ -1195,8 +1051,6 @@ namespace GameCult.Caching { public MemberInfo Member { get; set; } = default!; public Type MemberType { get; set; } = default!; - public int MetadataToken { get; set; } - public int? ExplicitSlot { get; set; } public int Slot { get; set; } public bool IsName { get; set; } public string? IndexAlias { get; set; } @@ -1210,7 +1064,7 @@ namespace GameCult.Caching MemberInfo member, Type memberType, Func getValue, - int? explicitSlot) + int slot) { var referenceAttribute = member.GetCustomAttribute(); var targetType = ResolveReferenceTarget(memberType, referenceAttribute?.TargetType); @@ -1219,9 +1073,7 @@ namespace GameCult.Caching { Member = member, MemberType = memberType, - MetadataToken = member.MetadataToken, - ExplicitSlot = explicitSlot, - Slot = explicitSlot ?? -1, + Slot = slot, IsName = member.GetCustomAttribute() != null, IndexAlias = ResolveIndexAlias(member), IsReference = targetType != null || referenceAttribute != null, @@ -1263,669 +1115,783 @@ namespace GameCult.Caching } } - /// - /// In-memory document cache with pluggable persisted backing stores. - /// + public enum CultCommitOutcome + { + Committed, + Mismatch, + Contended + } + + public sealed class CultCommitRequest + { + internal CultCommitRequest( + IReadOnlyList upserts, + IReadOnlyList deletes, + IReadOnlyList<(CultRecordKey Key, string? SchemaId, string? StoredAt)> expected, + bool expectUnchanged) + { + Upserts = upserts; + Deletes = deletes; + Expected = expected; + ExpectUnchanged = expectUnchanged; + } + + public IReadOnlyList Upserts { get; } + public IReadOnlyList Deletes { get; } + public IReadOnlyList<(CultRecordKey Key, string? SchemaId, string? StoredAt)> Expected { get; } + public bool ExpectUnchanged { get; } + public bool HasConditions => Expected.Count > 0 || ExpectUnchanged; + + // durable is what the store holds on disk now; observed is what its cache last loaded or committed. + // Identity is (schemaId, storedAt), sound because every write to a key mints a later storedAt. + public bool ConditionsHold(IReadOnlyCollection durable, IEnumerable observed) + { + var byKey = durable.ToDictionary(record => record.Key, StringComparer.Ordinal); + foreach (var (key, schemaId, storedAt) in Expected) + { + byKey.TryGetValue(key.Value, out var record); + var holds = schemaId == null + ? record == null + : record != null && record.SchemaId == schemaId && record.StoredAt == storedAt; + if (!holds) + return false; + } + + return !ExpectUnchanged || + durable.Select(record => Identity(record.Key, record.SchemaId, record.StoredAt)) + .OrderBy(identity => identity, StringComparer.Ordinal) + .SequenceEqual(observed + .Select(stored => Identity(stored.Key.Value, stored.Descriptor.SchemaId, stored.StoredAt)) + .OrderBy(identity => identity, StringComparer.Ordinal)); + } + + private static string Identity(string key, string schemaId, string storedAt) => $"{key}\n{schemaId}\n{storedAt}"; + } + + // An explicit value: nothing staged here is visible to anyone until Commit returns. + public sealed class CultCacheBatch + { + private readonly CultCache _cache; + internal readonly Dictionary Operations = new(StringComparer.Ordinal); + internal readonly List<(CultRecordKey Key, string? SchemaId, string? StoredAt)> Expected = new(); + internal bool ExpectsUnchanged; + internal bool Sealed; + + internal CultCacheBatch(CultCache cache) + { + _cache = cache; + } + + public CultRecordHandle Upsert(T document, CultRecordHandle? handle = null) + { + if (document == null) throw new ArgumentNullException(nameof(document)); + return new CultRecordHandle(Stage(document, handle?.Key)); + } + + public CultRecordKey Upsert(Type type, object document, CultRecordKey? key = null) + { + CultCache.RequireInstanceOf(type, document); + return Stage(document, key); + } + + public void Remove(CultRecordKey key) + { + ThrowIfSealed(); + Operations[key.Value] = (key, null); + } + + public void Expect(CultRecordKey key, object? current) + { + ThrowIfSealed(); + var observed = _cache.Observe(key, current); + Expected.Add((key, observed?.Descriptor.SchemaId, observed?.StoredAt)); + } + + public void ExpectUnchanged() + { + ThrowIfSealed(); + ExpectsUnchanged = true; + } + + private CultRecordKey Stage(object document, CultRecordKey? key) + { + ThrowIfSealed(); + var stored = _cache.CreateStoredDocument(document, key); + Operations[stored.Key.Value] = (stored.Key, stored); + return stored.Key; + } + + private void ThrowIfSealed() + { + if (Sealed) + throw new InvalidOperationException("This batch has already been committed or abandoned."); + } + } + public sealed class CultCache : IDisposable { private readonly CultDocumentRegistry _registry; - private readonly List _backingStores = new(); - private readonly ConcurrentDictionary _entries = new(StringComparer.Ordinal); - private readonly ConcurrentDictionary> _nameMaps = new(); - private readonly ConcurrentDictionary<(Type Type, string Alias), ConcurrentDictionary> _indexMaps = new(); - private readonly ConcurrentDictionary _globalKeys = new(); - private readonly ConditionalWeakTable _documentHandles = new(); - private readonly Subject _changes = new(); - private readonly CultCacheSoaStore _soa = new(); - private ILogger _logger = new NullLogger(); - private bool _hasUnflushedMutations; + private readonly List<(CacheBackingStore Store, Type[] Homes)> _stores = new(); + private readonly Dictionary _entries = new(StringComparer.Ordinal); + private readonly Dictionary> _names = new(); + private readonly Dictionary<(Type Type, string Alias), Dictionary> _indexes = new(); + private readonly Dictionary _globals = new(); + private readonly ConditionalWeakTable _handles = new(); + private readonly Subject _changes = new(); + private readonly object _gate = new(); + private long _sequence; + // The changes admitted by this thread's outermost hold, published by that hold when it exits. + [ThreadStatic] private static List<(Change Change, bool Loaded)>? _held; + private bool _dirtyInMemory; - /// - /// Creates a cache using the supplied document registry or the shared registry. - /// public CultCache(CultDocumentRegistry? registry = null) - : this(registry, initializeGlobals: true) - { - } - - /// - /// Creates a cache using the supplied document registry, optionally deferring global defaults - /// until a durable backing store has been hydrated. - /// - public CultCache(CultDocumentRegistry? registry, bool initializeGlobals) { _registry = registry ?? CultDocumentRegistry.Shared; - if (initializeGlobals) + } + + public CultDocumentRegistry Registry => _registry; + + public bool IsDirty + { + get { - MaterializeMissingGlobals(); + lock (_gate) + return _stores.Count == 0 ? _dirtyInMemory : _stores.Any(entry => entry.Store.IsDirty); } } - /// - /// Creates default instances for global document types that do not already have a record. - /// Durable open paths call this after hydration so persisted globals remain authoritative. - /// - public void MaterializeMissingGlobals() - { - InitializeGlobals(); - } - - /// - /// Gets or sets the cache logger. - /// - public ILogger Logger - { - get => _logger; - set => _logger = value ?? new NullLogger(); - } - - /// - /// Gets whether the cache currently holds unflushed mutations in any attached backing store or only in memory. - /// - public bool IsDirty => _hasUnflushedMutations || _backingStores.Any(store => store.IsDirty); - - /// - /// Gets the UTC timestamp of the last successful flush across all attached backing stores. - /// - public DateTimeOffset? LastSuccessfulFlushAtUtc { get; private set; } - - /// - /// Gets or sets whether disposing the cache should flush attached dirty backing stores first. - /// public bool FlushAttachedStoresOnDispose { get; set; } - /// - /// Raised when a backing store adds, updates, or removes a document. - /// public event Action? OnUpdate; - /// - /// Gets attached backing stores in registration order. - /// - public IReadOnlyList BackingStores => _backingStores; - - /// - /// Gets all document instances currently held by the cache. - /// - public IEnumerable AllEntries => _entries.Values.Select(entry => entry.Document); - - /// - /// Gets all stored document records currently held by the cache. - /// - public IEnumerable AllStoredDocuments => - _entries.Values.OrderBy(entry => entry.Descriptor.SchemaName, StringComparer.Ordinal) - .ThenBy(entry => entry.Key.Value, StringComparer.Ordinal); - - /// - /// Gets the document registry used by this cache. - /// - public CultDocumentRegistry Registry => _registry; - - /// - /// Opens a reactive POCO presentation for one cache-managed document. - /// - public CultManagedDocument Document(CultRecordKey key) where T : class + public IReadOnlyList BackingStores { - return new CultManagedDocument( - key, - () => Get(key), - async value => { await UpsertAsync(value, new CultRecordHandle(key)).ConfigureAwait(false); }, - WatchRecord(key) - .Where(change => change.Document != null) - .Select(change => change.Document!)); + get + { + lock (_gate) + return _stores.Select(entry => entry.Store).ToArray(); + } + } + + public IEnumerable AllEntries + { + get + { + lock (_gate) + return _entries.Values.Select(entry => entry.Document).ToArray(); + } + } + + public IEnumerable AllStoredDocuments + { + get + { + lock (_gate) + return _entries.Values + .OrderBy(entry => entry.Descriptor.SchemaName, StringComparer.Ordinal) + .ThenBy(entry => entry.Key.Value, StringComparer.Ordinal) + .ToArray(); + } } - /// - /// Watches all local cache changes assignable to the requested document type. - /// public Observable> Watch() where T : class { return _changes - .Where(change => change is CultCacheDocumentChange) - .Select(change => (CultCacheDocumentChange)change); + .Where(change => typeof(T).IsAssignableFrom(change.Stored.Descriptor.DocumentType)) + .Select(change => new CultCacheDocumentChange( + change.Kind, + change.Stored.Key, + change.Document as T, + change.Previous as T, + change.Sequence)); } - /// - /// Watches one local cache record. - /// public Observable> WatchRecord(CultRecordKey key) where T : class { return Watch().Where(change => change.Key.Equals(key)); } - /// - /// Gets the cache-owned structure-of-arrays table for a document type. - /// - public CultSoaTable Soa() where T : class - { - return _soa.Snapshot(); - } - - /// - /// Attaches a backing store to this cache. - /// - public void AddBackingStore(CacheBackingStore store) + // Attaching reads the store; there is no interval in which it is attached but unread. + public void AddBackingStore(CacheBackingStore store, params Type[] homes) { if (store == null) throw new ArgumentNullException(nameof(store)); - store.AttachRegistry(_registry); - store.Logger = Logger; - store.EntryAdded.Subscribe(entry => AddStoredDocumentInternal(entry, store, raiseUpdate: true).GetAwaiter().GetResult()); - store.EntryUpdated.Subscribe(entry => AddStoredDocumentInternal(entry, store, raiseUpdate: true).GetAwaiter().GetResult()); - store.EntryDeleted.Subscribe(entry => RemoveStoredDocumentInternal(entry, store, raiseUpdate: true)); - - foreach (var entry in _entries.Values.OrderBy(entry => entry.Key.Value, StringComparer.Ordinal)) + homes ??= Array.Empty(); + Held(() => { - store.Push(entry); - } + if (store.Loaded != null) + throw new InvalidOperationException($"Backing store {store} is already attached to a cache."); + if (store.IsDirty) + throw new InvalidOperationException($"Backing store {store} has staged writes; attach it clean."); + if (homes.Length == 0 && _stores.Any(entry => entry.Homes.Length == 0)) + throw new InvalidOperationException( + $"Backing store {store} would be a second untyped store; name the types it is home to."); + foreach (var home in homes) + { + var owner = _stores.FirstOrDefault(entry => entry.Homes.Contains(home)).Store; + if (owner != null) + throw new InvalidOperationException($"{home.FullName} is already routed to {owner}; it cannot also route to {store}."); + } - _backingStores.Add(store); + var candidate = (store, homes); + foreach (var type in _entries.Values.Select(entry => entry.Descriptor).Distinct()) + { + var before = Home(type.DocumentType); + _stores.Add(candidate); + var after = Home(type.DocumentType); + _stores.RemoveAt(_stores.Count - 1); + if (before != after) + throw new InvalidOperationException( + $"Attaching {store} would move {type.SchemaName} from {before?.ToString() ?? "memory"} to {after}; " + + "attach routed stores before the untyped store."); + } + + store.AttachRegistry(_registry); + store.Cache = this; + store.Loaded = (loaded, dropped) => Admit(loaded, dropped, store, _ => CultCommitOutcome.Committed); + _stores.Add(candidate); + try + { + store.PullAll(); + } + catch + { + _stores.Remove(candidate); + store.Loaded = null; + store.Cache = null; + throw; + } + return true; + }); } - /// - /// Pulls all documents from every attached backing store. - /// - public async Task PullAllBackingStoresAsync() + public Task PullAllBackingStoresAsync() { - foreach (var store in _backingStores) + // Our stores take the gate themselves; a third-party store might not, so the cache takes it here. + // One store's throwing OnUpdate handler does not stop the others from loading. + var failures = new List(); + foreach (var store in BackingStores) { - store.PullAll(); + try + { + Held(() => { store.PullAll(); return true; }); + } + catch (Exception exception) + { + failures.Add(exception); + } } - _hasUnflushedMutations = _backingStores.Any(store => store.IsDirty); - await Task.CompletedTask; - } - - /// - /// Pulls persisted records selected by durable metadata from every attached backing store. - /// Stores without indexed selection support preserve correctness by performing a full pull. - /// - public async Task PullBackingStoreRecordsAsync(Func selector) - { - if (selector == null) throw new ArgumentNullException(nameof(selector)); - foreach (var store in _backingStores) - { - store.PullSelected(selector); - } - - _hasUnflushedMutations = _backingStores.Any(store => store.IsDirty); - await Task.CompletedTask; - } - - /// - /// Flushes all attached backing stores. - /// - public void FlushAllBackingStores(bool soft = false) - { - foreach (var store in _backingStores) - { - FlushBackingStore(store, soft); - } - - RecomputeDirtyState(); - if (!IsDirty) - { - LastSuccessfulFlushAtUtc = DateTimeOffset.UtcNow; - } - } - - /// - /// Flushes all attached backing stores. - /// - public Task FlushAsync(bool soft = false) - { - FlushAllBackingStores(soft); + if (failures.Count == 1) + ExceptionDispatchInfo.Capture(failures[0]).Throw(); + if (failures.Count > 1) + throw new AggregateException(failures); return Task.CompletedTask; } - /// - /// Flushes one attached backing store. - /// - public void FlushBackingStore(CacheBackingStore store, bool soft = false) + public void FlushAllBackingStores() => Held(() => { - if (store == null) throw new ArgumentNullException(nameof(store)); - if (!_backingStores.Contains(store)) + foreach (var (store, _) in _stores) { - throw new InvalidOperationException("Backing store is not attached to this cache."); + if (!store.IsReadOnly && store.IsDirty) + store.PushAll(); } + return true; + }); - store.PushAll(soft); - RecomputeDirtyState(); - if (!store.IsDirty) - { - LastSuccessfulFlushAtUtc = DateTimeOffset.UtcNow; - } - } - - /// - /// Flushes attached backing stores at a lifecycle boundary such as shutdown or assembly reload. - /// - public void PrepareForReloadOrShutdown(bool soft = false) + public Task FlushAsync() { - FlushAllBackingStores(soft); - } - - /// - /// Flushes attached backing stores at a lifecycle boundary such as shutdown or assembly reload. - /// - public Task PrepareForReloadOrShutdownAsync(bool soft = false) - { - PrepareForReloadOrShutdown(soft); + FlushAllBackingStores(); return Task.CompletedTask; } - /// - /// Adds or replaces a typed document and returns its record handle. - /// - public async Task> AddAsync(T document, CultRecordHandle? handle = null) + public Task> AddAsync(T document, CultRecordHandle? handle = null) { - if (document == null) throw new ArgumentNullException(nameof(document)); - var stored = await AddStoredDocumentInternal( - CreateStoredDocument(typeof(T), document, handle?.Key), - source: null, - raiseUpdate: false); - return new CultRecordHandle(stored.Key); + return UpsertAsync(document, handle); } - /// - /// Adds or replaces a typed document and returns its record handle. - /// public Task> UpsertAsync(T document, CultRecordHandle? handle = null) { - return AddAsync(document, handle); - } - - /// - /// Adds or replaces a document whose concrete type is known at runtime. - /// - public async Task UpsertAsync(Type documentType, object document, CultRecordKey? key = null) - { - if (documentType == null) throw new ArgumentNullException(nameof(documentType)); if (document == null) throw new ArgumentNullException(nameof(document)); - if (!documentType.IsInstanceOfType(document)) - { - throw new ArgumentException( - $"Document instance must be assignable to {documentType.FullName}.", - nameof(document)); - } - - var stored = await AddStoredDocumentInternal( - CreateStoredDocument(documentType, document, key), - source: null, - raiseUpdate: false).ConfigureAwait(false); - return stored.Key; + return Task.FromResult(new CultRecordHandle(Write(document, handle?.Key))); + } + + public Task UpsertAsync(Type documentType, object document, CultRecordKey? key = null) + { + RequireInstanceOf(documentType, document); + return Task.FromResult(Write(document, key)); + } + + public bool Remove(CultRecordKey key) => Held(() => + { + if (!_entries.TryGetValue(key.Value, out var existing)) + return false; + Admit(Array.Empty(), new[] { existing }, null, home => + { + home?.Delete(existing); + return CultCommitOutcome.Committed; + }); + return true; + }); + + public void Remove(CultRecordHandle handle) + { + Remove(handle.Key); + } + + // false: a condition failed; nothing was written, changed in memory, or published. + public bool Commit(Action stage) + { + return Land(stage, wait: true) == CultCommitOutcome.Committed; + } + + public CultCommitOutcome TryCommit(Action stage) + { + return Land(stage, wait: false); } - /// - /// Gets the record handle for a document instance, if it is tracked. - /// public CultRecordHandle? TryGetHandle(T document) { if (document == null) throw new ArgumentNullException(nameof(document)); - return _documentHandles.TryGetValue(document, out var box) - ? new CultRecordHandle(box.Key) - : null; + lock (_gate) + return _handles.TryGetValue(document, out var box) ? new CultRecordHandle(box.Key) : null; } - /// - /// Gets a document by record key. - /// public object? Get(CultRecordKey key) { - return _entries.TryGetValue(key.Value, out var stored) - ? stored.Document - : null; + lock (_gate) + return _entries.TryGetValue(key.Value, out var stored) ? stored.Document : null; } - /// - /// Gets a typed document by record key. - /// public T? Get(CultRecordKey key) where T : class { return Get(key) as T; } - /// - /// Tries to get a typed document by record key. - /// + // Every change whose Sequence is at or below the returned one is reflected in the returned document. + public (object? Document, long Sequence) GetWithSequence(CultRecordKey key) + { + lock (_gate) + return (_entries.TryGetValue(key.Value, out var stored) ? stored.Document : null, _sequence); + } + public bool TryGet(CultRecordKey key, out T? document) where T : class { document = Get(key); return document != null; } - /// - /// Gets all cached documents assignable to the requested type. - /// public IEnumerable GetAll() where T : class { - var type = typeof(T); - return _entries.Values - .Where(entry => type.IsAssignableFrom(entry.Descriptor.DocumentType)) - .Select(entry => (T)entry.Document); + lock (_gate) + return _entries.Values.Select(entry => entry.Document).OfType().ToArray(); } - /// - /// Gets the global document for the requested type, if one exists. - /// public T? GetGlobal() where T : class { - return _globalKeys.TryGetValue(typeof(T), out var key) - ? Get(new CultRecordKey(key)) - : null; + return Single("global", () => _globals + .Where(pair => typeof(T).IsAssignableFrom(pair.Key)) + .Select(pair => pair.Value)); } - /// - /// Gets a typed document by its CultName value. - /// public T? GetByName(string name) where T : class { - var type = typeof(T); - if (_nameMaps.TryGetValue(type, out var map) && - map.TryGetValue(name, out var key)) - { - return Get(new CultRecordKey(key)); - } - - return null; + return Single($"name '{name}'", () => _names + .Where(pair => typeof(T).IsAssignableFrom(pair.Key)) + .SelectMany(pair => pair.Value.TryGetValue(name, out var key) ? new[] { key } : Array.Empty())); } - /// - /// Tries to get a typed document by its CultName value. - /// - public bool TryGetByName(string name, out T? document) where T : class - { - document = GetByName(name); - return document != null; - } - - /// - /// Gets a typed document by an indexed value. - /// public T? GetByIndex(string alias, string value) where T : class { - if (_indexMaps.TryGetValue((typeof(T), alias), out var map) && - map.TryGetValue(value, out var key)) - { - return Get(new CultRecordKey(key)); - } - - return null; + return Single($"index {alias}='{value}'", () => _indexes + .Where(pair => pair.Key.Alias == alias && typeof(T).IsAssignableFrom(pair.Key.Type)) + .SelectMany(pair => pair.Value.TryGetValue(value, out var key) ? new[] { key } : Array.Empty())); } - /// - /// Tries to get a typed document by an indexed value. - /// - public bool TryGetByIndex(string alias, string value, out T? document) where T : class - { - document = GetByIndex(alias, value); - return document != null; - } - - /// - /// Resolves a typed document reference against this cache. - /// - public T? Resolve(CultRecordRef reference) where T : class - { - return Get(reference.Key); - } - - /// - /// Removes a document by typed handle. - /// - public void Remove(CultRecordHandle handle) - { - if (_entries.TryGetValue(handle.Key.Value, out var stored)) - { - RemoveStoredDocumentInternal(stored, source: null, raiseUpdate: false); - } - } - - /// - /// Removes a document by cache-owned record key. - /// - public bool Remove(CultRecordKey key) - { - if (!_entries.TryGetValue(key.Value, out var stored)) - { - return false; - } - - RemoveStoredDocumentInternal(stored, source: null, raiseUpdate: false); - return true; - } - - /// - /// Removes a document by typed handle. - /// - public Task DeleteAsync(CultRecordHandle handle) - { - Remove(handle); - return Task.CompletedTask; - } - - /// - /// Disposes attached disposable backing stores. - /// public void Dispose() { - if (FlushAttachedStoresOnDispose && IsDirty) + Held(() => { - FlushAllBackingStores(); - } + if (FlushAttachedStoresOnDispose && IsDirty) + FlushAllBackingStores(); + return true; + }); - foreach (var store in _backingStores.OfType()) - { + foreach (var store in BackingStores) store.Dispose(); - } _changes.Dispose(); } - internal CultStoredDocument CreateStoredDocument(Type documentType, object document, CultRecordKey? key = null, string? storedAt = null) + internal static void RequireInstanceOf(Type documentType, object document) { - var descriptor = _registry.GetRequired(documentType); - var resolvedKey = key ?? ResolveKey(document, descriptor); - return new CultStoredDocument( - resolvedKey, - storedAt ?? DateTimeOffset.UtcNow.ToString("O"), - descriptor, - document); + if (documentType == null) throw new ArgumentNullException(nameof(documentType)); + if (document == null) throw new ArgumentNullException(nameof(document)); + if (!documentType.IsInstanceOfType(document)) + throw new ArgumentException($"Document instance must be assignable to {documentType.FullName}.", nameof(document)); } - private async Task AddStoredDocumentInternal( - CultStoredDocument stored, + // The runtime type decides the schema; a minted storedAt is always later than the record it replaces. + internal CultStoredDocument CreateStoredDocument(object document, CultRecordKey? key) + { + var descriptor = _registry.GetRequired(document.GetType()); + lock (_gate) + return Stamp( + key ?? (_handles.TryGetValue(document, out var box) ? box.Key + : descriptor.IsGlobal ? new CultRecordKey($"global:{descriptor.SchemaId}") + : new CultRecordKey(Guid.NewGuid().ToString("N"))), + descriptor, + document); + } + + // Call under the gate that admits the result, so no load can land a later storedAt in between. + private CultStoredDocument Stamp(CultRecordKey key, CultDocumentDescriptor descriptor, object document) => + new(key, MintStoredAt(_entries.TryGetValue(key.Value, out var previous) ? previous.StoredAt : null), descriptor, document); + + internal CultStoredDocument? Observe(CultRecordKey key, object? current) + { + if (current == null) + return null; + lock (_gate) + { + if (_handles.TryGetValue(current, out var box) && box.Key.Equals(key) && + _entries.TryGetValue(key.Value, out var stored) && ReferenceEquals(stored.Document, current)) + return stored; + } + + throw new ArgumentException($"Expect({key.Value}) was given an instance this cache does not hold at that key.", nameof(current)); + } + + private static string MintStoredAt(string? previous) + { + var now = DateTimeOffset.UtcNow; + if (previous != null && + DateTimeOffset.TryParse(previous, CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind, out var last) && + now <= last) + now = last.ToUniversalTime().AddTicks(1); + return now.ToString("O", CultureInfo.InvariantCulture); + } + + private CultRecordKey Write(object document, CultRecordKey? key) => Held(() => + { + var stored = CreateStoredDocument(document, key); + Admit(new[] { stored }, Array.Empty(), null, home => + { + home?.Push(stored); + return CultCommitOutcome.Committed; + }); + return stored.Key; + }); + + private CultCommitOutcome Land(Action stage, bool wait) + { + if (stage == null) throw new ArgumentNullException(nameof(stage)); + var batch = new CultCacheBatch(this); + try + { + stage(batch); + } + finally + { + batch.Sealed = true; + } + + // The in-process gate is always entered; only the store lock attempt honors wait, so Contended means another writer holds the store. + return Held(() => + { + // Staging stamped outside this gate; a pull since then may hold a later storedAt, so stamp again. + var upserts = batch.Operations.Values + .Where(op => op.Stored != null) + .Select(op => Stamp(op.Key, op.Stored!.Descriptor, op.Stored.Document)) + .ToArray(); + var deletes = batch.Operations.Values + .Where(op => op.Stored == null && _entries.ContainsKey(op.Key.Value)) + .Select(op => _entries[op.Key.Value]) + .ToArray(); + var request = new CultCommitRequest(upserts, deletes, batch.Expected.ToArray(), batch.ExpectsUnchanged); + return Admit(upserts, deletes, null, home => + { + if (home != null) + { + if (request.HasConditions && home.IsDirty) + throw new InvalidOperationException($"Backing store {home} has staged writes; flush before a conditional commit."); + return home.CommitBatch(request, wait); + } + + if (request.HasConditions && _stores.Count > 0) + throw new InvalidOperationException("A conditional batch names its home store through a record it upserts or removes."); + var inMemory = _entries.Values + .Select(entry => new CultPersistedRecord { Key = entry.Key.Value, SchemaId = entry.Descriptor.SchemaId, StoredAt = entry.StoredAt }) + .ToArray(); + return request.ConditionsHold(inMemory, _entries.Values) ? CultCommitOutcome.Committed : CultCommitOutcome.Mismatch; + }); + }); + } + + // Every add and remove passes here: single writes, committed batches, and loads (source set). + // Nothing lands unless all of it is admissible; land is the durable step between validation and memory. + private CultCommitOutcome Admit( + IReadOnlyList admitted, + IReadOnlyList evicted, CacheBackingStore? source, - bool raiseUpdate) + Func land) => Held(() => { - CultStoredDocument? existing = null; - _entries.TryGetValue(stored.Key.Value, out existing); - if (existing != null) + if (_held == null) + throw new InvalidOperationException("An admission reached the cache under a plain lock on its gate; every admission runs in a hold."); + var outcome = land(Validate(admitted, evicted, source)); + if (outcome != CultCommitOutcome.Committed) + return outcome; + foreach (var change in Apply(admitted, evicted, source)) + _held.Add((change, source != null)); + return CultCommitOutcome.Committed; + }); + + // Every admission runs in a hold. A nested hold on the same cache adds to the outermost one, which publishes + // exactly its own changes after it leaves the gate, before it returns, on its caller's thread: observers never + // run under the gate, and a write an observer makes is a new outermost hold. Cross-thread delivery order is not + // guaranteed; each change carries the Sequence it was admitted with. Every change is delivered even if an + // OnUpdate handler throws; then the first handler exception (an AggregateException for several) is rethrown. + // If the body threw, its exception wins. + internal T Held(Func body) + { + if (Monitor.IsEntered(_gate)) + return body(); + if (_held != null) + throw new InvalidOperationException( + "A cache hold was entered inside another cache's hold; a thread holds one cache's gate at a time."); + var mine = _held = new List<(Change Change, bool Loaded)>(); + T result; + try { - RemoveIndexes(existing); + lock (_gate) + result = body(); + } + catch + { + _held = null; + Publish(mine); + throw; } - _entries[stored.Key.Value] = stored; - _documentHandles.Remove(stored.Document); - _documentHandles.Add(stored.Document, new DocumentHandleBox(stored.Key)); - _soa.Upsert(stored); - AddIndexes(stored); + _held = null; + var failures = Publish(mine); + if (failures.Count == 1) + ExceptionDispatchInfo.Capture(failures[0]).Throw(); + if (failures.Count > 1) + throw new AggregateException(failures); + return result; + } - foreach (var store in _backingStores) + private List Publish(List<(Change Change, bool Loaded)> changes) + { + var failures = new List(); + foreach (var (change, loaded) in changes) { - if (store != source) + // R3 routes a throwing subscriber to its unhandled-exception handler; OnNext does not throw. + _changes.OnNext(change); + if (!loaded) + continue; + try { - store.Push(stored); + OnUpdate?.Invoke(change.Previous, change.Document); + } + catch (Exception exception) + { + failures.Add(exception); } } - if (source == null) - { - _hasUnflushedMutations = true; - } - else - { - RecomputeDirtyState(); - } - - PublishChange(stored, existing?.Document); - - if (raiseUpdate) - { - OnUpdate?.Invoke(existing?.Document, stored.Document); - } - - await Task.CompletedTask; - return stored; + return failures; } - private void RemoveStoredDocumentInternal( - CultStoredDocument stored, - CacheBackingStore? source, - bool raiseUpdate) + private CacheBackingStore? Validate( + IReadOnlyList admitted, + IReadOnlyList evicted, + CacheBackingStore? source) { - if (!_entries.TryRemove(stored.Key.Value, out var existing)) + var homes = new List(); + var globals = new Dictionary(_globals); + foreach (var stored in evicted) { - return; + var type = stored.Descriptor.DocumentType; + if (globals.TryGetValue(type, out var key) && key == stored.Key.Value) + globals.Remove(type); + if (source == null) + Claim(stored); } - RemoveIndexes(existing); - _documentHandles.Remove(existing.Document); - _soa.Remove(existing); - - foreach (var store in _backingStores) + foreach (var stored in admitted) { - if (store != source) + var descriptor = stored.Descriptor; + if (source == null) + Claim(stored); + else if (Home(descriptor.DocumentType) is var home && home != source) + throw new InvalidOperationException( + $"{descriptor.SchemaName} record {stored.Key.Value} was loaded from {source} but its home is {home?.ToString() ?? "no store"}."); + + // One key, one store: a key never moves between homes, and two stores never both deliver it. + if (_entries.TryGetValue(stored.Key.Value, out var present) && + Home(present.Descriptor.DocumentType) is var held && + Home(descriptor.DocumentType) is var incoming && + held != incoming) + throw new InvalidOperationException( + $"Record {stored.Key.Value} is held by {held?.ToString() ?? "memory"}; " + + $"{(source == null ? "writing" : "loading")} it as {descriptor.SchemaName} from {incoming?.ToString() ?? "memory"} would give the key a second store."); + + if (!descriptor.IsGlobal) + continue; + if (globals.TryGetValue(descriptor.DocumentType, out var existing) && existing != stored.Key.Value) + throw new InvalidOperationException( + $"{descriptor.SchemaName} is a global and already has record {existing}; {stored.Key.Value} would be a second."); + globals[descriptor.DocumentType] = stored.Key.Value; + } + + var distinct = homes.Distinct().ToArray(); + if (distinct.Length > 1) + throw new InvalidOperationException( + $"The batch spans {distinct[0]} and {distinct[1]}; a commit lands in one home store."); + return distinct.FirstOrDefault(); + + void Claim(CultStoredDocument stored) + { + var home = Home(stored.Descriptor.DocumentType); + if (home == null) { - store.Delete(existing); + if (_stores.Count > 0) + throw new InvalidOperationException($"No backing store is home to {stored.Descriptor.SchemaName}."); + return; } - } - if (source == null) - { - _hasUnflushedMutations = true; - } - else - { - RecomputeDirtyState(); - } - - PublishChange(existing, existing.Document, removed: true); - - if (raiseUpdate) - { - OnUpdate?.Invoke(existing.Document, null); + if (home.IsReadOnly) + throw new InvalidOperationException( + $"{stored.Descriptor.SchemaName} record {stored.Key.Value} routes to read-only backing store {home}."); + homes.Add(home); } } - private void InitializeGlobals() + private List Apply( + IReadOnlyList admitted, + IReadOnlyList evicted, + CacheBackingStore? source) { - foreach (var descriptor in _registry.AllDescriptors.Where(candidate => candidate.IsGlobal)) + var changes = new List(admitted.Count + evicted.Count); + foreach (var stored in evicted) { - var key = new CultRecordKey($"global:{descriptor.SchemaId}"); - if (_entries.ContainsKey(key.Value) || - _backingStores.Any(store => store.ContainsDurableRecord(key))) + if (!_entries.TryGetValue(stored.Key.Value, out var existing)) + continue; + Unindex(existing); + _entries.Remove(stored.Key.Value); + _handles.Remove(existing.Document); + changes.Add(new Change(CultCacheDocumentChangeKind.Removed, existing, null, existing.Document, ++_sequence)); + } + + foreach (var stored in admitted) + { + _entries.TryGetValue(stored.Key.Value, out var previous); + if (previous != null) { + Unindex(previous); + _handles.Remove(previous.Document); + } + + _entries[stored.Key.Value] = stored; + _handles.AddOrUpdate(stored.Document, new KeyBox(stored.Key)); + Index(stored); + changes.Add(new Change( + previous == null ? CultCacheDocumentChangeKind.Added : CultCacheDocumentChangeKind.Updated, + stored, + stored.Document, + previous?.Document, + ++_sequence)); + } + + if (source == null && _stores.Count == 0) + _dirtyInMemory = true; + return changes; + } + + // The most derived routed home type assignable from the document type, else the untyped store. + private CacheBackingStore? Home(Type documentType) + { + CacheBackingStore? untyped = null; + CacheBackingStore? routed = null; + Type? best = null; + foreach (var (store, homes) in _stores) + { + if (homes.Length == 0) + { + untyped = store; continue; } - if (descriptor.DocumentType.GetConstructor(Type.EmptyTypes) == null) + foreach (var home in homes) { - continue; + if (home.IsAssignableFrom(documentType) && (best == null || best.IsAssignableFrom(home))) + { + best = home; + routed = store; + } } + } - var instance = Activator.CreateInstance(descriptor.DocumentType); - if (instance == null) - { - continue; - } + return routed ?? untyped; + } - AddStoredDocumentInternal( - new CultStoredDocument( - key, - DateTimeOffset.UtcNow.ToString("O"), - descriptor, - instance), - source: null, - raiseUpdate: false).GetAwaiter().GetResult(); + private T? Single(string lookup, Func> keys) where T : class + { + lock (_gate) + { + var found = keys().Distinct().ToArray(); + if (found.Length > 1) + throw new InvalidOperationException( + $"{typeof(T).Name} {lookup} matches records {string.Join(", ", found)}; look it up by its concrete type."); + return found.Length == 1 && _entries.TryGetValue(found[0], out var stored) ? stored.Document as T : null; } } - private CultRecordKey ResolveKey(object document, CultDocumentDescriptor descriptor) - { - if (_documentHandles.TryGetValue(document, out var existing)) - { - return existing.Key; - } - - if (descriptor.IsGlobal) - { - return new CultRecordKey($"global:{descriptor.SchemaId}"); - } - - return new CultRecordKey(Guid.NewGuid().ToString("N")); - } - - private void AddIndexes(CultStoredDocument stored) + private void Index(CultStoredDocument stored) { + var type = stored.Descriptor.DocumentType; if (stored.Descriptor.IsGlobal) - { - _globalKeys[stored.Descriptor.DocumentType] = stored.Key.Value; - } - + _globals[type] = stored.Key.Value; if (stored.Descriptor.NameAccessor?.Invoke(stored.Document) is { Length: > 0 } name) - { - var map = _nameMaps.GetOrAdd( - stored.Descriptor.DocumentType, - _ => new ConcurrentDictionary(StringComparer.Ordinal)); - map[name] = stored.Key.Value; - } - + MapOf(_names, type)[name] = stored.Key.Value; foreach (var pair in stored.Descriptor.IndexAccessors) { var value = pair.Value(stored.Document); - if (string.IsNullOrWhiteSpace(value)) - { - continue; - } - - var map = _indexMaps.GetOrAdd( - (stored.Descriptor.DocumentType, pair.Key), - _ => new ConcurrentDictionary(StringComparer.Ordinal)); - map[value] = stored.Key.Value; + if (!string.IsNullOrWhiteSpace(value)) + MapOf(_indexes, (type, pair.Key))[value] = stored.Key.Value; } } - private void RemoveIndexes(CultStoredDocument stored) + // Documents are mutable, so a name or index value may have changed since it was indexed: drop by key. + private void Unindex(CultStoredDocument stored) { - if (stored.Descriptor.IsGlobal) - { - _globalKeys.TryRemove(stored.Descriptor.DocumentType, out _); - } - - if (stored.Descriptor.NameAccessor?.Invoke(stored.Document) is { Length: > 0 } name && - _nameMaps.TryGetValue(stored.Descriptor.DocumentType, out var nameMap)) - { - nameMap.TryRemove(name, out _); - } - - foreach (var pair in stored.Descriptor.IndexAccessors) - { - var value = pair.Value(stored.Document); - if (string.IsNullOrWhiteSpace(value)) - { - continue; - } - - if (_indexMaps.TryGetValue((stored.Descriptor.DocumentType, pair.Key), out var map)) - { - map.TryRemove(value, out _); - } - } + var type = stored.Descriptor.DocumentType; + if (_globals.TryGetValue(type, out var globalKey) && globalKey == stored.Key.Value) + _globals.Remove(type); + if (_names.TryGetValue(type, out var names)) + RemoveKey(names, stored.Key.Value); + foreach (var pair in _indexes.Where(pair => pair.Key.Type == type)) + RemoveKey(pair.Value, stored.Key.Value); } - private sealed class DocumentHandleBox + private static Dictionary MapOf(Dictionary> maps, TKey key) + where TKey : notnull { - public DocumentHandleBox(CultRecordKey key) + if (!maps.TryGetValue(key, out var map)) + maps[key] = map = new Dictionary(StringComparer.Ordinal); + return map; + } + + private static void RemoveKey(Dictionary map, string key) + { + foreach (var stale in map.Where(pair => pair.Value == key).Select(pair => pair.Key).ToArray()) + map.Remove(stale); + } + + private sealed class KeyBox + { + public KeyBox(CultRecordKey key) { Key = key; } @@ -1933,174 +1899,97 @@ namespace GameCult.Caching public CultRecordKey Key { get; } } - private void RecomputeDirtyState() + private sealed class Change { - _hasUnflushedMutations = _backingStores.Count == 0 - ? _hasUnflushedMutations - : _backingStores.Any(store => store.IsDirty); - } - - private void PublishChange(CultStoredDocument stored, object? previousDocument, bool removed = false) - { - var changeType = typeof(CultCacheDocumentChange<>).MakeGenericType(stored.Descriptor.DocumentType); - var change = Activator.CreateInstance( - changeType, - removed ? CultCacheDocumentChangeKind.Removed : - previousDocument == null ? CultCacheDocumentChangeKind.Added : - CultCacheDocumentChangeKind.Updated, - stored.Key, - removed ? null : stored.Document, - previousDocument); - if (change != null) + public Change(CultCacheDocumentChangeKind kind, CultStoredDocument stored, object? document, object? previous, long sequence) { - _changes.OnNext(change); + Kind = kind; + Stored = stored; + Document = document; + Previous = previous; + Sequence = sequence; } + + public CultCacheDocumentChangeKind Kind { get; } + public CultStoredDocument Stored { get; } + public object? Document { get; } + public object? Previous { get; } + public long Sequence { get; } } } - /// - /// Durable metadata available before a backing store hydrates a record payload. - /// - public sealed class CultPersistedRecordMetadata - { - /// Creates durable record metadata. - public CultPersistedRecordMetadata(string key, string schemaId, string storedAt) - { - Key = key; - SchemaId = schemaId; - StoredAt = storedAt; - } - - /// Gets the durable record key. - public string Key { get; } - /// Gets the durable schema identity. - public string SchemaId { get; } - /// Gets the record commit timestamp. - public string StoredAt { get; } - } - - /// - /// Base class for CultCache persistence adapters. - /// public abstract class CacheBackingStore : IDisposable { private CultDocumentRegistry? _registry; - private ILogger _logger = new NullLogger(); - private bool _isDirty; private CultSchemaMigrationReport[] _lastSchemaMigrationReports = Array.Empty(); - /// - /// Gets the attached document registry. - /// + protected CacheBackingStore(bool readOnly = false) + { + IsReadOnly = readOnly; + } + protected CultDocumentRegistry Registry => _registry ?? throw new InvalidOperationException("Backing store is not attached to a CultDocumentRegistry."); - /// - /// Gets the backing store entries by record key. - /// protected ConcurrentDictionary Entries { get; } = new(StringComparer.Ordinal); - /// - /// Gets or sets the backing store logger. - /// - public ILogger Logger - { - get => _logger; - set => _logger = value ?? new NullLogger(); - } + public bool IsReadOnly { get; } - /// - /// Gets whether the backing store holds staged mutations not yet durably flushed. - /// - public bool IsDirty - { - get => _isDirty; - protected set => _isDirty = value; - } + public bool IsDirty { get; protected set; } - /// - /// Gets the UTC timestamp of the last successful durable flush. - /// - public DateTimeOffset? LastSuccessfulFlushAtUtc { get; protected set; } - - /// - /// Gets or sets whether disposing the backing store should flush staged mutations first. - /// public bool FlushOnDispose { get; set; } - /// - /// Gets schema migration reports emitted during the most recent pull. - /// public IReadOnlyList LastSchemaMigrationReports => _lastSchemaMigrationReports; - /// - /// Publishes documents added by the backing store. - /// - public Subject EntryAdded { get; } = new(); - /// - /// Publishes documents updated by the backing store. - /// - public Subject EntryUpdated { get; } = new(); - /// - /// Publishes documents deleted by the backing store. - /// - public Subject EntryDeleted { get; } = new(); + // Set by the cache at attach. A pull hands over everything it loaded and dropped in one call before it adopts + // any of it; if the cache refuses a record the call throws and the store keeps its previous view. + protected internal Action, IReadOnlyList>? Loaded; + + private readonly object _detachedGate = new(); + internal CultCache? Cache; + + // Once attached, a hold on the cache's gate: the store and its cache share one lock, so a load calling back into + // the cache can never take the gate after the store lock, and a direct call publishes what it loaded when it + // returns. File locks are always taken inside it. + protected T Held(Func body) + { + if (Cache is { } cache) + return cache.Held(body); + lock (_detachedGate) + return body(); + } + + protected void Held(Action body) => Held(() => { body(); return true; }); internal void AttachRegistry(CultDocumentRegistry registry) { _registry = registry ?? throw new ArgumentNullException(nameof(registry)); } - /// - /// Pulls all persisted records into the backing store. - /// public abstract void PullAll(); - /// - /// Pulls records selected by durable metadata. Non-indexed stores fall back to a full pull. - /// - public virtual void PullSelected(Func selector) - { - if (selector == null) throw new ArgumentNullException(nameof(selector)); - PullAll(); - } - /// - /// Returns whether the backing store knows a record is durable, including records left cold by selective hydration. - /// - public virtual bool ContainsDurableRecord(CultRecordKey key) - { - return Entries.ContainsKey(key.Value); - } - /// - /// Pushes one stored document into the backing store. - /// public abstract void Push(CultStoredDocument entry); - /// - /// Deletes one stored document from the backing store. - /// public abstract void Delete(CultStoredDocument entry); - /// - /// Persists all current backing store entries. - /// - public abstract void PushAll(bool soft = false); - /// - /// Releases backing store event subjects. - /// + public abstract void PushAll(); + + // One durable step under the store's exclusive lock; conditions are evaluated against what is on disk then. + // A failure restores the store's staged view. + public abstract CultCommitOutcome CommitBatch(CultCommitRequest request, bool wait); + public virtual void Dispose() { - if (FlushOnDispose && IsDirty) + if (FlushOnDispose && IsDirty && !IsReadOnly) { PushAll(); } - - EntryAdded.Dispose(); - EntryUpdated.Dispose(); - EntryDeleted.Dispose(); } - /// - /// Converts a stored document into a persisted record. - /// + protected void ThrowIfReadOnly() + { + if (IsReadOnly) + throw new InvalidOperationException($"Backing store {this} is read-only."); + } + protected CultPersistedRecord ToPersistedRecord(CultStoredDocument entry, Func serializePayload) { return new CultPersistedRecord @@ -2112,9 +2001,6 @@ namespace GameCult.Caching }; } - /// - /// Converts a persisted record into a stored document. - /// protected CultStoredDocument ToStoredDocument( CultPersistedRecord record, IReadOnlyCollection catalog, @@ -2129,160 +2015,206 @@ namespace GameCult.Caching document); } - /// - /// Records schema migration reports captured during a pull. - /// protected void SetLastSchemaMigrationReports(IEnumerable reports) { _lastSchemaMigrationReports = reports?.ToArray() ?? Array.Empty(); } - /// - /// Marks the backing store as durably flushed. - /// protected void MarkFlushSucceeded() { IsDirty = false; - LastSuccessfulFlushAtUtc = DateTimeOffset.UtcNow; } } - /// - /// Base class for backing stores that persist a complete snapshot to one file. - /// public abstract class SingleFileBackingStore : CacheBackingStore { - /// - /// Creates a single-file backing store. - /// - protected SingleFileBackingStore(string filePath) + protected SingleFileBackingStore(string filePath, bool readOnly = false) + : base(readOnly) { FileInfo = new FileInfo(filePath); } - /// - /// Gets the file used by this backing store. - /// protected FileInfo FileInfo { get; } - /// - /// Serializes a full store snapshot. - /// protected abstract byte[] SerializeSnapshot(CultPersistedStoreSnapshot snapshot); - /// - /// Deserializes a full store snapshot. - /// protected abstract CultPersistedStoreSnapshot DeserializeSnapshot(byte[] data); - /// - /// Serializes one document payload. - /// protected abstract byte[] SerializePayload(object document); - /// - /// Deserializes one document payload. - /// protected abstract object DeserializePayload(Type documentType, byte[] payload); - /// - /// Loads every persisted record from disk. - /// - public override void PullAll() - { - FileInfo.Refresh(); - if (!FileInfo.Exists) - { - SetLastSchemaMigrationReports(Array.Empty()); - IsDirty = false; - return; - } + public override string ToString() => FileInfo.FullName; - // A single-file snapshot cannot distinguish local dirty keys from clean keys. - // Pulling while local mutations are staged would let an older disk snapshot - // erase those mutations before the next flush. Flush first; clean readers can - // still poll external snapshots incrementally. + public override void PullAll() => Held(PullAllCore); + + private void PullAllCore() + { + // A single-file snapshot cannot distinguish local dirty keys from clean keys. Pulling while local mutations + // are staged would let an older disk snapshot erase them before the next flush. Flush first. if (IsDirty) return; - CultPersistedStoreSnapshot snapshot; + var snapshot = ReadSnapshot(); + if (snapshot == null) + { + SetLastSchemaMigrationReports(Array.Empty()); + return; + } + + var reports = new List(snapshot.Records.Length); + var persisted = new Dictionary(StringComparer.Ordinal); + foreach (var record in snapshot.Records) + { + reports.Add(Registry.ResolvePersistedSchemaReport(record.SchemaId, snapshot.SchemaCatalog)); + var stored = ToStoredDocument(record, snapshot.SchemaCatalog, DeserializePayload); + persisted[stored.Key.Value] = stored; + } + + var loaded = persisted.Values + .Where(stored => !Entries.TryGetValue(stored.Key.Value, out var existing) || + existing.StoredAt != stored.StoredAt || + existing.Descriptor.SchemaId != stored.Descriptor.SchemaId) + .ToArray(); + var dropped = Entries.Values.Where(existing => !persisted.ContainsKey(existing.Key.Value)).ToArray(); + if (loaded.Length > 0 || dropped.Length > 0) + Loaded?.Invoke(loaded, dropped); + + foreach (var stored in dropped) + Entries.TryRemove(stored.Key.Value, out _); + foreach (var stored in loaded) + Entries[stored.Key.Value] = stored; + SetLastSchemaMigrationReports(reports); + } + + public override void Push(CultStoredDocument entry) + { + ThrowIfReadOnly(); + Held(() => + { + Entries[entry.Key.Value] = entry; + IsDirty = true; + }); + } + + public override void Delete(CultStoredDocument entry) + { + ThrowIfReadOnly(); + Held(() => + { + Entries.TryRemove(entry.Key.Value, out _); + IsDirty = true; + }); + } + + // A plain flush writes this store's whole view under the lock: two writers never interleave bytes, but the last + // one wins. Processes sharing a store must use conditional commit. + public override void PushAll() + { + ThrowIfReadOnly(); + Held(() => + { + using (AcquireLock(wait: true)) + { + WriteSnapshot( + Entries.Values.Select(entry => ToPersistedRecord(entry, SerializePayload)), + Entries.Values.Select(entry => entry.Descriptor.ToCatalogEntry())); + } + + MarkFlushSucceeded(); + }); + } + + public override CultCommitOutcome CommitBatch(CultCommitRequest request, bool wait) + { + ThrowIfReadOnly(); + return Held(() => CommitBatchCore(request, wait)); + } + + private CultCommitOutcome CommitBatchCore(CultCommitRequest request, bool wait) + { + using var fileLock = AcquireLock(wait); + if (fileLock == null) + return CultCommitOutcome.Contended; + + var disk = ReadSnapshot() ?? new CultPersistedStoreSnapshot(); + if (!request.ConditionsHold(disk.Records, Entries.Values)) + return CultCommitOutcome.Mismatch; + + // An unconditional commit writes what a flush would, this store's whole view plus the batch: last-writer-wins, + // and staged single writes land with it. A conditional commit (store clean) lands the batch onto the file as it is. + var ontoDisk = request.HasConditions && !IsDirty; + var records = ontoDisk + ? disk.Records.ToDictionary(record => record.Key, StringComparer.Ordinal) + : Entries.Values.Select(entry => ToPersistedRecord(entry, SerializePayload)).ToDictionary(record => record.Key, StringComparer.Ordinal); + var catalog = ontoDisk + ? disk.SchemaCatalog + : Entries.Values.Select(entry => entry.Descriptor.ToCatalogEntry()); + foreach (var entry in request.Deletes) + records.Remove(entry.Key.Value); + foreach (var entry in request.Upserts) + records[entry.Key.Value] = ToPersistedRecord(entry, SerializePayload); + + WriteSnapshot(records.Values, catalog.Concat(request.Upserts.Select(entry => entry.Descriptor.ToCatalogEntry()))); + foreach (var entry in request.Deletes) + Entries.TryRemove(entry.Key.Value, out _); + foreach (var entry in request.Upserts) + Entries[entry.Key.Value] = entry; + MarkFlushSucceeded(); + return CultCommitOutcome.Committed; + } + + private CultPersistedStoreSnapshot? ReadSnapshot() + { + FileInfo.Refresh(); + if (!FileInfo.Exists) + return null; try { - snapshot = DeserializeSnapshot(ReadAllBytesShared(FileInfo.FullName)); + return DeserializeSnapshot(ReadAllBytesShared(FileInfo.FullName)); } catch (FileNotFoundException) { - FileInfo.Refresh(); - if (!FileInfo.Exists) - return; - snapshot = DeserializeSnapshot(ReadAllBytesShared(FileInfo.FullName)); + return null; } - var reports = new List(snapshot.Records.Length); - var persistedKeys = new HashSet(StringComparer.Ordinal); - foreach (var record in snapshot.Records) - { - persistedKeys.Add(record.Key); - reports.Add(Registry.ResolvePersistedSchemaReport(record.SchemaId, snapshot.SchemaCatalog)); - var stored = ToStoredDocument(record, snapshot.SchemaCatalog, DeserializePayload); - if (Entries.TryGetValue(stored.Key.Value, out var existing) && - string.Equals(existing.StoredAt, stored.StoredAt, StringComparison.Ordinal) && - string.Equals(existing.Descriptor.SchemaId, stored.Descriptor.SchemaId, StringComparison.Ordinal)) - { - continue; - } - Entries[stored.Key.Value] = stored; - if (existing == null) - EntryAdded.OnNext(stored); - else - EntryUpdated.OnNext(stored); - } - - foreach (var removedKey in Entries.Keys.Where(key => !persistedKeys.Contains(key)).ToArray()) - { - if (Entries.TryRemove(removedKey, out var removed)) - EntryDeleted.OnNext(removed); - } - - SetLastSchemaMigrationReports(reports); - IsDirty = false; } - /// - /// Stages one stored document in memory. - /// - public override void Push(CultStoredDocument entry) - { - Entries[entry.Key.Value] = entry; - IsDirty = true; - } - - /// - /// Removes one staged document. - /// - public override void Delete(CultStoredDocument entry) - { - Entries.TryRemove(entry.Key.Value, out _); - IsDirty = true; - } - - /// - /// Writes the staged snapshot to disk. - /// - public override void PushAll(bool soft = false) + private void WriteSnapshot(IEnumerable records, IEnumerable catalog) { + var ordered = records.OrderBy(record => record.Key, StringComparer.Ordinal).ToArray(); + var used = new HashSet(ordered.Select(record => record.SchemaId), StringComparer.Ordinal); var snapshot = new CultPersistedStoreSnapshot { - SchemaCatalog = Entries.Values - .Select(entry => entry.Descriptor.ToCatalogEntry()) + SchemaCatalog = catalog .GroupBy(entry => entry.SchemaId, StringComparer.Ordinal) .Select(group => group.First()) + .Where(entry => used.Contains(entry.SchemaId)) .OrderBy(entry => entry.SchemaName, StringComparer.Ordinal) .ToArray(), - Records = Entries.Values - .OrderBy(entry => entry.Key.Value, StringComparer.Ordinal) - .Select(entry => ToPersistedRecord(entry, SerializePayload)) - .ToArray() + Records = ordered }; Directory.CreateDirectory(FileInfo.DirectoryName!); WriteSnapshotAtomically(FileInfo.FullName, SerializeSnapshot(snapshot)); - MarkFlushSucceeded(); + } + + // The lock is a sidecar opened exclusively, which excludes other handles in this process and in others alike. + private FileStream? AcquireLock(bool wait) + { + Directory.CreateDirectory(FileInfo.DirectoryName!); + var lockPath = FileInfo.FullName + ".lock"; + var started = Stopwatch.StartNew(); + while (true) + { + try + { + return new FileStream(lockPath, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None, bufferSize: 1); + } + catch (IOException) when (!wait) + { + return null; + } + catch (IOException) when (started.Elapsed < TimeSpan.FromSeconds(30)) + { + Thread.Sleep(10); + } + } } private static void WriteSnapshotAtomically(string path, byte[] payload) diff --git a/src/GameCult.Caching/CultDocumentContracts.cs b/src/GameCult.Caching/CultDocumentContracts.cs index 46dca479..0d4f1643 100644 --- a/src/GameCult.Caching/CultDocumentContracts.cs +++ b/src/GameCult.Caching/CultDocumentContracts.cs @@ -2,15 +2,9 @@ using System; namespace GameCult.Caching { - /// - /// Marks a type as a CultCache document with a stable schema name and version. - /// [AttributeUsage(AttributeTargets.Class, Inherited = false)] public sealed class CultDocumentAttribute : Attribute { - /// - /// Creates a document marker for the supplied schema identity. - /// public CultDocumentAttribute(string schemaName, string schemaVersion) { SchemaName = string.IsNullOrWhiteSpace(schemaName) @@ -21,172 +15,97 @@ namespace GameCult.Caching : schemaVersion; } - /// - /// Gets the stable schema name used by CultCache and CultNet. - /// public string SchemaName { get; } - /// - /// Gets the semantic schema version string for this document shape. - /// public string SchemaVersion { get; } } - /// - /// Marks a document type as having one global record in a cache. - /// [AttributeUsage(AttributeTargets.Class, Inherited = false)] public sealed class CultGlobalAttribute : Attribute { } - /// - /// Marks the member that supplies the human-readable document name. - /// [AttributeUsage(AttributeTargets.Field | AttributeTargets.Property, Inherited = true)] public sealed class CultNameAttribute : Attribute { } - /// - /// Marks a member as an indexed lookup value. - /// [AttributeUsage(AttributeTargets.Field | AttributeTargets.Property, Inherited = true, AllowMultiple = false)] public sealed class CultIndexAttribute : Attribute { - /// - /// Creates an index marker with an optional alias. - /// public CultIndexAttribute(string? alias = null) { Alias = alias; } - /// - /// Gets the external index alias, or null to use the member name. - /// public string? Alias { get; } } - /// - /// Marks a member as a reference to another CultCache document. - /// [AttributeUsage(AttributeTargets.Field | AttributeTargets.Property, Inherited = true)] public sealed class CultReferenceAttribute : Attribute { - /// - /// Creates a document reference marker. - /// public CultReferenceAttribute(Type? targetType = null, bool many = false) { TargetType = targetType; Many = many; } - /// - /// Gets the explicit referenced document type, when one is required. - /// public Type? TargetType { get; } - /// - /// Gets whether this member stores many references rather than one reference. - /// public bool Many { get; } } - /// - /// Identifies a record inside a CultCache backing store. - /// public readonly struct CultRecordKey : IEquatable { - /// - /// Creates a record key from its persisted string value. - /// public CultRecordKey(string value) { Value = value ?? string.Empty; } - /// - /// Gets the persisted key value. - /// public string Value { get; } - /// - /// Returns the persisted key value. - /// public override string ToString() => Value; - /// - /// Returns whether this key and another key contain the same persisted value. - /// public bool Equals(CultRecordKey other) => string.Equals(Value, other.Value, StringComparison.Ordinal); - /// - /// Returns whether this key and another object contain the same persisted value. - /// public override bool Equals(object? obj) => obj is CultRecordKey other && Equals(other); - /// - /// Returns a hash code for the persisted key value. - /// public override int GetHashCode() => StringComparer.Ordinal.GetHashCode(Value); } - /// - /// Carries the persisted record key for a typed document. - /// public readonly struct CultRecordHandle { - /// - /// Creates a typed document handle. - /// public CultRecordHandle(CultRecordKey key) { Key = key; } - /// - /// Gets the persisted record key. - /// public CultRecordKey Key { get; } - /// - /// Returns the persisted key value. - /// public override string ToString() => Key.ToString(); } - /// - /// Stores a typed reference to a CultCache document. - /// - public readonly struct CultRecordRef + // The key of any CultRecordRef, for code that holds one without knowing T. + public interface ICultRecordRef + { + CultRecordKey Key { get; } + } + + public readonly struct CultRecordRef : ICultRecordRef { - /// - /// Creates a typed document reference. - /// public CultRecordRef(CultRecordKey key) { Key = key; } - /// - /// Gets the referenced record key. - /// public CultRecordKey Key { get; } - /// - /// Resolves the referenced record against the provided cache. - /// public T? Resolve(CultCache cache) { if (cache == null) throw new ArgumentNullException(nameof(cache)); return cache.Get(Key) is T typed ? typed : default; } - /// - /// Returns the referenced key value. - /// public override string ToString() => Key.ToString(); } } diff --git a/src/GameCult.Caching/CultGeneratedDocumentMetadata.cs b/src/GameCult.Caching/CultGeneratedDocumentMetadata.cs deleted file mode 100644 index 7314c87f..00000000 --- a/src/GameCult.Caching/CultGeneratedDocumentMetadata.cs +++ /dev/null @@ -1,308 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Reflection; -using System.Text; - -namespace GameCult.Caching -{ - /// - /// Registers a generated CultCache document metadata provider with an assembly. - /// - [AttributeUsage(AttributeTargets.Assembly, AllowMultiple = true)] - public sealed class CultGeneratedDocumentMetadataProviderAttribute : Attribute - { - /// - /// Creates an assembly metadata provider registration. - /// - public CultGeneratedDocumentMetadataProviderAttribute(Type providerType) - { - ProviderType = providerType ?? throw new ArgumentNullException(nameof(providerType)); - } - - /// - /// Gets the provider type that can create generated document definitions. - /// - public Type ProviderType { get; } - } - - /// - /// Supplies generated CultCache document metadata for an assembly. - /// - public interface ICultGeneratedDocumentMetadataProvider - { - /// - /// Gets the generated document definitions exposed by the provider. - /// - IEnumerable GetDocumentDefinitions(); - } - - /// - /// Describes one generated CultCache document contract. - /// - public sealed class CultGeneratedDocumentDefinition - { - /// - /// Creates a generated document contract definition. - /// - public CultGeneratedDocumentDefinition( - Type documentType, - string schemaName, - string schemaVersion, - bool isGlobal, - string? nameMember, - Func? nameAccessor, - Func? serializePayload, - Func? deserializePayload, - IReadOnlyList indexAccessors, - IReadOnlyList members) - { - DocumentType = documentType ?? throw new ArgumentNullException(nameof(documentType)); - SchemaName = string.IsNullOrWhiteSpace(schemaName) - ? throw new ArgumentException("Schema name must be non-empty.", nameof(schemaName)) - : schemaName; - SchemaVersion = string.IsNullOrWhiteSpace(schemaVersion) - ? throw new ArgumentException("Schema version must be non-empty.", nameof(schemaVersion)) - : schemaVersion; - IsGlobal = isGlobal; - NameMember = nameMember; - NameAccessor = nameAccessor; - SerializePayload = serializePayload; - DeserializePayload = deserializePayload; - IndexAccessors = indexAccessors ?? Array.Empty(); - Members = members ?? Array.Empty(); - } - - /// - /// Gets the CLR document type. - /// - public Type DocumentType { get; } - - /// - /// Gets the stable CultCache schema name. - /// - public string SchemaName { get; } - - /// - /// Gets the schema version string. - /// - public string SchemaVersion { get; } - - /// - /// Gets whether this document type stores one global record. - /// - public bool IsGlobal { get; } - - /// - /// Gets the member used as the document name, if any. - /// - public string? NameMember { get; } - - /// - /// Gets the generated name accessor, if any. - /// - public Func? NameAccessor { get; } - - /// - /// Gets the generated payload serializer, if one was emitted. - /// - public Func? SerializePayload { get; } - - /// - /// Gets the generated payload deserializer, if one was emitted. - /// - public Func? DeserializePayload { get; } - - /// - /// Gets the generated index accessors. - /// - public IReadOnlyList IndexAccessors { get; } - - /// - /// Gets the generated persisted member definitions. - /// - public IReadOnlyList Members { get; } - } - - /// - /// Describes a generated accessor for one CultCache index. - /// - public sealed class CultGeneratedDocumentIndexAccessor - { - /// - /// Creates a generated index accessor. - /// - public CultGeneratedDocumentIndexAccessor(string alias, Func accessor) - { - Alias = string.IsNullOrWhiteSpace(alias) - ? throw new ArgumentException("Alias must be non-empty.", nameof(alias)) - : alias; - Accessor = accessor ?? throw new ArgumentNullException(nameof(accessor)); - } - - /// - /// Gets the index alias. - /// - public string Alias { get; } - - /// - /// Gets the function that extracts the index value. - /// - public Func Accessor { get; } - } - - /// - /// Describes one generated persisted document member. - /// - public sealed class CultGeneratedDocumentMemberDefinition - { - /// - /// Creates a generated persisted member definition. - /// - public CultGeneratedDocumentMemberDefinition( - string memberName, - int slot, - string typeName, - bool isReference, - bool isMany, - string? targetSchemaName, - bool isName, - string? indexAlias) - { - MemberName = string.IsNullOrWhiteSpace(memberName) - ? throw new ArgumentException("Member name must be non-empty.", nameof(memberName)) - : memberName; - Slot = slot; - TypeName = string.IsNullOrWhiteSpace(typeName) - ? throw new ArgumentException("Type name must be non-empty.", nameof(typeName)) - : typeName; - IsReference = isReference; - IsMany = isMany; - TargetSchemaName = targetSchemaName; - IsName = isName; - IndexAlias = indexAlias; - } - - /// - /// Gets the CLR member name. - /// - public string MemberName { get; } - - /// - /// Gets the persisted slot number. - /// - public int Slot { get; } - - /// - /// Gets the persisted type name. - /// - public string TypeName { get; } - - /// - /// Gets whether this member stores a document reference. - /// - public bool IsReference { get; } - - /// - /// Gets whether this member stores many references. - /// - public bool IsMany { get; } - - /// - /// Gets the target schema name for reference members. - /// - public string? TargetSchemaName { get; } - - /// - /// Gets whether this member is the document name. - /// - public bool IsName { get; } - - /// - /// Gets the index alias for indexed members. - /// - public string? IndexAlias { get; } - } - - internal static class CultGeneratedDocumentMetadataLoader - { - public static IEnumerable LoadDefinitions() - { - return AppDomain.CurrentDomain.GetAssemblies() - .Where(assembly => !assembly.IsDynamic) - .SelectMany(LoadDefinitions); - } - - public static IEnumerable LoadDefinitions(Assembly assembly) - { - foreach (var attribute in assembly.GetCustomAttributes()) - { - if (!typeof(ICultGeneratedDocumentMetadataProvider).IsAssignableFrom(attribute.ProviderType)) - { - continue; - } - - if (Activator.CreateInstance(attribute.ProviderType) is not ICultGeneratedDocumentMetadataProvider provider) - { - continue; - } - - foreach (var definition in provider.GetDocumentDefinitions()) - { - yield return definition; - } - } - } - } - - internal static class CultSchemaTypeNames - { - public static string FromType(Type type) - { - if (type.IsArray) - { - return FromType(type.GetElementType()!) + "[]"; - } - - if (type.IsGenericType) - { - var definition = type.GetGenericTypeDefinition(); - var name = definition.FullName ?? definition.Name; - var tickIndex = name.IndexOf('`'); - if (tickIndex >= 0) - { - name = name[..tickIndex]; - } - - var arguments = string.Join(", ", type.GetGenericArguments().Select(FromType)); - return $"{name}<{arguments}>"; - } - - return type.FullName ?? type.Name; - } - - public static string EscapeForLiteral(string value) - { - if (string.IsNullOrEmpty(value)) - { - return value; - } - - var builder = new StringBuilder(value.Length); - foreach (var character in value) - { - builder.Append(character switch - { - '\\' => "\\\\", - '"' => "\\\"", - '\r' => "\\r", - '\n' => "\\n", - '\t' => "\\t", - _ => character.ToString() - }); - } - - return builder.ToString(); - } - } -} diff --git a/src/GameCult.Unity/Assets/Caching/Runtime/CultCacheInspectorAttributes.cs b/src/GameCult.Caching/CultInspectorAttributes.cs similarity index 57% rename from src/GameCult.Unity/Assets/Caching/Runtime/CultCacheInspectorAttributes.cs rename to src/GameCult.Caching/CultInspectorAttributes.cs index b7202791..881208ea 100644 --- a/src/GameCult.Unity/Assets/Caching/Runtime/CultCacheInspectorAttributes.cs +++ b/src/GameCult.Caching/CultInspectorAttributes.cs @@ -1,10 +1,10 @@ using System; -namespace GameCult.Unity.Caching +namespace GameCult.Caching { - /// - /// Overrides the label shown for a CultCache document member in Unity editor tooling. - /// + // Inspection metadata for document members. Engine-free: headless models carry it, and every inspector lowering + // (the Unity editor Studio, a runtime CultUI panel) reads it through CultInspectorModel. + [AttributeUsage(AttributeTargets.Field | AttributeTargets.Property | AttributeTargets.Class, Inherited = true)] public sealed class CultInspectorLabelAttribute : Attribute { @@ -16,25 +16,17 @@ namespace GameCult.Unity.Caching public string Label { get; } } - /// - /// Hides a CultCache document member from Unity editor tooling. - /// [AttributeUsage(AttributeTargets.Field | AttributeTargets.Property, Inherited = true)] public sealed class CultInspectorHiddenAttribute : Attribute { } - /// - /// Shows a CultCache document member without allowing editor mutation. - /// [AttributeUsage(AttributeTargets.Field | AttributeTargets.Property, Inherited = true)] public sealed class CultInspectorReadOnlyAttribute : Attribute { } - /// - /// Controls member ordering in Unity editor tooling. Lower values appear first. - /// + // Lower values appear first; members without it keep their slot order. [AttributeUsage(AttributeTargets.Field | AttributeTargets.Property, Inherited = true)] public sealed class CultInspectorOrderAttribute : Attribute { @@ -46,9 +38,6 @@ namespace GameCult.Unity.Caching public int Order { get; } } - /// - /// Draws a string member as a multi-line text area. - /// [AttributeUsage(AttributeTargets.Field | AttributeTargets.Property, Inherited = true)] public sealed class CultInspectorTextAreaAttribute : Attribute { @@ -62,9 +51,6 @@ namespace GameCult.Unity.Caching public int MaxLines { get; } } - /// - /// Draws numeric members with a slider. - /// [AttributeUsage(AttributeTargets.Field | AttributeTargets.Property, Inherited = true)] public sealed class CultInspectorRangeAttribute : Attribute { @@ -78,17 +64,33 @@ namespace GameCult.Unity.Caching public float Max { get; } } - /// - /// Draws a string member as a Unity asset path picker. - /// + // A string member holding an engine asset path. AssetType is the engine's asset type, named by the consumer + // (typeof(UnityEngine.Texture2D) in a Unity assembly); a lowering that does not know the type ignores it. [AttributeUsage(AttributeTargets.Field | AttributeTargets.Property, Inherited = true)] public sealed class CultInspectorAssetPathAttribute : Attribute { - public CultInspectorAssetPathAttribute(Type assetType = null) + public CultInspectorAssetPathAttribute(Type? assetType = null) { AssetType = assetType; } - public Type AssetType { get; } + public Type? AssetType { get; } + } + + // Marks a lowering's drawer class as the drawer for a claimed type. Claimed is either an Attribute subclass (the + // drawer draws every member carrying that attribute) or a value type (every value of that type, or of any closed + // form when Claimed is an open generic definition). Attribute claims win over type claims, which win over the + // built-in kinds. A claim two drawers make is used by neither. + [AttributeUsage(AttributeTargets.Class, Inherited = false)] + public sealed class CultInspectorDrawerAttribute : Attribute + { + public CultInspectorDrawerAttribute(Type claimed) + { + Claimed = claimed ?? throw new ArgumentNullException(nameof(claimed)); + } + + public Type Claimed { get; } + + public bool ClaimsAttribute => typeof(Attribute).IsAssignableFrom(Claimed); } } diff --git a/src/GameCult.Caching/CultInspectorModel.cs b/src/GameCult.Caching/CultInspectorModel.cs new file mode 100644 index 00000000..f40d9055 --- /dev/null +++ b/src/GameCult.Caching/CultInspectorModel.cs @@ -0,0 +1,630 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using MessagePack; + +namespace GameCult.Caching +{ + // The engine-free inspection model: what an inspector draws for a document and which edits it may make. Lowerings + // (the Unity editor Studio's IMGUI, a runtime CultUI panel) own only widgets and layout; members, metadata, value + // shapes, drawer claims and edit rules come from here. + + public enum CultInspectorValueKind + { + Unsupported, + String, + Integer, + Float, + Bool, + Enum, + RecordRef, + List, + Dictionary, + Union, + Nested + } + + // What the inspection attributes on one member say. A bare value has none. + public sealed class CultInspectorMetadata + { + internal static readonly CultInspectorMetadata None = new CultInspectorMetadata(null); + + internal CultInspectorMetadata(MemberInfo? member) + { + Attributes = member == null ? Array.Empty() : member.GetCustomAttributes(true).OfType().ToArray(); + var label = Find()?.Label; + Label = string.IsNullOrWhiteSpace(label) ? null : label; + Hidden = Find() != null; + ReadOnly = Find() != null; + Order = Find()?.Order; + Range = Find(); + TextArea = Find(); + AssetPath = Find(); + } + + public IReadOnlyList Attributes { get; } + + // The declared label, or null; the lowering derives one from the member name. + public string? Label { get; } + public bool Hidden { get; } + public bool ReadOnly { get; } + public int? Order { get; } + public CultInspectorRangeAttribute? Range { get; } + public CultInspectorTextAreaAttribute? TextArea { get; } + public CultInspectorAssetPathAttribute? AssetPath { get; } + + public T? Find() where T : Attribute => Attributes.OfType().FirstOrDefault(); + } + + public sealed class CultInspectorMember + { + internal CultInspectorMember(MemberInfo member, Type valueType, int slot, bool assignable, CultInspectorMetadata metadata) + { + Member = member; + ValueType = valueType; + Slot = slot; + IsAssignable = assignable; + Metadata = metadata; + IsReadOnly = metadata.ReadOnly || !assignable; + } + + public MemberInfo Member { get; } + public string Name => Member.Name; + public Type ValueType { get; } + public int Slot { get; } + public CultInspectorMetadata Metadata { get; } + + // SetValue works: MessagePack's own rule, the one the registry persists by (a public setter or non-readonly field, or + // any setter or readonly field under AllowPrivate). + public bool IsAssignable { get; } + + // The lowering offers no edit: declared [CultInspectorReadOnly], or not assignable. + public bool IsReadOnly { get; } + + public object? GetValue(object target) => + Member is FieldInfo field ? field.GetValue(target) : ((PropertyInfo)Member).GetValue(target); + + public void SetValue(object target, object? value) + { + if (!IsAssignable) + throw new InvalidOperationException($"{Member.DeclaringType?.Name}.{Name} is not assignable."); + if (Member is FieldInfo field) field.SetValue(target, value); + else ((PropertyInfo)Member).SetValue(target, value); + } + } + + public sealed class CultInspectorShape + { + internal CultInspectorShape(Type type, CultInspectorValueKind kind) + { + Type = type; + Kind = kind; + } + + public Type Type { get; } + public CultInspectorValueKind Kind { get; } + + // List + public Type? ElementType { get; internal set; } + + // Dictionary + public Type? KeyType { get; internal set; } + public Type? ValueType { get; internal set; } + + // RecordRef: the T of CultRecordRef. + public Type? RecordTarget { get; internal set; } + + // Union: the declared [Union] subtypes in key order, and nothing else. + public IReadOnlyList UnionChoices { get; internal set; } = Array.Empty(); + + // Nested: the members in inspection order. + public IReadOnlyList Members { get; internal set; } = Array.Empty(); + + // Unsupported: why. + public string? Reason { get; internal set; } + + internal Type? BuildType { get; set; } + } + + public readonly struct CultInspectorClaim + { + internal CultInspectorClaim(Type? drawer, string? conflict) + { + Drawer = drawer; + Conflict = conflict; + } + + // The claiming drawer class, or null for the lowering's built-in drawing. + public Type? Drawer { get; } + + // Set when the claim is contested; the lowering shows it instead of drawing. + public string? Conflict { get; } + } + + // Which lowering drawer draws a value: a claim on one of the member's attributes, then a claim on the value's type + // (exact, then its open generic definition), then none. A lowering builds this once from the drawer classes it + // discovers; an invalid drawer or a contested claim is reported in Errors and draws nothing. An attribute claim + // reaches every value drawn under the member, list elements and dictionary keys and values included, so the drawer + // decides by the value type it is handed. + public sealed class CultInspectorDrawerClaims + { + private readonly Dictionary _claims = new Dictionary(); + private readonly Dictionary _contested = new Dictionary(); + private readonly List _errors = new List(); + private readonly Dictionary<(Type, MemberInfo?), CultInspectorClaim> _resolved = new Dictionary<(Type, MemberInfo?), CultInspectorClaim>(); + + public CultInspectorDrawerClaims(IEnumerable drawerTypes, Type drawerContract) + { + if (drawerTypes == null) throw new ArgumentNullException(nameof(drawerTypes)); + if (drawerContract == null) throw new ArgumentNullException(nameof(drawerContract)); + foreach (var drawer in drawerTypes) + { + var attribute = drawer.GetCustomAttribute(false); + if (attribute == null) + continue; + if (!drawerContract.IsAssignableFrom(drawer) || drawer.IsAbstract || drawer.GetConstructor(Type.EmptyTypes) == null) + { + _errors.Add($"{drawer.FullName} is marked [CultInspectorDrawer] but is not a concrete {drawerContract.Name} with a parameterless constructor."); + continue; + } + + var claimed = attribute.Claimed; + if (_contested.TryGetValue(claimed, out var claimants)) + { + _contested[claimed] = claimants + ", " + drawer.FullName; + } + else if (_claims.TryGetValue(claimed, out var holder)) + { + _claims.Remove(claimed); + _contested[claimed] = holder.FullName + ", " + drawer.FullName; + } + else + { + _claims[claimed] = drawer; + } + } + + foreach (var pair in _contested.OrderBy(pair => pair.Key.FullName, StringComparer.Ordinal)) + _errors.Add($"{pair.Value} all claim {pair.Key.FullName}; none of them draws it."); + } + + public IReadOnlyList Errors => _errors; + + public CultInspectorClaim Resolve(Type valueType, MemberInfo? member) + { + if (valueType == null) throw new ArgumentNullException(nameof(valueType)); + lock (_resolved) + { + if (!_resolved.TryGetValue((valueType, member), out var claim)) + _resolved[(valueType, member)] = claim = ResolveCore(valueType, member); + return claim; + } + } + + private CultInspectorClaim ResolveCore(Type valueType, MemberInfo? member) + { + Type? drawer = null; + foreach (var attribute in member?.GetCustomAttributes(true).OfType() ?? Enumerable.Empty()) + { + for (var type = attribute.GetType(); type != null && type != typeof(Attribute); type = type.BaseType) + { + if (_contested.TryGetValue(type, out var claimants)) + return new CultInspectorClaim(null, $"[{type.Name}] is claimed by {claimants}"); + if (!_claims.TryGetValue(type, out var claimant)) + continue; + if (drawer != null && drawer != claimant) + return new CultInspectorClaim(null, $"{member!.Name} carries attributes claimed by {drawer.FullName} and {claimant.FullName}"); + drawer = claimant; + break; + } + } + + if (drawer != null) + return new CultInspectorClaim(drawer, null); + foreach (var type in valueType.IsGenericType ? new[] { valueType, valueType.GetGenericTypeDefinition() } : new[] { valueType }) + { + if (_contested.TryGetValue(type, out var claimants)) + return new CultInspectorClaim(null, $"{type.FullName} is claimed by {claimants}"); + if (_claims.TryGetValue(type, out var claimant)) + return new CultInspectorClaim(claimant, null); + } + + return default; + } + } + + // A private copy of one stored document. A lowering draws into Document; the cached object is never touched. + public sealed class CultInspectorEdit + { + internal CultInspectorEdit(CultStoredDocument source, object document) + { + Source = source; + Document = document; + } + + public CultStoredDocument Source { get; } + public object Document { get; } + public bool IsSpent { get; private set; } + + public bool IsFor(CultStoredDocument record) => !IsSpent && ReferenceEquals(Source, record); + + // Upserts the copy under the source key. Committed, the cache holds the copy; refused, the cached object is + // unchanged. Either way the edit is spent and the copy belongs to nobody: begin a new edit from the cache. + public bool Commit(CultCache cache, out string? error) + { + if (cache == null) throw new ArgumentNullException(nameof(cache)); + if (IsSpent) throw new InvalidOperationException("This edit was already committed; begin a new edit."); + IsSpent = true; + try + { + cache.UpsertAsync(Source.Descriptor.DocumentType, Document, Source.Key).GetAwaiter().GetResult(); + error = null; + return true; + } + catch (Exception exception) + { + error = exception.GetBaseException().Message; + return false; + } + } + } + + public sealed class CultInspectorModel + { + private const string RecordRefIdentity = "ref:"; + private const BindingFlags Public = BindingFlags.Public | BindingFlags.Instance; + + private static readonly HashSet IntegerTypes = new HashSet + { + typeof(int), typeof(long), typeof(uint), typeof(short), typeof(ushort), typeof(byte), typeof(sbyte) + }; + + private readonly Func _serialize; + private readonly Func _deserialize; + private readonly Dictionary _shapes = new Dictionary(); + private readonly Dictionary _metadata = new Dictionary(); + + // serialize (value, value type, owning document type) and deserialize are the store's codec + // (CultCacheMessagePack.CreateInspectorModel for .cc stores). Edits clone documents through both; dictionary keys + // compare by serialize under their owning document, exactly as the store writes them inside it, so serialize must + // take any value, not only documents. + public CultInspectorModel(CultDocumentRegistry registry, Func serialize, Func deserialize) + { + Registry = registry ?? throw new ArgumentNullException(nameof(registry)); + _serialize = serialize ?? throw new ArgumentNullException(nameof(serialize)); + _deserialize = deserialize ?? throw new ArgumentNullException(nameof(deserialize)); + } + + public CultDocumentRegistry Registry { get; } + + public static string RecordLabel(CultStoredDocument record) + { + var name = record.Descriptor.NameAccessor?.Invoke(record.Document); + return string.IsNullOrWhiteSpace(name) ? record.Key.Value : name!; + } + + // The key a CultRecordRef holds; null and "" are both the empty key. + public static string RecordKey(object? recordRef) => (recordRef as ICultRecordRef)?.Key.Value ?? string.Empty; + + public CultInspectorShape ShapeOf(Type type) + { + if (type == null) throw new ArgumentNullException(nameof(type)); + lock (_shapes) + { + if (!_shapes.TryGetValue(type, out var shape)) + _shapes[type] = shape = Classify(type); + return shape; + } + } + + public IReadOnlyList MembersOf(Type type) + { + var shape = ShapeOf(type); + return shape.Kind == CultInspectorValueKind.Nested ? shape.Members : Array.Empty(); + } + + public CultInspectorMetadata MetadataOf(MemberInfo? member) + { + if (member == null) + return CultInspectorMetadata.None; + lock (_metadata) + { + if (!_metadata.TryGetValue(member, out var metadata)) + _metadata[member] = metadata = new CultInspectorMetadata(member); + return metadata; + } + } + + public object Clone(object document, Type type) => _deserialize(type, _serialize(document, type, type)); + + public CultInspectorEdit BeginEdit(CultStoredDocument record) + { + if (record == null) throw new ArgumentNullException(nameof(record)); + return new CultInspectorEdit(record, Clone(record.Document, record.Descriptor.DocumentType)); + } + + // Whether CreateDefault makes a value: a string, struct, list or dictionary, or a concrete class with a parameterless constructor. + public bool CanCreate(Type type) => + type == typeof(string) || type.IsValueType || + ShapeOf(type).Kind is CultInspectorValueKind.List or CultInspectorValueKind.Dictionary || + !type.IsAbstract && !type.IsInterface && type.GetConstructor(Type.EmptyTypes) != null; + + public object? CreateDefault(Type type) + { + if (!CanCreate(type)) return null; + if (type == typeof(string)) return string.Empty; + var shape = ShapeOf(type); + if (type.IsArray) return Array.CreateInstance(shape.ElementType!, 0); + return Activator.CreateInstance(shape.Kind == CultInspectorValueKind.Dictionary ? shape.BuildType! : type); + } + + // What a new value for a slot of elementType (a list element, a dictionary value, a union pick) may be: a union's + // declared subtypes, or elementType itself. + public IReadOnlyList ElementChoices(Type elementType) + { + var shape = ShapeOf(elementType); + return shape.Kind == CultInspectorValueKind.Union ? shape.UnionChoices : new[] { elementType }; + } + + // A new value of choice for a slot of elementType, or null with a notice when choice cannot be made. + public object? CreateElement(Type elementType, Type choice, out string? notice) + { + // A slot's own type is always offered; an abstract one is refused with the notice below. + if (choice != elementType && !ElementChoices(elementType).Contains(choice)) + throw new ArgumentException($"{choice.Name} is not a choice for {elementType.Name}.", nameof(choice)); + var created = CreateDefault(choice); + // Null is a value only for a Nullable slot; anywhere else it is nothing made. + notice = created == null && !choice.IsValueType + ? $"{choice.Name} is abstract or has no parameterless constructor; nothing was created." + : null; + return created; + } + + // An integer edit as a value of integerType, clamped to that type's range instead of overflowing. + public object NarrowInteger(Type integerType, long value) + { + if (!IntegerTypes.Contains(integerType)) + throw new ArgumentException($"{integerType.Name} is not an integer the inspector edits.", nameof(integerType)); + long Bound(string name) => Convert.ToInt64(integerType.GetField(name)!.GetValue(null)); + return Convert.ChangeType(Math.Max(Bound("MinValue"), Math.Min(Bound("MaxValue"), value)), integerType); + } + + // A new collection holding items; the value it replaces is never mutated. + public object BuildList(Type listType, IReadOnlyList items) + { + var shape = ShapeOf(listType); + if (shape.Kind != CultInspectorValueKind.List) + throw new ArgumentException($"{listType.Name} is not a list the inspector builds.", nameof(listType)); + if (listType.IsArray) + { + var array = Array.CreateInstance(shape.ElementType!, items.Count); + for (var i = 0; i < items.Count; i++) array.SetValue(items[i], i); + return array; + } + + var list = (IList)Activator.CreateInstance(listType)!; + foreach (var item in items) list.Add(item); + return list; + } + + public object BuildDictionary(Type dictionaryType, IEnumerable> entries) + { + var dictionary = (IDictionary)Activator.CreateInstance(DictionaryShape(dictionaryType).BuildType!)!; + foreach (var entry in entries) dictionary.Add(entry.Key!, entry.Value); + return dictionary; + } + + // The key entry `index` of a dictionary in a documentType document holding `keys` takes when a lowering offers + // `candidate`: the candidate, or the kept key and a notice saying why. An unchanged key is never refused. Any other + // key must be non-null, not an empty record reference, and not taken by another entry, so BuildDictionary cannot + // throw. A key the document cannot serialize is refused: the store could not write it. + public object? ReplaceKey(Type documentType, Type dictionaryType, IReadOnlyList keys, int index, object? candidate, out string? notice) + { + var shape = DictionaryShape(dictionaryType); + string? refusal; + try + { + var identity = KeyIdentity(documentType, shape, candidate); + refusal = identity == KeyIdentity(documentType, shape, keys[index]) ? null + : identity == null ? "a null key" + : identity == RecordRefIdentity ? "an empty record reference as a key" + : Taken(documentType, shape, candidate!, keys.Where((_, i) => i != index)) ? "a duplicate key" + : null; + } + catch (Exception exception) + { + refusal = $"a key {documentType.Name} cannot serialize ({exception.GetBaseException().Message})"; + } + + notice = refusal == null ? null : $"Refused {refusal} on entry {index}; its key was kept."; + return refusal == null ? candidate : keys[index]; + } + + // A key to add beside `keys` in a documentType document: a record-reference key takes the first candidate record not + // already used; any other key is the type's default. Null with a notice when no such key is free or serializable. + public object? FreshKey(Type documentType, Type dictionaryType, IReadOnlyList keys, IEnumerable records, out string? notice) + { + var shape = DictionaryShape(dictionaryType); + var keyType = shape.KeyType!; + var candidates = ShapeOf(keyType).Kind == CultInspectorValueKind.RecordRef + ? RecordCandidates(keyType, records).Select(record => CreateRecordRef(keyType, record.Key.Value)) + : new[] { CreateDefault(keyType) }; + try + { + var fresh = candidates.FirstOrDefault(key => key != null && !Taken(documentType, shape, key, keys)); + notice = fresh == null ? $"No unused {keyType.Name} key is available; nothing was added." : null; + return fresh; + } + catch (Exception exception) + { + notice = $"{documentType.Name} cannot serialize a {keyType.Name} key ({exception.GetBaseException().Message}); nothing was added."; + return null; + } + } + + // A key is taken when another serializes the same (a CultRecordRef by its key string, null and "" alike) or when + // the rebuilt dictionary type's default comparer calls them equal (0.0 and -0.0 serialize apart but are one double key). + private bool Taken(Type documentType, CultInspectorShape dictionary, object key, IEnumerable keys) + { + var identity = KeyIdentity(documentType, dictionary, key); + var probe = (IDictionary)Activator.CreateInstance(dictionary.BuildType!)!; + var filler = dictionary.ValueType!.IsValueType ? Activator.CreateInstance(dictionary.ValueType) : null; + foreach (var other in keys.Where(other => other != null)) + { + if (KeyIdentity(documentType, dictionary, other) == identity) return true; + probe[other!] = filler; + } + + return probe.Contains(key); + } + + // A key's bytes as its declared key type under the owning document's serializer, the store's own path. Throws when + // that serializer cannot write the key. + private string? KeyIdentity(Type documentType, CultInspectorShape dictionary, object? key) => + key == null ? null + : key is ICultRecordRef reference ? RecordRefIdentity + reference.Key.Value + : "value:" + Convert.ToBase64String(_serialize(key, dictionary.KeyType!, documentType)); + + private CultInspectorShape DictionaryShape(Type dictionaryType) + { + var shape = ShapeOf(dictionaryType); + return shape.Kind == CultInspectorValueKind.Dictionary + ? shape + : throw new ArgumentException($"{dictionaryType.Name} is not a dictionary the inspector builds.", nameof(dictionaryType)); + } + + // The records a CultRecordRef may point at: every record whose document is a T, by label. + public IReadOnlyList RecordCandidates(Type recordRefType, IEnumerable records) + { + var target = ShapeOf(recordRefType).RecordTarget + ?? throw new ArgumentException($"{recordRefType.Name} is not a CultRecordRef.", nameof(recordRefType)); + return records + .Where(record => target.IsInstanceOfType(record.Document) && record.Key.Value.Length > 0) + .OrderBy(RecordLabel, StringComparer.OrdinalIgnoreCase) + .ToArray(); + } + + public object CreateRecordRef(Type recordRefType, string key) + { + if (ShapeOf(recordRefType).Kind != CultInspectorValueKind.RecordRef) + throw new ArgumentException($"{recordRefType.Name} is not a CultRecordRef.", nameof(recordRefType)); + return Activator.CreateInstance(recordRefType, new CultRecordKey(key))!; + } + + private CultInspectorShape Classify(Type type) + { + if (type == typeof(string)) return new CultInspectorShape(type, CultInspectorValueKind.String); + if (type == typeof(bool)) return new CultInspectorShape(type, CultInspectorValueKind.Bool); + if (type.IsEnum) return new CultInspectorShape(type, CultInspectorValueKind.Enum); + if (IntegerTypes.Contains(type)) return new CultInspectorShape(type, CultInspectorValueKind.Integer); + if (type == typeof(float) || type == typeof(double)) return new CultInspectorShape(type, CultInspectorValueKind.Float); + if (type.IsArray && type.GetArrayRank() != 1) + return Unsupported(type, $"{type.Name} is a multi-dimensional array, which the inspector does not edit."); + if (type.IsGenericType && type.GetGenericTypeDefinition() == typeof(CultRecordRef<>)) + return new CultInspectorShape(type, CultInspectorValueKind.RecordRef) { RecordTarget = type.GetGenericArguments()[0] }; + + var dictionary = type.GetInterfaces().Prepend(type) + .FirstOrDefault(candidate => candidate.IsGenericType && candidate.GetGenericTypeDefinition() == typeof(IDictionary<,>)); + if (dictionary != null) + { + var arguments = dictionary.GetGenericArguments(); + var build = type.IsInterface || type.IsAbstract ? typeof(Dictionary<,>).MakeGenericType(arguments) : type; + return typeof(IDictionary).IsAssignableFrom(build) && build.GetConstructor(Type.EmptyTypes) != null + ? new CultInspectorShape(type, CultInspectorValueKind.Dictionary) { KeyType = arguments[0], ValueType = arguments[1], BuildType = build } + : Unsupported(type, $"{type.Name} is a dictionary the inspector cannot rebuild."); + } + + if (type.IsArray) + return new CultInspectorShape(type, CultInspectorValueKind.List) { ElementType = type.GetElementType() }; + if (type.IsGenericType && !type.IsAbstract && !type.IsInterface && typeof(IList).IsAssignableFrom(type) && type.GetConstructor(Type.EmptyTypes) != null) + return new CultInspectorShape(type, CultInspectorValueKind.List) { ElementType = type.GetGenericArguments()[0] }; + + if (type.IsAbstract || type.IsInterface) + { + var choices = type.GetCustomAttributes(false).OrderBy(union => union.Key).Select(union => union.SubType).ToArray(); + return choices.Length == 0 + ? Unsupported(type, $"{type.Name} declares no [Union] subtypes.") + : new CultInspectorShape(type, CultInspectorValueKind.Union) { UnionChoices = choices }; + } + + var keyed = KeyedMembers(type); + if (keyed.Length > 0) + return new CultInspectorShape(type, CultInspectorValueKind.Nested) { Members = keyed }; + // An unkeyed struct is edited in place. With public fields it shows only them, readonly ones read-only: its + // properties alias those values (a Quaternion's eulerAngles, a Rect's min and max). Without, it shows its public + // properties with a setter, init-only ones read-only (record structs, matrices over private rows). Get-only + // properties are computed or kept by the in-place edit, so they are never rows. + if (type.IsValueType) + { + var fields = type.GetFields(Public); + var members = (fields.Length > 0 + ? fields.OrderBy(field => field.MetadataToken) + .Select(field => (Member: (MemberInfo)field, Type: field.FieldType, Assignable: !field.IsInitOnly)) + : type.GetProperties(Public).OrderBy(property => property.MetadataToken) + .Where(property => property.GetIndexParameters().Length == 0 && property.GetGetMethod() != null && property.GetSetMethod() != null) + .Select(property => (Member: (MemberInfo)property, Type: property.PropertyType, Assignable: !IsInitOnly(property.GetSetMethod()!)))) + .Select((member, slot) => new CultInspectorMember(member.Member, member.Type, slot, member.Assignable, MetadataOf(member.Member))) + .ToArray(); + if (members.Length > 0) + return new CultInspectorShape(type, CultInspectorValueKind.Nested) { Members = members }; + } + + return Unsupported(type, "no drawer for " + type.FullName); + } + + // Documents take their members and slots from the registry's catalog; other MessagePack objects from their [Key]s. + private CultInspectorMember[] KeyedMembers(Type type) + { + var allowPrivate = type.GetCustomAttribute(true)?.AllowPrivate == true; + var candidates = type.GetMembers(Public) + .Where(member => member is FieldInfo field && !field.IsLiteral || + member is PropertyInfo property && property.GetGetMethod() != null && property.GetIndexParameters().Length == 0) + .Where(member => !member.IsDefined(typeof(IgnoreMemberAttribute), true)) + // A member hidden with `new` is its most-derived declaration, as the registry's own-member scan reads it. + .GroupBy(member => member.Name) + .Select(hides => hides.OrderByDescending(member => Ancestry(member.DeclaringType)).First()); + IEnumerable<(MemberInfo Member, int Slot)> slotted; + if (type.IsDefined(typeof(CultDocumentAttribute), false)) + { + var slots = Registry.GetRequired(type).ToCatalogEntry().Members.ToDictionary(member => member.MemberName, member => member.Slot, StringComparer.Ordinal); + slotted = candidates.Where(member => slots.ContainsKey(member.Name)).Select(member => (member, slots[member.Name])); + } + else + { + slotted = candidates + .Select(member => (Member: member, Slot: member.GetCustomAttribute(true)?.IntKey)) + .Where(entry => entry.Slot != null) + .Select(entry => (entry.Member, entry.Slot!.Value)); + } + + return slotted + .Select(entry => new CultInspectorMember(entry.Member, TypeOf(entry.Member), entry.Slot, Assignable(entry.Member, allowPrivate), MetadataOf(entry.Member))) + .OrderBy(member => member.Metadata.Order ?? member.Slot) + .ThenBy(member => member.Slot) + .ToArray(); + } + + private static CultInspectorShape Unsupported(Type type, string reason) => + new CultInspectorShape(type, CultInspectorValueKind.Unsupported) { Reason = reason }; + + private static int Ancestry(Type? type) => type == null ? 0 : 1 + Ancestry(type.BaseType); + + // `init` is a setter whose return carries the IsExternalInit modreq; the language contract, matched by name because + // netstandard2.1 does not ship the type. A runtime that does not report the modreq (unverified on Unity's Mono) shows + // init members editable instead of read-only; SetValue still assigns them, so their edits round-trip either way. + private static bool IsInitOnly(MethodInfo setter) => + setter.ReturnParameter.GetRequiredCustomModifiers().Any(modifier => modifier.FullName == "System.Runtime.CompilerServices.IsExternalInit"); + + private static Type TypeOf(MemberInfo member) => member is FieldInfo field ? field.FieldType : ((PropertyInfo)member).PropertyType; + + private static bool Assignable(MemberInfo member, bool allowPrivate) + { + if (member is FieldInfo field) + return !field.IsInitOnly || allowPrivate; + var setter = ((PropertyInfo)member).GetSetMethod(true); + return setter != null && (setter.IsPublic || allowPrivate); + } + } +} diff --git a/src/GameCult.Caching/CultManagedDocument.cs b/src/GameCult.Caching/CultManagedDocument.cs index 1919ea88..00a6b2c2 100644 --- a/src/GameCult.Caching/CultManagedDocument.cs +++ b/src/GameCult.Caching/CultManagedDocument.cs @@ -7,421 +7,40 @@ using R3; namespace GameCult.Caching { - /// - /// Describes how a CultCache-managed document changed. - /// public enum CultCacheDocumentChangeKind { - /// - /// A document was added. - /// Added, - /// - /// A document was updated. - /// Updated, - /// - /// A document was removed. - /// Removed } - /// - /// One reactive CultCache document change. - /// public sealed class CultCacheDocumentChange where T : class { - /// - /// Creates a document change. - /// public CultCacheDocumentChange( CultCacheDocumentChangeKind kind, CultRecordKey key, T? document, - T? previousDocument) + T? previousDocument, + long sequence = 0) { Kind = kind; Key = key; Document = document; PreviousDocument = previousDocument; + Sequence = sequence; } - /// - /// Gets the change kind. - /// + // Per-cache, in memory, assigned at admission. A consumer keeping a latest value ignores a lower one. + public long Sequence { get; } + public CultCacheDocumentChangeKind Kind { get; } - /// - /// Gets the document record key. - /// public CultRecordKey Key { get; } - /// - /// Gets the current POCO presentation, when present. - /// public T? Document { get; } - /// - /// Gets the previous POCO presentation, when present. - /// public T? PreviousDocument { get; } } - - /// - /// Reactive POCO presentation for one CultCache-managed document. - /// - public sealed class CultManagedDocument where T : class - { - private readonly Func _read; - private readonly Func _commit; - private readonly Observable _watch; - - /// - /// Creates a managed document presentation. - /// - public CultManagedDocument( - CultRecordKey key, - Func read, - Func commit, - Observable watch) - { - Key = key; - _read = read ?? throw new ArgumentNullException(nameof(read)); - _commit = commit ?? throw new ArgumentNullException(nameof(commit)); - _watch = watch ?? throw new ArgumentNullException(nameof(watch)); - } - - /// - /// Gets the document record key. - /// - public CultRecordKey Key { get; } - - /// - /// Gets the current POCO presentation. - /// - public T? Value => _read(); - - /// - /// Watches committed values for this document. - /// - public Observable Watch() - { - return _watch; - } - - /// - /// Commits the current POCO value through the owner-supplied commit path. - /// - public Task CommitAsync() - { - var value = Value ?? throw new InvalidOperationException($"Document '{Key.Value}' is not present."); - return _commit(value); - } - - /// - /// Replaces the current value through the owner-supplied commit path. - /// - public Task ReplaceAsync(T value) - { - if (value == null) throw new ArgumentNullException(nameof(value)); - return _commit(value); - } - } - - /// - /// CPU-local structure-of-arrays table owned by CultCache for one document type. - /// - public sealed class CultSoaTable where TDocument : class - { - private readonly Dictionary _columns; - - internal CultSoaTable( - CultRecordKey[] keys, - TDocument[] documents, - Dictionary columns) - { - Keys = keys; - Documents = documents; - _columns = columns; - } - - /// - /// Gets record keys in row order. - /// - public IReadOnlyList Keys { get; } - - /// - /// Gets POCO presentations in row order. - /// - public IReadOnlyList Documents { get; } - - /// - /// Gets the row count. - /// - public int Count => Keys.Count; - - /// - /// Gets a contiguous unmanaged field/property column. - /// - public CultSoaColumn Column(string name) where TValue : unmanaged - { - if (!_columns.TryGetValue(name, out var column)) - { - throw new KeyNotFoundException($"SoA table for {typeof(TDocument).Name} has no '{name}' column."); - } - - if (column is not TValue[] values) - { - throw new InvalidOperationException( - $"SoA table column '{name}' stores {column.GetType().GetElementType()?.FullName}, not {typeof(TValue).FullName}."); - } - - return new CultSoaColumn(name, values); - } - } - - /// - /// Contiguous typed SoA column. - /// - public readonly struct CultSoaColumn where TValue : unmanaged - { - internal CultSoaColumn(string name, TValue[] values) - { - Name = name; - Values = values; - } - - /// - /// Gets the column name. - /// - public string Name { get; } - - /// - /// Gets the backing array. - /// - public TValue[] Values { get; } - - /// - /// Gets the backing array as a span. - /// - public Span Span => Values.AsSpan(); - } - - internal sealed class CultCacheSoaStore - { - private readonly Dictionary _tables = new(); - - public void Upsert(CultStoredDocument stored) - { - Table(stored.Descriptor.DocumentType).Upsert(stored.Key, stored.Document); - } - - public void Remove(CultStoredDocument stored) - { - if (_tables.TryGetValue(stored.Descriptor.DocumentType, out var table)) - { - table.Remove(stored.Key); - } - } - - public CultSoaTable Snapshot() where TDocument : class - { - return Table(typeof(TDocument)).Snapshot(); - } - - private CultCacheSoaTypeTable Table(Type documentType) - { - if (!_tables.TryGetValue(documentType, out var table)) - { - table = new CultCacheSoaTypeTable(documentType); - _tables[documentType] = table; - } - - return table; - } - } - - internal sealed class CultCacheSoaTypeTable - { - private readonly Type _documentType; - private readonly List _keys = new(); - private readonly List _documents = new(); - private readonly Dictionary _members; - private readonly Dictionary _columns; - private readonly Dictionary _rowsByKey = new(StringComparer.Ordinal); - - public CultCacheSoaTypeTable(Type documentType) - { - _documentType = documentType; - _members = CultCacheSoaMember.Discover(documentType) - .ToDictionary(member => member.Name, StringComparer.Ordinal); - _columns = _members.Values.ToDictionary( - member => member.Name, - member => Array.CreateInstance(member.ValueType, 0) as object, - StringComparer.Ordinal); - } - - public void Upsert(CultRecordKey key, object document) - { - if (_rowsByKey.TryGetValue(key.Value, out var row)) - { - _documents[row] = document; - } - else - { - row = _keys.Count; - _rowsByKey[key.Value] = row; - _keys.Add(key); - _documents.Add(document); - EnsureColumnCapacity(_keys.Count); - } - - WriteRow(row, document); - } - - public void Remove(CultRecordKey key) - { - if (!_rowsByKey.TryGetValue(key.Value, out var row)) - { - return; - } - - var lastRow = _keys.Count - 1; - if (row != lastRow) - { - _keys[row] = _keys[lastRow]; - _documents[row] = _documents[lastRow]; - _rowsByKey[_keys[row].Value] = row; - CopyRow(lastRow, row); - } - - _rowsByKey.Remove(key.Value); - _keys.RemoveAt(lastRow); - _documents.RemoveAt(lastRow); - } - - public CultSoaTable Snapshot() where TDocument : class - { - if (!typeof(TDocument).IsAssignableFrom(_documentType)) - { - throw new InvalidOperationException($"{_documentType.FullName} is not assignable to {typeof(TDocument).FullName}."); - } - - var count = _keys.Count; - var columns = new Dictionary(StringComparer.Ordinal); - foreach (var pair in _columns) - { - var member = _members[pair.Key]; - var source = (Array)pair.Value; - var target = Array.CreateInstance(member.ValueType, count); - Array.Copy(source, target, count); - columns[pair.Key] = target; - } - - return new CultSoaTable( - _keys.ToArray(), - _documents.Cast().ToArray(), - columns); - } - - private void EnsureColumnCapacity(int count) - { - foreach (var member in _members.Values) - { - var column = (Array)_columns[member.Name]; - if (column.Length >= count) - { - continue; - } - - var nextLength = Math.Max(count, Math.Max(4, column.Length * 2)); - var next = Array.CreateInstance(member.ValueType, nextLength); - Array.Copy(column, next, column.Length); - _columns[member.Name] = next; - } - } - - private void WriteRow(int row, object document) - { - foreach (var member in _members.Values) - { - ((Array)_columns[member.Name]).SetValue(member.Get(document), row); - } - } - - private void CopyRow(int sourceRow, int targetRow) - { - foreach (var member in _members.Values) - { - var column = (Array)_columns[member.Name]; - column.SetValue(column.GetValue(sourceRow), targetRow); - } - } - } - - internal sealed class CultCacheSoaMember - { - private CultCacheSoaMember(string name, Type valueType, Func getter) - { - Name = name; - ValueType = valueType; - Get = getter; - } - - public string Name { get; } - public Type ValueType { get; } - public Func Get { get; } - - public static IEnumerable Discover(Type documentType) - { - const BindingFlags flags = BindingFlags.Instance | BindingFlags.Public; - foreach (var field in documentType.GetFields(flags).OrderBy(field => field.MetadataToken)) - { - if (IsColumnType(field.FieldType)) - { - yield return new CultCacheSoaMember(field.Name, field.FieldType, field.GetValue); - } - } - - foreach (var property in documentType.GetProperties(flags).OrderBy(property => property.MetadataToken)) - { - if (property.GetMethod != null && - property.GetIndexParameters().Length == 0 && - IsColumnType(property.PropertyType)) - { - yield return new CultCacheSoaMember(property.Name, property.PropertyType, property.GetValue); - } - } - } - - private static bool IsColumnType(Type type) - { - return type.IsValueType && - !type.IsByRefLike && - !ContainsManagedReferences(type, new HashSet()); - } - - private static bool ContainsManagedReferences(Type type, HashSet visited) - { - if (type.IsPointer) - { - return false; - } - - if (!type.IsValueType) - { - return true; - } - - if (type.IsPrimitive || type.IsEnum || !visited.Add(type)) - { - return false; - } - - const BindingFlags flags = BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic; - return type.GetFields(flags).Any(field => ContainsManagedReferences(field.FieldType, visited)); - } - } } diff --git a/src/GameCult.Caching/CultSchemaTypeNames.cs b/src/GameCult.Caching/CultSchemaTypeNames.cs new file mode 100644 index 00000000..ada0d594 --- /dev/null +++ b/src/GameCult.Caching/CultSchemaTypeNames.cs @@ -0,0 +1,57 @@ +using System; +using System.Linq; +using System.Text; + +namespace GameCult.Caching +{ + internal static class CultSchemaTypeNames + { + public static string FromType(Type type) + { + if (type.IsArray) + { + return FromType(type.GetElementType()!) + "[]"; + } + + if (type.IsGenericType) + { + var definition = type.GetGenericTypeDefinition(); + var name = definition.FullName ?? definition.Name; + var tickIndex = name.IndexOf('`'); + if (tickIndex >= 0) + { + name = name[..tickIndex]; + } + + var arguments = string.Join(", ", type.GetGenericArguments().Select(FromType)); + return $"{name}<{arguments}>"; + } + + return type.FullName ?? type.Name; + } + + public static string EscapeForLiteral(string value) + { + if (string.IsNullOrEmpty(value)) + { + return value; + } + + var builder = new StringBuilder(value.Length); + foreach (var character in value) + { + builder.Append(character switch + { + '\\' => "\\\\", + '"' => "\\\"", + '\r' => "\\r", + '\n' => "\\n", + '\t' => "\\t", + _ => character.ToString() + }); + } + + return builder.ToString(); + } + } +} diff --git a/src/GameCult.Caching/GameCult.Caching.csproj b/src/GameCult.Caching/GameCult.Caching.csproj index 194872bc..a34cdfd3 100644 --- a/src/GameCult.Caching/GameCult.Caching.csproj +++ b/src/GameCult.Caching/GameCult.Caching.csproj @@ -1,10 +1,11 @@ CultCache: Concurrent polymorphic cache with modular backing stores for .NET + $(NoWarn);CS1591 - + diff --git a/src/GameCult.Caching/GameCult.Caching.txt b/src/GameCult.Caching/GameCult.Caching.txt deleted file mode 100644 index ab959478..00000000 --- a/src/GameCult.Caching/GameCult.Caching.txt +++ /dev/null @@ -1,1213 +0,0 @@ - - -// E:\Projects\CultLib\src\GameCult.Caching\CultCache.cs - -using System; -using System.Collections.Concurrent; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Reflection; -using System.Threading.Tasks; -using ConcurrentCollections; -using GameCult.Logging; -using MessagePack; -using R3; - -namespace GameCult.Caching -{ - /// - /// Specifies that the decorated class contains global settings. - /// Use this attribute to mark classes intended to hold application-wide configuration or settings. - /// - [AttributeUsage(AttributeTargets.Class)] - public class GlobalSettingsAttribute : Attribute { } - - /// - /// Represents an in-memory and persistent cache for objects, supporting global entries, - /// named entries, generic field indexes, and multiple backing stores. Provides thread-safe operations for adding, - /// retrieving, indexing, and removing entries, with support for event notifications and logging. - /// - /// - /// - /// Features: - /// - /// Supports multiple s for persistence and synchronization. - /// Handles global settings entries via . - /// Provides fast lookup by ID, name (for ), and indexed fields. - /// Allows registering generic indexes for any public field of a type. - /// Thread-safe via and . - /// Notifies on entry updates via event. - /// Supports both synchronous and asynchronous add operations. - /// Integrates with for error and event logging. - /// - /// - /// - /// Usage: - /// - /// var cache = new CultCache(); - /// cache.RegisterIndex<PlayerData>("Email"); - /// cache.AddAsync(new PlayerData { ... }).GetAwaiter().GetResult(); - /// var player = cache.GetByIndex<PlayerData>("Email", "user@example.com"); - /// - /// - /// - public class CultCache : IDisposable - { - private readonly object addLock = new object(); // Retained for synchronous Add compatibility - - /// - /// Occurs when a is updated. - /// The event provides the old and new instances. - /// - /// - /// The first parameter is the previous entry, and the second parameter is the updated entry. - /// The first parameter will be null if the entry is newly added, - /// and the second parameter will be null if the entry is deleted. - /// - public event Action? OnUpdate; - - private readonly List _backingStores = new(); - private readonly ConcurrentDictionary _typeStores = new(); - private readonly ConcurrentDictionary _storeTypes = new(); - private readonly ConcurrentDictionary _entries = new(); - private readonly ConcurrentDictionary _globals = new(); - private readonly ConcurrentDictionary> _types = new(); - private readonly ConcurrentDictionary> _nameToIdMap = new(); - // Generic indexes: Map (Type, FieldName) -> (Value -> ID). Allows registering indexes for any field - private readonly ConcurrentDictionary<(Type Type, string FieldName), ConcurrentDictionary> _indexes = new(); - // Cache getters per type (list of (field, getter) for that type only). Avoids scanning all indexes. - private readonly ConcurrentDictionary Getter)>> _typeToGetters = new(); - private ILogger _logger = new NullLogger(); - - /// - /// Gets or sets the instance used for logging within the cache. - /// If set to null, a will be used as a fallback. - /// - public ILogger Logger - { - get => _logger; - set => _logger = value ?? new NullLogger(); - } - - /// - /// Gets all objects currently stored in the cache. - /// - public IEnumerable AllEntries => _entries.Values; - - /// - /// Initializes a new instance of the class. - /// Sets up the cache by registering all child classes of . - /// For each child type: - /// - /// Adds a concurrent hash set for caching entries of that type. - /// If the type is marked with , adds it to the global settings and creates an instance. - /// If the type implements , initializes a name-to-ID mapping dictionary. - /// - /// Also sets the static cache reference in . - /// - public CultCache() - { - DatabaseLinkBase.Cache = this; - - foreach (var type in typeof(DatabaseEntry).GetAllChildClasses()) - { - _types.TryAdd(type, new ConcurrentHashSet()); - if (type.GetCustomAttribute() != null) - { - _globals.TryAdd(type, null); - try - { - var global = (DatabaseEntry)Activator.CreateInstance(type)!; - AddAsync(global).GetAwaiter().GetResult(); - _globals[type] = global; // Update after successful add - } - catch (Exception ex) - { - Logger.LogError($"Failed to initialize global {type.Name}: {ex.Message}"); - _globals[type] = null; - } - } - // Initialize name-to-ID map for INamedEntry types - if (typeof(INamedEntry).IsAssignableFrom(type)) - { - _nameToIdMap.TryAdd(type, new ConcurrentDictionary()); - } - } - } - - /// - /// Adds a backing store to the cache system and configures domain type associations and event subscriptions. - /// - /// The instance to add. - /// - /// Optional array of objects representing domain types associated with the backing store. - /// If provided, the store is mapped to these types; otherwise, the store is added to the general backing store list and subscribes to existing stores. - /// - /// - /// - /// If no domain types are specified, the store subscribes to all existing backing stores to receive their updates, - /// ensuring synchronization across stores, with the first store added acting as the primary source of truth. - /// - /// - /// Specifying domain types allows for targeted storage and retrieval of specific types, - /// for example, using a database store for user data and a file-based store for configuration data or game assets. - /// - /// - public void AddBackingStore(CacheBackingStore store, params Type[] domain) - { - if (domain.Length > 0) - { - _storeTypes.TryAdd(store, domain); - foreach (var t in domain) _typeStores.TryAdd(t, store); - } - else - { - foreach (var existingStore in _backingStores) store.SubscribeTo(existingStore); - _backingStores.Add(store); - } - store.EntryAdded.Subscribe(async entry => - { - try // Try-catch logs without swallowing. Added to EntryDeleted for consistency (even though sync) - { - await AddAsync(entry, store); - OnUpdate?.Invoke(null, entry); - } - catch (Exception ex) - { - Logger.LogError($"EntryAdded processing failed for {entry?.ID}: {ex.Message}"); - } - }); - store.EntryUpdated.Subscribe(async entry => - { - try - { - await AddAsync(entry, store); - OnUpdate?.Invoke(entry, entry); - } - catch (Exception ex) - { - Logger.LogError($"EntryUpdated processing failed for {entry?.ID}: {ex.Message}"); - } - }); - store.EntryDeleted.Subscribe(entry => - { - try - { - Remove(entry, store); - OnUpdate?.Invoke(entry, null); - } - catch (Exception ex) - { - Logger.LogError($"EntryDeleted processing failed for {entry?.ID}: {ex.Message}"); - } - }); - store.Logger = Logger; - } - - /// - /// Asynchronously pulls all data from the backing stores. - /// Initiates parallel tasks to execute the PullAll method on each backing store and store type, - /// and waits for all tasks to complete. - /// - /// A task that represents the asynchronous operation. - public async Task PullAllBackingStoresAsync() - { - var tasks = _backingStores.Select(store => Task.Run(store.PullAll)) - .Concat(_storeTypes.Keys.Select(store => Task.Run(store.PullAll))) - .ToArray(); - await Task.WhenAll(tasks); - } - - /// - /// Adds a to the cache asynchronously, updating all relevant maps and indexes. - /// If the entry already exists, cleans up old mappings before updating. - /// Also persists the entry to the backing store if available. - /// - /// The to add or update in the cache. - /// - /// The source from which the entry originated, used internally to avoid redundant writes. - /// Optional; defaults to null. - /// - /// A representing the asynchronous operation. - public async Task AddAsync(DatabaseEntry entry, CacheBackingStore? source = null) - { - if (entry == null) return; - - var type = entry.GetType(); - - // Pre-fetch existing to detect updates (for map/index cleanup) - if (!_entries.TryGetValue(entry.ID, out var existing)) existing = null; - - // Cleanup old maps/indexes BEFORE updating _entries - if (existing != null) - { - // For name map (if updating existing) - if (existing is INamedEntry existingNamed && _nameToIdMap.TryGetValue(type, out var nameMap)) - { - nameMap.TryRemove(existingNamed.EntryName, out _); - } - - // For generic indexes (if registered) - if (_typeToGetters.TryGetValue(type, out var getters)) - { - foreach (var (fieldName, getter) in getters) - { - var oldValue = getter(existing); - if (!string.IsNullOrEmpty(oldValue)) - { - var indexKey = (type, fieldName); - _indexes[indexKey].TryRemove(oldValue, out _); - } - } - } - - // For type sets - _types[existing.GetType()].TryRemove(existing); - foreach (var parentType in existing.GetType().GetParentTypes()) - { - if (_types.TryGetValue(parentType, out var typeSet)) - typeSet.TryRemove(existing); - } - } - - _entries[entry.ID] = entry; - _types[type].Add(entry); // Assumes exists; throws if not (edge case) - foreach (var parentType in type.GetParentTypes()) - { - if (_types.TryGetValue(parentType, out var typeSet)) - typeSet.Add(entry); - } - - if (_globals.ContainsKey(type)) - { - _globals.AddOrUpdate(type, entry, (_, old) => entry); - } - - // Add new values (always, for both new and updates) - // Update name-to-ID map if INamedEntry - if (entry is INamedEntry namedEntry && _nameToIdMap.TryGetValue(type, out var nameMap2)) - { - nameMap2[namedEntry.EntryName] = entry.ID; - } - - // Update generic indexes if registered for this type - if (_typeToGetters.TryGetValue(type, out var getters2)) - { - foreach (var (fieldName, getter) in getters2) - { - var value = getter(entry); - if (!string.IsNullOrEmpty(value)) - { - var indexKey = (type, fieldName); - _indexes[indexKey][value] = entry.ID; - } - } - } - - // Async file write - try - { - if (_typeStores.TryGetValue(type, out var store)) - await Task.Run(() => store.Push(entry)); - else if (_backingStores.Any() && _backingStores.First() != source) - await Task.Run(() => _backingStores.First().Push(entry)); - } - catch (Exception ex) - { - // Log error but don't revert cache update to ensure consistency - Logger.LogError($"Failed to persist entry {entry.ID}: {ex.Message}"); - } - } - - /// - /// Determines whether the specified is a global settings entry. - /// - /// The to check. - /// true if the entry is global; otherwise, false. - public bool IsGlobal(DatabaseEntry entry) => _globals.ContainsKey(entry.GetType()); - - /// - /// Adds all objects from the specified collection to the cache asynchronously. - /// - /// The collection of objects to add. - /// - /// The source from which the entries originated, used to avoid redundant writes. - /// Optional; defaults to null. - /// - /// A representing the asynchronous operation. - public Task AddAllAsync(IEnumerable entries, CacheBackingStore? source = null) - { - var tasks = entries.Select(entry => AddAsync(entry, source)).ToArray(); - return Task.WhenAll(tasks); - } - - /// - /// Retrieves a from the cache by its unique identifier. - /// - /// The unique identifier of the entry to retrieve. - /// The associated with the specified , or null if not found. - public DatabaseEntry Get(Guid guid) - { - _entries.TryGetValue(guid, out var entry); - return entry; - } - - /// - /// Retrieves a of type from the cache by its unique identifier. - /// - /// The type of to retrieve. - /// The unique identifier of the entry to retrieve. - /// The entry of type associated with the specified , or null if not found or not of type . - public T? Get(Guid guid) where T : DatabaseEntry - { - return Get(guid) as T; - } - - /// - /// Retrieves the global instance for the specified type, if it exists. - /// - /// The type of the global entry to retrieve. - /// The global instance associated with the specified type, or null if not found. - public DatabaseEntry? GetGlobal(Type type) - { - _globals.TryGetValue(type, out var entry); - return entry; - } - - /// - /// Retrieves the global instance of type , if it exists. - /// - /// The type of the global entry to retrieve. - /// The global entry of type , or null if not found. - public T? GetGlobal() where T : DatabaseEntry - { - return GetGlobal(typeof(T)) as T; - } - - /// - /// Retrieves all objects of the specified type from the cache. - /// - /// The type of to retrieve. - /// An enumerable collection of objects of the specified type. - public IEnumerable GetAll(Type type) - { - return _types.TryGetValue(type, out var entries) ? entries : Enumerable.Empty(); - } - - /// - /// Retrieves all objects of type from the cache. - /// - /// The type of to retrieve. - /// An enumerable collection of objects from the cache. - public IEnumerable GetAll() where T : DatabaseEntry - { - return _types.TryGetValue(typeof(T), out var entries) ? entries.Cast() : Enumerable.Empty(); - } - - /// - /// Retrieves the unique identifier () of an entry of type by its name. - /// - /// The type of that implements . - /// The name of the entry to look up. - /// The of the entry if found; otherwise, null. - public Guid? GetIdByName(string name) where T : DatabaseEntry, INamedEntry - { - var type = typeof(T); - if (_nameToIdMap.TryGetValue(type, out var nameMap) && nameMap.TryGetValue(name, out var id)) - return id; - return null; - } - - /// - /// Retrieves an entry of type by its name from the cache. - /// - /// The type of that implements . - /// The name of the entry to retrieve. - /// The entry of type if found; otherwise, null. - public T? GetByName(string name) where T : DatabaseEntry, INamedEntry - { - return GetIdByName(name) is { } id ? Get(id) : null; - } - - /// - /// Registers an index for a public member (field or property) of the specified type. - /// This enables fast lookup of entries by the value of the indexed member. - /// Call this method once per member to be indexed. - /// - /// The type of for which to register the index. - /// The name of the public member to index. - /// Thrown if the specified member does not exist on the type. - public void RegisterIndex(string memberName) where T : DatabaseEntry - { - var type = typeof(T); - var key = (type, memberName); - if (_indexes.ContainsKey(key)) - return; // Already registered; ignore - - _indexes[key] = new ConcurrentDictionary(); - - // Cache a getter func for perf. - if (type.GetField(memberName) is { } field) - { - var getter = (Func)(entry => field.GetValue(entry)?.ToString() ?? string.Empty); - _typeToGetters.AddOrUpdate(type, - _ => new List<(string, Func)> { (memberName, getter) }, - (_, list) => { list.Add((memberName, getter)); return list; }); - } - else if (type.GetProperty(memberName) is { } property) - { - var getter = (Func)(entry => property.GetValue(entry)?.ToString() ?? string.Empty); - _typeToGetters.AddOrUpdate(type, - _ => new List<(string, Func)> { (memberName, getter) }, - (_, list) => { list.Add((memberName, getter)); return list; }); - } - else - { - throw new ArgumentException($"No public field or property '{memberName}' on {type.Name}"); - } - } - - /// - /// Retrieves the unique identifier () of an entry of type by the value of an indexed field. - /// - /// The type of for which the index is registered. - /// The name of the indexed field. - /// The value of the indexed field to look up. - /// The of the entry if found; otherwise, null. - public Guid? GetIdByIndex(string fieldName, string value) where T : DatabaseEntry - { - var key = (typeof(T), fieldName); - if (_indexes.TryGetValue(key, out var indexMap) && indexMap.TryGetValue(value, out var id)) - return id; - return null; - } - - /// - /// Retrieves an entry of type by the value of an indexed field from the cache. - /// - /// The type of for which the index is registered. - /// The name of the indexed field. - /// The value of the indexed field to look up. - /// The entry of type if found; otherwise, null. - public T? GetByIndex(string fieldName, string value) where T : DatabaseEntry - { - return GetIdByIndex(fieldName, value) is { } id ? Get(id) : null; - } - - /// - /// Removes the specified from the cache, including all associated indexes, type stores, - /// global references, parent type sets, and name-to-ID mappings. - /// - /// The to remove from the cache. - /// - /// The to exclude from deletion. Used internally to avoid redundant deletions when the entry originates from a specific store. - /// - public void Remove(DatabaseEntry entry, CacheBackingStore? source = null) - { - var type = entry.GetType(); - if (!_entries.TryRemove(entry.ID, out _)) return; - - // Clean up type sets - _types[type].TryRemove(entry); - foreach (var parentType in type.GetParentTypes()) - { - if (_types.TryGetValue(parentType, out var typeSet)) - typeSet.TryRemove(entry); - } - - // Clean up global if applicable - if (_globals.TryGetValue(type, out var global) && global != null && global.Equals(entry)) _globals[type] = null!; - - // Propagate delete to backing stores (except source) - if (_typeStores.TryGetValue(type, out var typeStore)) - typeStore.Delete(entry); - else - { - foreach (var store in _backingStores) - { - if (store != source) - store.Delete(entry); - } - } - - // Clean up name-to-ID map if INamedEntry - if (entry is INamedEntry namedEntry && _nameToIdMap.TryGetValue(type, out var nameMap)) - { - nameMap.TryRemove(namedEntry.EntryName, out _); - } - - // Clean up generic indexes if registered for this type - if (_typeToGetters.TryGetValue(type, out var getters)) - { - foreach (var (fieldName, getter) in getters) - { - var value = getter(entry); - if (!string.IsNullOrEmpty(value)) - { - var indexKey = (type, fieldName); - _indexes[indexKey].TryRemove(value, out _); - } - } - } - } - - /// - public void Dispose() - { - foreach (var store in _backingStores) - if (store is IDisposable d) d.Dispose(); - foreach (var store in _storeTypes.Keys) - if (store is IDisposable d) d.Dispose(); - } - } - - /// - /// Represents a backing store that supports observing real-time changes. - /// - public interface IRealtimeBackingStore - { - /// - /// Begins observing real-time changes in the backing store and triggers appropriate events when changes occur. - /// - public void ObserveChanges(); - } - - /// - /// Represents an abstract backing store for caching objects. - /// Provides mechanisms for pulling, pushing, deleting, and synchronizing entries, as well as event notifications for entry changes. - /// - public abstract class CacheBackingStore : IDisposable - { - private ILogger _logger = new NullLogger(); - - /// - /// Gets or sets the instance used for logging within the cache. - /// If set to null, a will be used as a fallback. - /// - public ILogger Logger - { - get => _logger; - set => _logger = value ?? new NullLogger(); - } - - /// - /// Initializes a new instance of the class. - /// Sets up subjects for entry added, deleted, and updated events. - /// - protected CacheBackingStore() - { - EntryAdded = new Subject(); - EntryDeleted = new Subject(); - EntryUpdated = new Subject(); - } - - /// - /// Pulls all entries from the backing store into the cache. - /// Implementations should load all persisted objects and notify via . - /// - public abstract void PullAll(); - /// - /// Persists the specified to the backing store. - /// Implementations should save or update the entry in the underlying storage. - /// - /// The to persist. - public abstract void Push(DatabaseEntry entry); - /// - /// Deletes the specified from the backing store. - /// Implementations should remove the entry from the underlying storage. - /// - /// The to delete. - public abstract void Delete(DatabaseEntry entry); - /// - /// Persists all objects to the backing store. - /// Implementations should save or update all entries in the underlying storage. - /// - /// - /// If true, performs a soft push (implementation-specific); otherwise, performs a full push. - /// - public abstract void PushAll(bool soft = false); - - /// - /// Gets the subject that notifies when a is added to the backing store. - /// - public Subject EntryAdded { get; } - /// - /// Gets the subject that notifies when a is deleted from the backing store. - /// - public Subject EntryDeleted { get; } - /// - /// Gets the subject that notifies when a is updated in the backing store. - /// - public Subject EntryUpdated { get; } - - /// - /// Stores all objects managed by this backing store, keyed by their unique identifier. - /// - protected ConcurrentDictionary Entries = new(); - - /// - /// Subscribes this backing store to the entry events of another , - /// so that added, deleted, and updated entries in the target store are automatically pushed or deleted in this store. - /// - /// The target to subscribe to. - public void SubscribeTo(CacheBackingStore targetStore) - { - targetStore.EntryAdded.Subscribe(Push); - targetStore.EntryDeleted.Subscribe(Delete); - targetStore.EntryUpdated.Subscribe(Push); - } - - /// - public void Dispose() - { - EntryAdded?.Dispose(); - EntryDeleted?.Dispose(); - EntryUpdated?.Dispose(); - } - } - - /// - /// An abstract cache backing store that persists objects as individual files, - /// organizing them by entry type in subdirectories under a root directory. Supports real-time change observation - /// via and provides serialization/deserialization hooks for custom formats. - /// - /// - /// - /// Each entry type is mapped to its own directory, and entries are stored as files with a configurable extension. - /// The class maintains a mapping between file paths and entry IDs for efficient lookup and change tracking. - /// - /// - /// Real-time changes (creation, modification, deletion) to files are observed and propagated to upstream subscribers - /// using Rx subjects and debounced event handling. - /// - /// - /// Subclasses must implement , , and - /// to define the file format. - /// - /// - public abstract class MultiFileBackingStore : CacheBackingStore, IRealtimeBackingStore, IDisposable - { - /// - /// Gets the root directory where all entry files are stored. - /// - public DirectoryInfo RootDirectory { get; } - /// - /// Maps each type to its corresponding subdirectory. - /// - protected readonly Dictionary EntryTypeDirectories = new(); - /// - /// Maps file paths to their corresponding IDs for quick lookup during change detection. - /// - protected readonly ConcurrentDictionary _filePathToIdMap = new(); - private FileSystemWatcher? watcher; - private Subject? _changedSubject; - - /// - /// Serializes a into a byte array for file storage. - /// - /// The to serialize. - /// A byte array representing the serialized entry. - public abstract byte[] Serialize(DatabaseEntry entry); - /// - /// Deserializes a byte array into a object. - /// - /// The byte array containing the serialized entry. - /// The deserialized object. - public abstract DatabaseEntry Deserialize(byte[] data); - /// - /// Gets the file extension used for storing entries (e.g., ".json", ".msgpack"). - /// - public abstract string Extension { get; } - - /// - /// Initializes a new instance of the class. - /// Creates the root directory if it does not exist and sets up subdirectories for each child type. - /// - /// The path to the root directory for storing entry files. - public MultiFileBackingStore(string path) - { - RootDirectory = new DirectoryInfo(path); - foreach (var type in typeof(DatabaseEntry).GetAllChildClasses()) - { - EntryTypeDirectories[type] = GetDirectoryForType(type); - } - } - - private DirectoryInfo GetDirectoryForType(Type type) - { - var stack = new Stack(); - var t = type; - while (t != null && t != typeof(DatabaseEntry)) - { - stack.Push(t.Name); - t = t.BaseType; - } - stack.Push(RootDirectory.FullName); - // ToArray() copies in LIFO order (top/first: root, then parents, deepest child last) - // Ensures Path.Combine yields: Root\Parent\...\Child - return Directory.CreateDirectory(Path.Combine(stack.ToArray())); - } - - /// - /// Loads all objects from individual files in the root directory and its subdirectories, - /// organized by entry type. Each file is deserialized using , - /// added to the local dictionary, and notifies upstream via . - /// Skips files that cannot be deserialized, logging errors without failing the entire operation. - /// Only processes files matching the . - /// - public override void PullAll() - { - if (!RootDirectory.Exists) return; - - foreach (var directory in EntryTypeDirectories.Values) - { - foreach (var file in directory.EnumerateFiles($"*.{Extension}")) - { - try - { - var entry = Deserialize(File.ReadAllBytes(file.FullName)); - Entries[entry.ID] = entry; - EntryAdded.OnNext(entry); - _filePathToIdMap[file.FullName] = entry.ID; - } - catch (Exception ex) - { - Logger.LogError($"Failed to pull entry from {file.FullName}: {ex.Message}"); - } - } - } - } - - private string GetFileName(DatabaseEntry entry) - { - string name = entry is INamedEntry namedEntry ? namedEntry.EntryName : entry.ID.ToString(); - // Sanitize to prevent invalid chars/path traversal - foreach (char invalid in Path.GetInvalidFileNameChars()) - name = name.Replace(invalid, '_'); - return $"{name}.{Extension}"; - } - - /// - /// Persists the specified to a file in the appropriate type subdirectory. - /// The file name is generated using , which prefers the entry name if available. - /// The entry is serialized via and written to disk. - /// Updates the local dictionary and the for change tracking. - /// Logs errors if the write fails but re-throws to allow caller handling. - /// - /// The to persist to a file. - public override void Push(DatabaseEntry entry) - { - var type = entry.GetType(); - Entries[entry.ID] = entry; - var directory = EntryTypeDirectories[type]; - var filePath = Path.Combine(directory.FullName, GetFileName(entry)); - File.WriteAllBytes(filePath, Serialize(entry)); - _filePathToIdMap[filePath] = entry.ID; - } - - /// - /// Removes the specified from the local dictionary - /// and deletes its corresponding file in the type subdirectory. - /// Updates the and notifies upstream via . - /// File deletion failures are logged but do not prevent local removal or notification. - /// - /// The to delete from storage. - public override void Delete(DatabaseEntry entry) - { - if (Entries.ContainsKey(entry.ID)) - { - var type = entry.GetType(); - Entries.TryRemove(entry.ID, out _); - var directory = EntryTypeDirectories[type]; - var filePath = Path.Combine(directory.FullName, GetFileName(entry)); - _filePathToIdMap.TryRemove(filePath, out _); - var file = new FileInfo(filePath); - if (file.Exists) - file.Delete(); - } - } - - /// - /// Persists all entries in to their respective files by invoking for each. - /// The parameter is ignored in this implementation, as all pushes are full writes. - /// Errors during individual pushes are logged but do not halt the process. - /// Suitable for batch synchronization after bulk changes. - /// - /// If true, performs a soft push (not supported; treated as full push). - public override void PushAll(bool soft = false) - { - foreach (var entry in Entries.Values.ToArray()) Push(entry); - } - - /// - /// Starts monitoring the root directory and its subdirectories for file system changes related to entry files. - /// Initializes a with filters for creation, modification, and deletion events on files matching . - /// Uses debouncing for change events to handle bursty writes (100ms delay via and R3). - /// Disposes any existing watcher before creating a new one to prevent duplicates. - /// Logs a warning if the root directory does not exist and skips observation. - /// Logs info when observation starts successfully. - /// Call this method to enable real-time synchronization with external file modifications. - /// - public void ObserveChanges() - { - if (watcher != null) // Guard against re-creation - Dispose(); - - watcher = new FileSystemWatcher(RootDirectory.FullName, $"*.{Extension}") - { - NotifyFilter = NotifyFilters.LastWrite | NotifyFilters.FileName | NotifyFilters.DirectoryName, - EnableRaisingEvents = true, - IncludeSubdirectories = true - }; - - // Use R3 to throttle events if file writes are bursty - _changedSubject = new Subject(); // For paths - _changedSubject.Debounce(TimeSpan.FromMilliseconds(100)).Subscribe(HandleFileChange); - - watcher.Changed += (sender, args) => - { - _changedSubject.OnNext(args.FullPath); // Debounced handling - }; - watcher.Created += (sender, args) => - { - HandleFileCreation(args.FullPath); - }; - watcher.Deleted += (sender, args) => - { - HandleFileRemoval(args.FullPath); - }; - } - - private void HandleFileCreation(string fullPath) - { - try - { - var entry = Deserialize(File.ReadAllBytes(fullPath)); - Entries[entry.ID] = entry; - _filePathToIdMap[fullPath] = entry.ID; - EntryAdded.OnNext(entry); // Upstream notify - } - catch (Exception ex) - { - Logger.LogError($"Failed to handle creation of {fullPath}: {ex.Message}"); - } - } - - private void HandleFileChange(string fullPath) - { - try - { - var entry = Deserialize(File.ReadAllBytes(fullPath)); - Entries[entry.ID] = entry; - _filePathToIdMap[fullPath] = entry.ID; - EntryUpdated.OnNext(entry); // Upstream notify - } - catch (Exception ex) - { - Logger.LogError($"Failed to handle change in {fullPath}: {ex.Message}"); - } - } - - private void HandleFileRemoval(string fullPath) - { - try - { - // External deletion detected—handle without deserializing - if (_filePathToIdMap.TryGetValue(fullPath, out var id)) - { - _filePathToIdMap.TryRemove(fullPath, out _); // Clean map immediately - - // Emit delete if we have the entry locally - if (Entries.TryGetValue(id, out var entry)) - { - Entries.TryRemove(id, out _); // Sync local dict - EntryDeleted.OnNext(entry); - } - else - { - // Fallback: Stub if race/removed already (rare) - var type = GetTypeFromPath(fullPath); // See helper below - var stub = CreateStubEntry(id, type); - EntryDeleted.OnNext(stub); - Logger.LogWarning($"External deletion of {fullPath} (ID: {id}) with no local entry; emitted stub for cleanup."); - } - } - } - catch (Exception ex) - { - Logger.LogError($"Failed to handle deletion of {fullPath}: {ex.Message}"); - } - } - - // Helper: Infer type from path (e.g., based on directory name matching EntryTypeDirectories keys) - private Type GetTypeFromPath(string filePath) - { - var currentDir = new DirectoryInfo(Path.GetDirectoryName(filePath) ?? string.Empty); - // Walk up the directory tree to find matching EntryTypeDirectories (handles sub-subdirs) - while (currentDir is { Exists: true }) - { - var matchingType = EntryTypeDirectories.FirstOrDefault(kvp => kvp.Value.FullName == currentDir.FullName).Key; - if (matchingType != null) - return matchingType; - currentDir = currentDir.Parent; // Walk up (e.g., from "PlayerData/backup" to "PlayerData") - } - return typeof(DatabaseEntry); // Fallback if no match (e.g., manual file) - } - - // Stub creator (minimal; only for rare fallback—extend if Deletion events need more data) - private DatabaseEntry CreateStubEntry(Guid id, Type type) - { - var entry = (DatabaseEntry)Activator.CreateInstance(type); - typeof(DatabaseEntry).GetField("ID")?.SetValue(entry, id); // Set ID via reflection - return entry; - } - - /// - public new void Dispose() - { - base.Dispose(); - watcher?.Dispose(); - _changedSubject?.Dispose(); - } - } - - /// - /// An abstract cache backing store that persists all objects in a single file. - /// Supports serialization of the entire collection and provides hooks for custom binary formats. - /// Suitable for smaller datasets or when atomic updates are preferred over individual file management. - /// - /// - /// - /// All entries are loaded and saved as a single array in the specified file. - /// Changes do not trigger a write, must manually call PushAll to persist. - /// - /// - /// Subclasses must implement and - /// to define the storage format (e.g., JSON array, MessagePack array). - /// - /// - public abstract class SingleFileBackingStore : CacheBackingStore - { - /// - /// Gets the file where all entries are stored. - /// - public FileInfo FileInfo { get; } - - /// - /// Serializes an array of objects into a byte array for single-file storage. - /// - /// The array of objects to serialize. - /// A byte array representing the serialized entries. - public abstract byte[] Serialize(DatabaseEntry[] entries); - /// - /// Deserializes a byte array into an array of objects. - /// - /// The byte array containing the serialized entries. - /// An array of deserialized objects. - public abstract DatabaseEntry[] Deserialize(byte[] data); - - /// - /// Initializes a new instance of the class. - /// - /// The path to the single file for storing all entries. - public SingleFileBackingStore(string filePath) - { - FileInfo = new FileInfo(filePath); - } - - /// - /// Loads all objects from the single storage file by reading its contents, - /// deserializing via into an array, and adding each to the local dictionary. - /// Notifies upstream via for each entry. - /// If the file does not exist or deserialization fails, logs an error and returns without adding entries. - /// Logs the number of loaded entries for debugging. - /// - public override void PullAll() - { - if (!FileInfo.Exists) return; - - foreach (var entry in Deserialize(File.ReadAllBytes(FileInfo.FullName))) - { - Entries[entry.ID] = entry; - EntryAdded.OnNext(entry); - } - } - - /// - /// Adds or updates the specified in the local dictionary. - /// Does not immediately persist to the single file; call to save changes. - /// - /// The to persist. - public override void Push(DatabaseEntry entry) - { - var type = entry.GetType(); - Entries[entry.ID] = entry; - } - - /// - /// Removes the specified from the local dictionary. - /// Does not immediately persist to the single file; call to save changes. - /// - /// The to delete. - public override void Delete(DatabaseEntry entry) - { - if (Entries.TryGetValue(entry.ID, out _)) - { - Entries.TryRemove(entry.ID, out _); - } - } - - /// - /// Serializes all entries in to a byte array using - /// and writes it to the single storage file, overwriting any existing content. - /// The parameter is ignored in this implementation, as saves are always full rewrites. - /// Logs the number of saved entries and errors if the write fails, re-throwing for caller handling. - /// - /// If true, performs a soft push (not supported; treated as full push). - public override void PushAll(bool soft = false) - { - var entriesArray = Entries.Values.ToArray(); - File.WriteAllBytes(FileInfo.FullName, Serialize(entriesArray)); - } - } -} - -// E:\Projects\CultLib\src\GameCult.Caching\DatabaseEntry.cs - -using System; -using MessagePack; - -namespace GameCult.Caching -{ - public interface INamedEntry - { - string EntryName { get; set; } - } - - /// - /// Base abstract class for all database entries in the cache. - /// For optional MessagePack integration, define concrete subclasses with [MessagePackObject] and [Key(n)] on fields. - /// The ID field can be serialized via a custom formatter in MessagePack stores. - /// - public abstract class DatabaseEntry - { - /// - /// Unique identifier for the entry. Handled by custom formatters in MessagePack integration. - /// - public Guid ID = Guid.NewGuid(); - - /// - public override int GetHashCode() - { - return ID.GetHashCode(); - } - - /// - public override bool Equals(object obj) - { - if (obj is DatabaseEntry entry) return entry.ID == ID; - return false; - } - } - - /// - /// Generic link to a in the cache. - /// Value is computed from the cache; serializes only LinkID. - /// - public class DatabaseLink : DatabaseLinkBase where T : DatabaseEntry - { - /// - /// Computed value from the cache (not serialized; regenerated post-deserialize). - /// - public T? Value => Cache.Get(LinkID); - } - - /// - /// Base for database links. LinkID is serialized directly; Value is computed in subclasses. - /// - public class DatabaseLinkBase - { - /// - /// ID of the linked entry. - /// - public Guid LinkID; - - /// - /// Static reference to the cache (not serialized). - /// - public static CultCache? Cache; - } - - // Example of a DatabaseEntry implementation in user code - public class PlayerData : DatabaseEntry, INamedEntry - { - [Key(1)] - public string Email; - - [Key(2)] - public string PasswordHash; - - [Key(3)] - public string Username; - - [IgnoreMember] - public string EntryName - { - get => Username; - set => Username = value; - } - } -} - -// E:\Projects\CultLib\src\GameCult.Caching\ReflectionExtensions.cs - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Reflection; -using System.Text; -using System.Text.RegularExpressions; -using System.Threading.Tasks; - -namespace GameCult.Caching -{ - public static class ReflectionExtensions - { - private static Dictionary ParentTypes = new Dictionary(); - - public static Type[] GetParentTypes(this Type type) - { - if (ParentTypes.ContainsKey(type)) - return ParentTypes[type]; - return ParentTypes[type] = type.GetParents().ToArray(); - } - - private static IEnumerable GetParents(this Type type) - { - // is there any base type? - if (type == null) - { - yield break; - } - - // return all inherited types - var currentBaseType = type.BaseType; - while (currentBaseType != null) - { - yield return currentBaseType; - currentBaseType = currentBaseType.BaseType; - } - } - - private static Dictionary ChildClasses = new Dictionary(); - public static Type[] GetAllChildClasses(this Type type) - { - if (ChildClasses.ContainsKey(type)) - return ChildClasses[type]; - return ChildClasses[type] = AppDomain.CurrentDomain.GetAssemblies() - .SelectMany(ass => ass.GetTypes()).Where(type.IsAssignableFrom).ToArray(); - } - - } -} diff --git a/src/GameCult.Caching/README.md b/src/GameCult.Caching/README.md index 7e51227b..e9edb63d 100644 --- a/src/GameCult.Caching/README.md +++ b/src/GameCult.Caching/README.md @@ -37,11 +37,9 @@ Available cache attributes: - `CultGlobal` - `CultReference(targetType?, many: false)` -If you want compile-time document metadata instead of runtime assembly -spelunking, add the `GameCult.Caching.MessagePack.Generator` analyzer to the -project that declares your `[CultDocument]` types. It emits a metadata provider -the cache registry can load directly, while reflection stays behind as fallback -for projects that have not taken the hint yet. +`CultDocumentRegistry` builds every document descriptor by reflection over +these attributes and MessagePack's `[Key]`/`[IgnoreMember]`; it is the only +descriptor authority. ## Main Concepts diff --git a/src/GameCult.Geometry/CultGeometryDocuments.cs b/src/GameCult.Geometry/CultGeometryDocuments.cs index afb6d305..306b5849 100644 --- a/src/GameCult.Geometry/CultGeometryDocuments.cs +++ b/src/GameCult.Geometry/CultGeometryDocuments.cs @@ -4,7 +4,6 @@ using System.Linq; using System.Security.Cryptography; using System.Text; using GameCult.Caching; -using CultMath; using MessagePack; using static GameCult.Geometry.CultGeometryDocuments; @@ -15,10 +14,10 @@ namespace GameCult.Geometry /// public static class CultGeometrySchemaVersions { - public const string Domain = "gamecult.geometry.domain.v2"; - public const string BuildRequest = "gamecult.geometry.build_request.v2"; - public const string SelectedCut = "gamecult.geometry.selected_cut.v2"; - public const string ChunkArtifact = "gamecult.geometry.chunk_artifact.v2"; + public const string Domain = "gamecult.geometry.domain.v1"; + public const string BuildRequest = "gamecult.geometry.build_request.v1"; + public const string SelectedCut = "gamecult.geometry.selected_cut.v1"; + public const string ChunkArtifact = "gamecult.geometry.chunk_artifact.v1"; } /// @@ -68,10 +67,10 @@ namespace GameCult.Geometry public string Kind { get; set; } = string.Empty; [Key(2)] - public float3 Translation { get; set; } + public float[] Translation { get; set; } = Array.Empty(); [Key(3)] - public quaternion Rotation { get; set; } = quaternion.identity; + public float[] RotationXyzw { get; set; } = Array.Empty(); [Key(4)] public ulong Seed { get; set; } @@ -89,7 +88,7 @@ namespace GameCult.Geometry Name, Kind, StableVector(Translation), - StableVector(Rotation), + StableVector(RotationXyzw), Seed.ToString(CultureInfo.InvariantCulture) } .Concat(Claims.Select(claim => claim.StableFingerprint())) @@ -110,16 +109,16 @@ namespace GameCult.Geometry public string Name { get; set; } = string.Empty; [Key(1)] - public float3 Translation { get; set; } + public float[] Translation { get; set; } = Array.Empty(); [Key(2)] - public quaternion Rotation { get; set; } = quaternion.identity; + public float[] RotationXyzw { get; set; } = Array.Empty(); [Key(3)] - public float3 SupportCenter { get; set; } + public float[] SupportCenter { get; set; } = Array.Empty(); [Key(4)] - public float3 SupportSize { get; set; } + public float[] SupportSize { get; set; } = Array.Empty(); [Key(5)] public string Kind { get; set; } = string.Empty; @@ -133,7 +132,7 @@ namespace GameCult.Geometry public string StableFingerprint() => StableArray([ Name, StableVector(Translation), - StableVector(Rotation), + StableVector(RotationXyzw), StableVector(SupportCenter), StableVector(SupportSize), Kind, @@ -162,13 +161,13 @@ namespace GameCult.Geometry public string WorkerGroup { get; set; } = string.Empty; [Key(3)] - public float3 CameraPosition { get; set; } + public float[] CameraPosition { get; set; } = Array.Empty(); [Key(4)] - public float3 FrustumMin { get; set; } + public float[] FrustumMin { get; set; } = Array.Empty(); [Key(5)] - public float3 FrustumMax { get; set; } + public float[] FrustumMax { get; set; } = Array.Empty(); [Key(6)] public float ViewportHeightPixels { get; set; } @@ -206,9 +205,9 @@ namespace GameCult.Geometry StableVector(request.CameraPosition), StableVector(request.FrustumMin), StableVector(request.FrustumMax), - StableFloat(request.ViewportHeightPixels), - StableFloat(request.VerticalFovRadians), - StableFloat(request.TargetError), + request.ViewportHeightPixels.ToString("R", CultureInfo.InvariantCulture), + request.VerticalFovRadians.ToString("R", CultureInfo.InvariantCulture), + request.TargetError.ToString("R", CultureInfo.InvariantCulture), request.TriangleBudget.ToString(CultureInfo.InvariantCulture), request.ColliderBudget.ToString(CultureInfo.InvariantCulture), StableArray(request.SemanticFilter), @@ -322,10 +321,10 @@ namespace GameCult.Geometry public string SelectedCutId { get; set; } = string.Empty; [Key(3)] - public float3 BoundsMin { get; set; } + public float[] BoundsMin { get; set; } = Array.Empty(); [Key(4)] - public float3 BoundsMax { get; set; } + public float[] BoundsMax { get; set; } = Array.Empty(); [Key(5)] public string[] SourceDomainKeys { get; set; } = Array.Empty(); @@ -375,13 +374,13 @@ namespace GameCult.Geometry public sealed class CultGeometryTriangleMesh { [Key(0)] - public float3[] Positions { get; set; } = Array.Empty(); + public float[] Positions { get; set; } = Array.Empty(); [Key(1)] - public float3[] Normals { get; set; } = Array.Empty(); + public float[] Normals { get; set; } = Array.Empty(); [Key(2)] - public float2[] Uvs { get; set; } = Array.Empty(); + public float[] Uvs { get; set; } = Array.Empty(); [Key(3)] public uint[] Indices { get; set; } = Array.Empty(); @@ -410,20 +409,10 @@ namespace GameCult.Geometry return string.Concat(bytes.Select(value => value.ToString("x2"))); } - public static string StableVector(float2 value) => - string.Join(",", StableFloat(value.x), StableFloat(value.y)); - - public static string StableVector(float3 value) => - string.Join(",", StableFloat(value.x), StableFloat(value.y), StableFloat(value.z)); - - public static string StableVector(quaternion value) => - string.Join(",", StableFloat(value.x), StableFloat(value.y), StableFloat(value.z), StableFloat(value.w)); - - public static string StableVector(float2[] values) => - string.Join(",", values.SelectMany(value => new[] { value.x, value.y }).Select(StableFloat)); - - public static string StableVector(float3[] values) => - string.Join(",", values.SelectMany(value => new[] { value.x, value.y, value.z }).Select(StableFloat)); + public static string StableVector(float[] values) + { + return string.Join(",", values.Select(StableFloat)); + } public static string StableFloat(float value) { diff --git a/src/GameCult.Geometry/CultGeometryIsoSurface.cs b/src/GameCult.Geometry/CultGeometryIsoSurface.cs new file mode 100644 index 00000000..6fc9355c --- /dev/null +++ b/src/GameCult.Geometry/CultGeometryIsoSurface.cs @@ -0,0 +1,193 @@ +using System; +using System.Collections.Generic; +using CultMath; + +namespace GameCult.Geometry +{ + /// + /// Extracts engine-neutral triangle meshes from sampled scalar fields. + /// + public static class CultGeometryIsoSurface + { + private static readonly float3[] CubeCorners = + { + new(0f, 0f, 0f), + new(1f, 0f, 0f), + new(1f, 1f, 0f), + new(0f, 1f, 0f), + new(0f, 0f, 1f), + new(1f, 0f, 1f), + new(1f, 1f, 1f), + new(0f, 1f, 1f), + }; + + // A fixed six-tetrahedra decomposition keeps neighboring cube faces consistent. + private static readonly int[,] CubeTetrahedra = + { + { 0, 5, 1, 6 }, + { 0, 1, 2, 6 }, + { 0, 2, 3, 6 }, + { 0, 3, 7, 6 }, + { 0, 7, 4, 6 }, + { 0, 4, 5, 6 }, + }; + + /// + /// Extracts the surface with marching tetrahedra. + /// Values less than or equal to the isovalue are treated as inside. + /// + public static CultGeometryTriangleMesh Extract( + float[,,] samples, + float isoValue = 0f, + CultVec3 origin = default, + float cellSize = 1f) + { + if (samples == null) throw new ArgumentNullException(nameof(samples)); + if (samples.GetLength(0) < 2 || samples.GetLength(1) < 2 || samples.GetLength(2) < 2) + { + throw new ArgumentException("An isosurface field requires at least two samples on every axis.", nameof(samples)); + } + + if (!(cellSize > 0f) || float.IsInfinity(cellSize)) + { + throw new ArgumentOutOfRangeException(nameof(cellSize), "Cell size must be finite and positive."); + } + + var positions = new List(); + var normals = new List(); + var indices = new List(); + var originValue = new float3(origin.X, origin.Y, origin.Z); + var cornerPositions = new float3[8]; + var cornerValues = new float[8]; + var tetraPositions = new float3[4]; + var tetraValues = new float[4]; + + for (var x = 0; x < samples.GetLength(0) - 1; x++) + for (var y = 0; y < samples.GetLength(1) - 1; y++) + for (var z = 0; z < samples.GetLength(2) - 1; z++) + { + var cubeOrigin = originValue + new float3(x, y, z) * cellSize; + for (var corner = 0; corner < CubeCorners.Length; corner++) + { + var offset = CubeCorners[corner]; + cornerPositions[corner] = cubeOrigin + offset * cellSize; + cornerValues[corner] = samples[ + x + (int)offset.x, + y + (int)offset.y, + z + (int)offset.z]; + } + + for (var tetrahedron = 0; tetrahedron < CubeTetrahedra.GetLength(0); tetrahedron++) + { + for (var vertex = 0; vertex < 4; vertex++) + { + var corner = CubeTetrahedra[tetrahedron, vertex]; + tetraPositions[vertex] = cornerPositions[corner]; + tetraValues[vertex] = cornerValues[corner]; + } + + PolygonizeTetrahedron(tetraPositions, tetraValues, isoValue, positions, normals, indices); + } + } + + return new CultGeometryTriangleMesh + { + Positions = positions.ToArray(), + Normals = normals.ToArray(), + Indices = indices.ToArray(), + Uvs = Array.Empty(), + TriangleMaterials = new uint[indices.Count / 3], + }; + } + + private static void PolygonizeTetrahedron( + float3[] positions, + float[] values, + float isoValue, + List outputPositions, + List outputNormals, + List outputIndices) + { + Span inside = stackalloc int[4]; + Span outside = stackalloc int[4]; + var insideCount = 0; + var outsideCount = 0; + + for (var vertex = 0; vertex < 4; vertex++) + { + if (values[vertex] <= isoValue) inside[insideCount++] = vertex; + else outside[outsideCount++] = vertex; + } + + if (insideCount == 0 || insideCount == 4) return; + + var outward = Centroid(positions, outside, outsideCount) - Centroid(positions, inside, insideCount); + if (insideCount == 1 || outsideCount == 1) + { + var loneIsInside = insideCount == 1; + var lone = loneIsInside ? inside[0] : outside[0]; + var others = loneIsInside ? outside : inside; + var a = Interpolate(positions[lone], values[lone], positions[others[0]], values[others[0]], isoValue); + var b = Interpolate(positions[lone], values[lone], positions[others[1]], values[others[1]], isoValue); + var c = Interpolate(positions[lone], values[lone], positions[others[2]], values[others[2]], isoValue); + EmitOrientedTriangle(a, b, c, outward, outputPositions, outputNormals, outputIndices); + return; + } + + var p00 = Interpolate(positions[inside[0]], values[inside[0]], positions[outside[0]], values[outside[0]], isoValue); + var p01 = Interpolate(positions[inside[0]], values[inside[0]], positions[outside[1]], values[outside[1]], isoValue); + var p10 = Interpolate(positions[inside[1]], values[inside[1]], positions[outside[0]], values[outside[0]], isoValue); + var p11 = Interpolate(positions[inside[1]], values[inside[1]], positions[outside[1]], values[outside[1]], isoValue); + EmitOrientedTriangle(p00, p01, p11, outward, outputPositions, outputNormals, outputIndices); + EmitOrientedTriangle(p00, p11, p10, outward, outputPositions, outputNormals, outputIndices); + } + + private static float3 Interpolate(float3 first, float firstValue, float3 second, float secondValue, float isoValue) + { + var delta = secondValue - firstValue; + var amount = math.abs(delta) <= 1e-20f ? 0.5f : (isoValue - firstValue) / delta; + return math.lerp(first, second, amount); + } + + private static float3 Centroid(float3[] positions, Span vertices, int count) + { + var sum = float3.zero; + for (var index = 0; index < count; index++) sum += positions[vertices[index]]; + return sum / count; + } + + private static void EmitOrientedTriangle( + float3 a, + float3 b, + float3 c, + float3 outward, + List positions, + List normals, + List indices) + { + var normal = math.cross(b - a, c - a); + if (math.dot(normal, outward) < 0f) + { + (b, c) = (c, b); + normal = -normal; + } + + normal = math.normalize(normal); + var firstIndex = (uint)(positions.Count / 3); + Append(a, positions); + Append(b, positions); + Append(c, positions); + for (var vertex = 0; vertex < 3; vertex++) Append(normal, normals); + indices.Add(firstIndex); + indices.Add(firstIndex + 1); + indices.Add(firstIndex + 2); + } + + private static void Append(float3 value, List destination) + { + destination.Add(value.x); + destination.Add(value.y); + destination.Add(value.z); + } + } +} diff --git a/src/GameCult.Geometry/CultGeometryPrimitives.cs b/src/GameCult.Geometry/CultGeometryPrimitives.cs index 4636a3bc..99587b8e 100644 --- a/src/GameCult.Geometry/CultGeometryPrimitives.cs +++ b/src/GameCult.Geometry/CultGeometryPrimitives.cs @@ -1,10 +1,126 @@ using System; using System.Text.Json.Serialization; -using CultMath; using MessagePack; namespace GameCult.Geometry { + /// + /// Deterministic two-dimensional float vector for cross-runtime geometry and query contracts. + /// + [MessagePackObject] + public readonly struct CultVec2 : IEquatable + { + /// Creates a two-dimensional vector. + public CultVec2(float x, float y) + { + X = x; + Y = y; + } + + /// Gets the x component. + [Key(0)] + public float X { get; } + + /// Gets the y component. + [Key(1)] + public float Y { get; } + + /// Gets the zero vector. + [IgnoreMember] + [JsonIgnore] + public static CultVec2 Zero => new(0f, 0f); + + /// Gets the squared vector length. + [IgnoreMember] + [JsonIgnore] + public float LengthSquared => (X * X) + (Y * Y); + + /// Returns whether this vector is equal to another vector. + public bool Equals(CultVec2 other) => X.Equals(other.X) && Y.Equals(other.Y); + + /// + public override bool Equals(object? obj) => obj is CultVec2 other && Equals(other); + + /// + public override int GetHashCode() => HashCode.Combine(X, Y); + + /// + public override string ToString() => $"({X}, {Y})"; + + public static CultVec2 operator +(CultVec2 left, CultVec2 right) => new(left.X + right.X, left.Y + right.Y); + public static CultVec2 operator -(CultVec2 left, CultVec2 right) => new(left.X - right.X, left.Y - right.Y); + public static CultVec2 operator *(CultVec2 value, float scalar) => new(value.X * scalar, value.Y * scalar); + public static CultVec2 operator /(CultVec2 value, float scalar) => new(value.X / scalar, value.Y / scalar); + public static bool operator ==(CultVec2 left, CultVec2 right) => left.Equals(right); + public static bool operator !=(CultVec2 left, CultVec2 right) => !left.Equals(right); + } + + /// + /// Deterministic three-dimensional float vector for cross-runtime geometry and query contracts. + /// + [MessagePackObject] + public readonly struct CultVec3 : IEquatable + { + /// Creates a three-dimensional vector. + public CultVec3(float x, float y, float z) + { + X = x; + Y = y; + Z = z; + } + + /// Gets the x component. + [Key(0)] + public float X { get; } + + /// Gets the y component. + [Key(1)] + public float Y { get; } + + /// Gets the z component. + [Key(2)] + public float Z { get; } + + /// Gets the zero vector. + [IgnoreMember] + [JsonIgnore] + public static CultVec3 Zero => new(0f, 0f, 0f); + + /// Gets the XY plane projection. + [IgnoreMember] + [JsonIgnore] + public CultVec2 Xy => new(X, Y); + + /// Gets the XZ plane projection for Unity compatibility adapters. + [IgnoreMember] + [JsonIgnore] + public CultVec2 Xz => new(X, Z); + + /// Gets the squared vector length. + [IgnoreMember] + [JsonIgnore] + public float LengthSquared => (X * X) + (Y * Y) + (Z * Z); + + /// Returns whether this vector is equal to another vector. + public bool Equals(CultVec3 other) => X.Equals(other.X) && Y.Equals(other.Y) && Z.Equals(other.Z); + + /// + public override bool Equals(object? obj) => obj is CultVec3 other && Equals(other); + + /// + public override int GetHashCode() => HashCode.Combine(X, Y, Z); + + /// + public override string ToString() => $"({X}, {Y}, {Z})"; + + public static CultVec3 operator +(CultVec3 left, CultVec3 right) => new(left.X + right.X, left.Y + right.Y, left.Z + right.Z); + public static CultVec3 operator -(CultVec3 left, CultVec3 right) => new(left.X - right.X, left.Y - right.Y, left.Z - right.Z); + public static CultVec3 operator *(CultVec3 value, float scalar) => new(value.X * scalar, value.Y * scalar, value.Z * scalar); + public static CultVec3 operator /(CultVec3 value, float scalar) => new(value.X / scalar, value.Y / scalar, value.Z / scalar); + public static bool operator ==(CultVec3 left, CultVec3 right) => left.Equals(right); + public static bool operator !=(CultVec3 left, CultVec3 right) => !left.Equals(right); + } + /// /// Axis-aligned two-dimensional rectangle represented by canonical min and max corners. /// @@ -12,52 +128,52 @@ namespace GameCult.Geometry public readonly struct CultRect : IEquatable { /// Creates a rectangle and canonicalizes min/max corners. - public CultRect(float2 min, float2 max) + public CultRect(CultVec2 min, CultVec2 max) { - Min = new float2(Math.Min(min.x, max.x), Math.Min(min.y, max.y)); - Max = new float2(Math.Max(min.x, max.x), Math.Max(min.y, max.y)); + Min = new CultVec2(Math.Min(min.X, max.X), Math.Min(min.Y, max.Y)); + Max = new CultVec2(Math.Max(min.X, max.X), Math.Max(min.Y, max.Y)); } /// Creates a rectangle and canonicalizes min/max corners. public CultRect(float minX, float minY, float maxX, float maxY) - : this(new float2(minX, minY), new float2(maxX, maxY)) + : this(new CultVec2(minX, minY), new CultVec2(maxX, maxY)) { } /// Gets the canonical minimum corner. [Key(0)] - public float2 Min { get; } + public CultVec2 Min { get; } /// Gets the canonical maximum corner. [Key(1)] - public float2 Max { get; } + public CultVec2 Max { get; } /// Gets the rectangle size. [IgnoreMember] [JsonIgnore] - public float2 Size => Max - Min; + public CultVec2 Size => Max - Min; /// Gets the rectangle center. [IgnoreMember] [JsonIgnore] - public float2 Center => (Min + Max) * 0.5f; + public CultVec2 Center => (Min + Max) * 0.5f; /// Returns whether the point lies inside or on the rectangle edge. - public bool Contains(float2 point) + public bool Contains(CultVec2 point) { - return point.x >= Min.x && - point.x <= Max.x && - point.y >= Min.y && - point.y <= Max.y; + return point.X >= Min.X && + point.X <= Max.X && + point.Y >= Min.Y && + point.Y <= Max.Y; } /// Returns whether this rectangle intersects another rectangle. public bool Intersects(CultRect other) { - return Min.x <= other.Max.x && - Max.x >= other.Min.x && - Min.y <= other.Max.y && - Max.y >= other.Min.y; + return Min.X <= other.Max.X && + Max.X >= other.Min.X && + Min.Y <= other.Max.Y && + Max.Y >= other.Min.Y; } /// Returns whether this rectangle is equal to another rectangle. @@ -83,7 +199,7 @@ namespace GameCult.Geometry public readonly struct CultCircle : IEquatable { /// Creates a circle. - public CultCircle(float2 center, float radius) + public CultCircle(CultVec2 center, float radius) { if (radius < 0f) { @@ -96,7 +212,7 @@ namespace GameCult.Geometry /// Gets the circle center. [Key(0)] - public float2 Center { get; } + public CultVec2 Center { get; } /// Gets the circle radius. [Key(1)] @@ -106,24 +222,24 @@ namespace GameCult.Geometry [IgnoreMember] [JsonIgnore] public CultRect Bounds => new( - Center.x - Radius, - Center.y - Radius, - Center.x + Radius, - Center.y + Radius); + Center.X - Radius, + Center.Y - Radius, + Center.X + Radius, + Center.Y + Radius); /// Returns whether the point lies inside or on the circle edge. - public bool Contains(float2 point) + public bool Contains(CultVec2 point) { var delta = point - Center; - return ((delta.x * delta.x) + (delta.y * delta.y)) <= Radius * Radius; + return delta.LengthSquared <= Radius * Radius; } /// Returns whether this circle intersects a rectangle. public bool Intersects(CultRect rect) { - var clampedX = Math.Max(rect.Min.x, Math.Min(Center.x, rect.Max.x)); - var clampedY = Math.Max(rect.Min.y, Math.Min(Center.y, rect.Max.y)); - return Contains(new float2(clampedX, clampedY)); + var clampedX = Math.Max(rect.Min.X, Math.Min(Center.X, rect.Max.X)); + var clampedY = Math.Max(rect.Min.Y, Math.Min(Center.Y, rect.Max.Y)); + return Contains(new CultVec2(clampedX, clampedY)); } /// Returns whether this circle is equal to another circle. @@ -146,7 +262,7 @@ namespace GameCult.Geometry public readonly struct CultSphere : IEquatable { /// Creates a sphere. - public CultSphere(float3 center, float radius) + public CultSphere(CultVec3 center, float radius) { if (radius < 0f) { @@ -159,7 +275,7 @@ namespace GameCult.Geometry /// Gets the sphere center. [Key(0)] - public float3 Center { get; } + public CultVec3 Center { get; } /// Gets the sphere radius. [Key(1)] @@ -168,13 +284,13 @@ namespace GameCult.Geometry /// Gets the XY circle projection. [IgnoreMember] [JsonIgnore] - public CultCircle XyCircle => new(Center.xy, Radius); + public CultCircle XyCircle => new(Center.Xy, Radius); /// Returns whether the point lies inside or on the sphere surface. - public bool Contains(float3 point) + public bool Contains(CultVec3 point) { var delta = point - Center; - return ((delta.x * delta.x) + (delta.y * delta.y) + (delta.z * delta.z)) <= Radius * Radius; + return delta.LengthSquared <= Radius * Radius; } /// Returns whether this sphere is equal to another sphere. diff --git a/src/GameCult.Geometry/CultGeometryWorkerDocuments.cs b/src/GameCult.Geometry/CultGeometryWorkerDocuments.cs deleted file mode 100644 index 280f7c4f..00000000 --- a/src/GameCult.Geometry/CultGeometryWorkerDocuments.cs +++ /dev/null @@ -1,87 +0,0 @@ -using System; -using GameCult.Caching; -using MessagePack; - -namespace GameCult.Geometry -{ - public static class CultGeometryWorkerSchemaVersions - { - public const string State = "gamecult.geometry.worker_state.v1"; - } - - [MessagePackObject] - [CultDocument("gamecult.geometry.worker_state", CultGeometryWorkerSchemaVersions.State)] - public sealed class CultGeometryWorkerState - { - [Key(0), CultName] - public string WorkerId { get; set; } = string.Empty; - - [Key(1)] - public string Phase { get; set; } = string.Empty; - - [Key(2), CultReference(typeof(CultGeometryBuildRequest))] - public string ActiveRequestKey { get; set; } = string.Empty; - - [Key(3), CultReference(typeof(CultGeometrySelectedCutManifest))] - public string LastSelectedCutKey { get; set; } = string.Empty; - - [Key(4)] - public string[] LastArtifactKeys { get; set; } = Array.Empty(); - - [Key(5)] - public string LastError { get; set; } = string.Empty; - - [Key(6)] - public string UpdatedAt { get; set; } = string.Empty; - - [Key(7)] - public string ServedPackageVersion { get; set; } = string.Empty; - - public static CultRecordKey CreateRecordKey(string workerId) - { - if (string.IsNullOrWhiteSpace(workerId)) throw new ArgumentException("Value must be non-empty.", nameof(workerId)); - return new CultRecordKey("geometry:worker:" + workerId); - } - } - - [MessagePackObject] - public sealed class CultGeometryBuildCommand - { - [Key(0)] - public string RequestKey { get; set; } = string.Empty; - } - - [MessagePackObject] - public sealed class CultGeometryBuildReceipt - { - [Key(0)] - public string RequestKey { get; set; } = string.Empty; - - [Key(1)] - public string SelectedCutKey { get; set; } = string.Empty; - - [Key(2)] - public string[] ArtifactKeys { get; set; } = Array.Empty(); - - [Key(3)] - public string[] ContentHashes { get; set; } = Array.Empty(); - } - - public sealed class CultGeometryBuildOutput - { - public CultGeometrySelectedCutManifest SelectedCut { get; init; } = new(); - public CultGeometryChunkArtifact[] Artifacts { get; init; } = Array.Empty(); - } - - public sealed class CultGeometryDevelopmentProbe - { - public string Owner { get; init; } = string.Empty; - public string SchemaVersion { get; init; } = string.Empty; - public string SourceRecordKey { get; init; } = string.Empty; - public string SelectedCutKey { get; init; } = string.Empty; - public string[] SelectedNodes { get; init; } = Array.Empty(); - public string[] ArtifactKeys { get; init; } = Array.Empty(); - public string[] ContentHashes { get; init; } = Array.Empty(); - public string ServedPackageVersion { get; init; } = string.Empty; - } -} diff --git a/src/GameCult.Geometry/CultGeometryWorkerProvider.cs b/src/GameCult.Geometry/CultGeometryWorkerProvider.cs deleted file mode 100644 index 745da5a1..00000000 --- a/src/GameCult.Geometry/CultGeometryWorkerProvider.cs +++ /dev/null @@ -1,124 +0,0 @@ -using System; -using System.Linq; -using System.Reflection; -using System.Threading.Tasks; -using GameCult.Caching; -using GameCult.Mesh; -using GameCult.Networking; - -namespace GameCult.Geometry -{ - public interface ICultGeometryBuildPipeline - { - Task BuildAsync( - CultGeometryDomainDocument domain, - CultGeometryBuildRequest request); - } - - /// Owns geometry worker commands and the authoritative output commit path. - public sealed class CultGeometryWorkerProvider - { - public const string Owner = "GameCult.Geometry"; - public const string BuildOperationId = "gamecult.geometry.worker.build"; - - private readonly string _workerId; - private readonly CultNetDatabase _database; - private readonly ICultGeometryBuildPipeline _pipeline; - - public CultGeometryWorkerProvider( - string workerId, - CultNetDatabase database, - ICultGeometryBuildPipeline pipeline) - { - _workerId = string.IsNullOrWhiteSpace(workerId) - ? throw new ArgumentException("Value must be non-empty.", nameof(workerId)) - : workerId; - _database = database ?? throw new ArgumentNullException(nameof(database)); - _pipeline = pipeline ?? throw new ArgumentNullException(nameof(pipeline)); - BuildOperation = new CultMeshOperationHandle( - BuildOperationId, - (command, _) => BuildAsync(command)); - } - - public CultMeshOperationHandle BuildOperation { get; } - - public CultRecordKey WorkerStateKey => CultGeometryWorkerState.CreateRecordKey(_workerId); - - public async Task BuildAsync(CultGeometryBuildCommand command) - { - if (command == null) throw new ArgumentNullException(nameof(command)); - if (string.IsNullOrWhiteSpace(command.RequestKey)) - throw new ArgumentException("Request key must be non-empty.", nameof(command)); - - var requestKey = new CultRecordKey(command.RequestKey); - var request = await RequireAsync(requestKey).ConfigureAwait(false); - var domain = await RequireAsync(new CultRecordKey(request.DomainKey)).ConfigureAwait(false); - var output = await _pipeline.BuildAsync(domain, request).ConfigureAwait(false) - ?? throw new InvalidOperationException("Geometry build pipeline returned no output."); - - output.SelectedCut.RequestKey = requestKey.Value; - var cutKey = CultGeometrySelectedCutManifest.CreateRecordKey(output.SelectedCut); - await _database.PutAsync(cutKey, output.SelectedCut).ConfigureAwait(false); - - var artifactKeys = new string[output.Artifacts.Length]; - for (var index = 0; index < output.Artifacts.Length; index++) - { - var artifact = output.Artifacts[index]; - artifact.CutKey = cutKey.Value; - artifact.SelectedCutId = output.SelectedCut.CutId; - var artifactKey = CultGeometryChunkArtifact.CreateRecordKey(artifact); - await _database.PutAsync(artifactKey, artifact).ConfigureAwait(false); - artifactKeys[index] = artifactKey.Value; - } - - var receipt = new CultGeometryBuildReceipt - { - RequestKey = requestKey.Value, - SelectedCutKey = cutKey.Value, - ArtifactKeys = artifactKeys, - ContentHashes = artifactKeys.Select(ContentHashFromKey).ToArray() - }; - await _database.PutAsync(WorkerStateKey, new CultGeometryWorkerState - { - WorkerId = _workerId, - Phase = "completed", - ActiveRequestKey = requestKey.Value, - LastSelectedCutKey = cutKey.Value, - LastArtifactKeys = artifactKeys, - UpdatedAt = DateTimeOffset.UtcNow.ToString("O"), - ServedPackageVersion = PackageVersion - }).ConfigureAwait(false); - return receipt; - } - - public async Task ProbeAsync() - { - var state = await RequireAsync(WorkerStateKey).ConfigureAwait(false); - var cut = await RequireAsync(new CultRecordKey(state.LastSelectedCutKey)).ConfigureAwait(false); - return new CultGeometryDevelopmentProbe - { - Owner = Owner, - SchemaVersion = CultGeometrySchemaVersions.SelectedCut, - SourceRecordKey = state.ActiveRequestKey, - SelectedCutKey = state.LastSelectedCutKey, - SelectedNodes = cut.SelectedNodes, - ArtifactKeys = state.LastArtifactKeys, - ContentHashes = state.LastArtifactKeys.Select(ContentHashFromKey).ToArray(), - ServedPackageVersion = PackageVersion - }; - } - - private async Task RequireAsync(CultRecordKey key) where T : class => - await _database.GetAsync(key).ConfigureAwait(false) - ?? throw new InvalidOperationException($"Required geometry document '{key.Value}' was not found."); - - private static string ContentHashFromKey(string key) => - key[(key.LastIndexOf(':') + 1)..]; - - private static string PackageVersion => - typeof(CultGeometryWorkerProvider).Assembly - .GetCustomAttribute()?.InformationalVersion - ?? typeof(CultGeometryWorkerProvider).Assembly.GetName().Version?.ToString() - ?? "unknown"; - } -} diff --git a/src/GameCult.Geometry/CultMathMessagePackResolver.cs b/src/GameCult.Geometry/CultMathMessagePackResolver.cs deleted file mode 100644 index 6e0075cf..00000000 --- a/src/GameCult.Geometry/CultMathMessagePackResolver.cs +++ /dev/null @@ -1,106 +0,0 @@ -using CultMath; -using GameCult.Caching.MessagePack; -using MessagePack; -using MessagePack.Formatters; - -[assembly: CultDocumentMessagePackResolvers(typeof(GameCult.Geometry.CultMathMessagePackResolver))] - -namespace GameCult.Geometry; - -/// -/// Owns the persisted positional representation of CultMath values used by -/// GameCult.Geometry documents. CultMath remains serialization-agnostic. -/// -public sealed class CultMathMessagePackResolver : IFormatterResolver -{ - public IMessagePackFormatter? GetFormatter() - { - if (typeof(T) == typeof(float2)) return (IMessagePackFormatter)(object)Float2Formatter.Instance; - if (typeof(T) == typeof(float3)) return (IMessagePackFormatter)(object)Float3Formatter.Instance; - if (typeof(T) == typeof(float4)) return (IMessagePackFormatter)(object)Float4Formatter.Instance; - if (typeof(T) == typeof(quaternion)) return (IMessagePackFormatter)(object)QuaternionFormatter.Instance; - return null; - } - - private static void RequireComponents(ref MessagePackReader reader, int expected, string typeName) - { - var actual = reader.ReadArrayHeader(); - if (actual != expected) - { - throw new MessagePackSerializationException($"{typeName} requires exactly {expected} components; payload contained {actual}."); - } - } - - internal sealed class Float2Formatter : IMessagePackFormatter - { - public static readonly Float2Formatter Instance = new(); - public void Serialize(ref MessagePackWriter writer, float2 value, MessagePackSerializerOptions options) - { - writer.WriteArrayHeader(2); - writer.Write(value.x); - writer.Write(value.y); - } - - public float2 Deserialize(ref MessagePackReader reader, MessagePackSerializerOptions options) - { - RequireComponents(ref reader, 2, nameof(float2)); - return new float2(reader.ReadSingle(), reader.ReadSingle()); - } - } - - internal sealed class Float3Formatter : IMessagePackFormatter - { - public static readonly Float3Formatter Instance = new(); - public void Serialize(ref MessagePackWriter writer, float3 value, MessagePackSerializerOptions options) - { - writer.WriteArrayHeader(3); - writer.Write(value.x); - writer.Write(value.y); - writer.Write(value.z); - } - - public float3 Deserialize(ref MessagePackReader reader, MessagePackSerializerOptions options) - { - RequireComponents(ref reader, 3, nameof(float3)); - return new float3(reader.ReadSingle(), reader.ReadSingle(), reader.ReadSingle()); - } - } - - internal sealed class Float4Formatter : IMessagePackFormatter - { - public static readonly Float4Formatter Instance = new(); - public void Serialize(ref MessagePackWriter writer, float4 value, MessagePackSerializerOptions options) - { - writer.WriteArrayHeader(4); - writer.Write(value.x); - writer.Write(value.y); - writer.Write(value.z); - writer.Write(value.w); - } - - public float4 Deserialize(ref MessagePackReader reader, MessagePackSerializerOptions options) - { - RequireComponents(ref reader, 4, nameof(float4)); - return new float4(reader.ReadSingle(), reader.ReadSingle(), reader.ReadSingle(), reader.ReadSingle()); - } - } - - internal sealed class QuaternionFormatter : IMessagePackFormatter - { - public static readonly QuaternionFormatter Instance = new(); - public void Serialize(ref MessagePackWriter writer, quaternion value, MessagePackSerializerOptions options) - { - writer.WriteArrayHeader(4); - writer.Write(value.x); - writer.Write(value.y); - writer.Write(value.z); - writer.Write(value.w); - } - - public quaternion Deserialize(ref MessagePackReader reader, MessagePackSerializerOptions options) - { - RequireComponents(ref reader, 4, nameof(quaternion)); - return new quaternion(reader.ReadSingle(), reader.ReadSingle(), reader.ReadSingle(), reader.ReadSingle()); - } - } -} diff --git a/src/GameCult.Geometry/GameCult.Geometry.csproj b/src/GameCult.Geometry/GameCult.Geometry.csproj index 705fa2f4..a3ad9457 100644 --- a/src/GameCult.Geometry/GameCult.Geometry.csproj +++ b/src/GameCult.Geometry/GameCult.Geometry.csproj @@ -1,33 +1,21 @@ - Spatial values, planetary geometry, queries, and typed geometry state built on CultMath primitives + CultGeometry algorithms and typed CultCache geometry artifacts GameCult.Geometry - 0.1.0 - false - game;geometry;planetary;csg;lod;cultcache;cultmesh + game;geometry;csg;lod;cultcache;cultmesh;vibegeometry;fensalir README.md - - LICENSES.md $(NoWarn);1591 - 0.1.0 - 1.0.3 - - - - - + + - - - - + diff --git a/src/GameCult.Geometry/ISOSURFACE-PROVENANCE.md b/src/GameCult.Geometry/ISOSURFACE-PROVENANCE.md new file mode 100644 index 00000000..a15a4933 --- /dev/null +++ b/src/GameCult.Geometry/ISOSURFACE-PROVENANCE.md @@ -0,0 +1,32 @@ +# Isosurface extraction provenance + +`CultGeometryIsoSurface` is a clean implementation of marching tetrahedra over +a regular scalar grid. The implementation follows the tetrahedral case shape +described by Peter Shirley and Allan Tuchman, “A Polygonal Approximation to +Direct Scalar Volume Rendering,” Computer Graphics 24(5), 1990, +doi:10.1145/99308.99322. + +The DELVE/HOLD port was behaviorally motivated by these unversioned local Unity +sources: + +- `D:\WIP4\Projects\VoxelTerrain\Assets\Scripts\Procedural\GenerateTerrain.cs` +- `D:\WIP4\Projects\VoxelTerrain\Assets\Plugins\MarchingCubes.cs` + +The source directory is not a Git checkout and the plugin carries no license or +author notice. Its code and lookup tables were therefore not copied. The +CultGeometry implementation uses a fresh six-tetrahedra decomposition, explicit +inside/outside classification, interpolation, winding, and flat-normal output. + +Deliberate behavior retained: + +- scalar samples less than or equal to the isovalue are inside; +- intersections are linearly interpolated along cell edges; +- vertices are emitted per triangle rather than welded; +- output is engine-neutral positions, normals, indices, and material slots. + +Deliberate changes: + +- no Unity mesh, physics, coroutine, navigation, or random dependency; +- no mutable global target, mode, or winding state; +- outward winding is derived from inside-to-outside direction; +- invalid fields and cell sizes fail at the public boundary. diff --git a/src/GameCult.Geometry/LICENSES.md b/src/GameCult.Geometry/LICENSES.md deleted file mode 100644 index 67455b87..00000000 --- a/src/GameCult.Geometry/LICENSES.md +++ /dev/null @@ -1,13 +0,0 @@ -# GameCult.Geometry licenses - -GameCult.Geometry is a mixed-license package. License follows file ownership; -the package as a whole is not relicensed under one blanket expression. - -- CultLib-authored integration, persistence, schema, and packaging files are - licensed under the MIT License published by the CultLib repository. -- Files under `Planetary/` and `Shaders/` transferred from CultMath remain - licensed under the Mozilla Public License 2.0. Their source notices and - provenance are preserved in `THIRD-PARTY-NOTICES.md`. - -Downstream redistributors must preserve the applicable notices for the files -they redistribute. diff --git a/src/GameCult.Geometry/Planetary/AdvancedErosionFilter.cs b/src/GameCult.Geometry/Planetary/AdvancedErosionFilter.cs deleted file mode 100644 index 487cdc87..00000000 --- a/src/GameCult.Geometry/Planetary/AdvancedErosionFilter.cs +++ /dev/null @@ -1,141 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this file, -// You can obtain one at https://mozilla.org/MPL/2.0/. -// -// Mathematical procedure adapted from Rune Skovbo Johansen's Advanced Terrain -// Erosion Filter (2025): https://www.shadertoy.com/view/wXcfWn - -using CultMath; - -namespace GameCult.Geometry; - -public readonly record struct AdvancedErosionParameters( - float Scale, - float Strength, - float GullyWeight, - float Detail, - float4 Rounding, - float4 Onset, - float2 AssumedSlope, - float CellScale, - float Normalization, - int Octaves, - float Lacunarity, - float Gain) -{ - public static AdvancedErosionParameters Default => new( - 0.15f, 0.22f, 0.5f, 1.5f, - new float4(0.1f, 0.0f, 0.1f, 2.0f), - new float4(1.25f, 1.25f, 2.8f, 1.5f), - new float2(0.7f, 1.0f), - 0.7f, 0.5f, 5, 2.0f, 0.5f); -} - -public readonly record struct AdvancedErosionResult(float3 Delta, float Magnitude, float RidgeMap, float FadeTarget); - -public static class AdvancedErosionFilter -{ - public static AdvancedErosionResult Sample(float2 position, float3 baseHeightAndSlope, float fadeTarget, AdvancedErosionParameters p) - => Sample(position, baseHeightAndSlope, fadeTarget, p, new ErosionBandSelection(p.Octaves, 1.0f, 0.0f, 0.0f)); - - public static AdvancedErosionResult Sample(float2 position, float3 baseHeightAndSlope, float fadeTarget, AdvancedErosionParameters p, ErosionBandSelection band) - { - var strength = p.Strength * p.Scale; - fadeTarget = math.clamp(fadeTarget, -1.0f, 1.0f); - var input = baseHeightAndSlope; - var heightAndSlope = input; - var frequency = 1.0f / math.max(p.Scale * p.CellScale, 1.0e-10f); - var slope = new float2(heightAndSlope.y, heightAndSlope.z); - var slopeLength = math.max(math.length(slope), 1.0e-10f); - var magnitude = 0.0f; - var roundingMultiplier = 1.0f; - - var inputRounding = math.lerp(p.Rounding.y, p.Rounding.x, math.saturate(fadeTarget + 0.5f)) * p.Rounding.z; - var combinedMask = EaseOut(SmoothStart(slopeLength * p.Onset.x, inputRounding * p.Onset.x)); - var ridgeMask = EaseOut(slopeLength * p.Onset.z); - var ridgeFade = fadeTarget; - var gullySlope = math.lerp(slope, slope / slopeLength * p.AssumedSlope.x, p.AssumedSlope.y); - - var octaves = math.min(math.min(math.max(p.Octaves, 0), 8), math.max(band.ActiveOctaves, 0)); - for (var octave = 0; octave < octaves; octave++) - { - var phacelle = Phacelle(position * frequency, SafeNormalize(gullySlope), p.CellScale, 0.25f, p.Normalization); - var derivativeDirection = new float2(phacelle.z, phacelle.w) * -frequency; - var sloping = math.abs(phacelle.y); - gullySlope += math.sign(phacelle.y) * derivativeDirection * strength * p.GullyWeight; - - var gullies = new float3(phacelle.x, phacelle.y * derivativeDirection.x, phacelle.y * derivativeDirection.y); - var faded = math.lerp(new float3(fadeTarget, 0.0f, 0.0f), gullies * p.GullyWeight, combinedMask); - var octaveWeight = octave == octaves - 1 ? math.saturate(band.FinalOctaveWeight) : 1.0f; - heightAndSlope += faded * strength * octaveWeight; - magnitude += strength * octaveWeight; - fadeTarget = faded.x; - - var octaveRounding = math.lerp(p.Rounding.y, p.Rounding.x, math.saturate(phacelle.x + 0.5f)) * roundingMultiplier; - var newMask = EaseOut(SmoothStart(sloping * p.Onset.y, octaveRounding * p.Onset.y)); - combinedMask = PowInverse(combinedMask, p.Detail) * newMask; - - ridgeFade = math.lerp(ridgeFade, math.lerp(ridgeFade, gullies.x, ridgeMask), octaveWeight); - ridgeMask *= math.lerp(1.0f, EaseOut(sloping * p.Onset.w), octaveWeight); - strength *= p.Gain; - frequency *= p.Lacunarity; - roundingMultiplier *= p.Rounding.w; - } - - return new AdvancedErosionResult(heightAndSlope - input, magnitude, ridgeFade * (1.0f - ridgeMask), fadeTarget); - } - - public static float4 Phacelle(float2 position, float2 normalDirection, float frequency, float offsetCycles, float normalization) - { - var sideDirection = new float2(-normalDirection.y, normalDirection.x) * frequency * math.TAU; - var offset = offsetCycles * math.TAU; - var cell = math.floor(position); - var local = math.frac(position); - var phase = float2.zero; - var weightSum = 0.0f; - - for (var x = -1; x <= 2; x++) - for (var y = -1; y <= 2; y++) - { - var gridOffset = new float2(x, y); - var gridPoint = cell + gridOffset; - var randomOffset = Hash2(gridPoint) * 0.5f; - var delta = local - gridOffset - randomOffset; - var weight = math.max(0.0f, math.exp(-math.dot(delta, delta) * 2.0f) - 0.01111f); - weightSum += weight; - var wave = math.dot(delta, sideDirection) + offset; - phase += new float2(math.cos(wave), math.sin(wave)) * weight; - } - - var interpolated = phase / math.max(weightSum, 1.0e-20f); - var magnitude = math.max(1.0f - math.saturate(normalization), math.length(interpolated)); - var normalized = interpolated / math.max(magnitude, 1.0e-20f); - return new float4(normalized, sideDirection.x, sideDirection.y); - } - - private static float2 Hash2(float2 input) - { - var x = (int)input.x; - var y = (int)input.y; - return new float2(HashUnit(x, y, 0x68bc21ebu), HashUnit(x, y, 0x02e5be93u)) * 2.0f - float2.one; - } - - private static float HashUnit(int x, int y, uint salt) - { - var value = unchecked((uint)x * 0x8da6b343u ^ (uint)y * 0xd8163841u ^ salt); - value ^= value >> 16; value *= 0x7feb352du; value ^= value >> 15; value *= 0x846ca68bu; value ^= value >> 16; - return (value & 0x00ffffffu) * (1.0f / 16777216.0f); - } - - private static float2 SafeNormalize(float2 value) - { - var length = math.length(value); - return math.abs(length) > 1.0e-10f ? value / length : value; - } - - private static float PowInverse(float value, float power) => 1.0f - math.pow(1.0f - math.saturate(value), power); - private static float EaseOut(float value) { var inverse = 1.0f - math.saturate(value); return 1.0f - inverse * inverse; } - private static float SmoothStart(float value, float smoothing) => smoothing <= 1.0e-10f || value >= smoothing - ? value - 0.5f * smoothing - : 0.5f * value * value / smoothing; -} diff --git a/src/GameCult.Geometry/Planetary/ErosionFrequencyBands.cs b/src/GameCult.Geometry/Planetary/ErosionFrequencyBands.cs deleted file mode 100644 index 8cadbf11..00000000 --- a/src/GameCult.Geometry/Planetary/ErosionFrequencyBands.cs +++ /dev/null @@ -1,63 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this file, -// You can obtain one at https://mozilla.org/MPL/2.0/. - -using CultMath; - -namespace GameCult.Geometry; - -public readonly record struct ErosionBandSelection( - int ActiveOctaves, - float FinalOctaveWeight, - float FinestIncludedWavelength, - float UnresolvedHeightBound) -{ - public bool HasSamples => ActiveOctaves > 0; -} - -public static class ErosionFrequencyBands -{ - public static ErosionBandSelection Select( - float baseWavelength, - float sampleSpacing, - int maximumOctaves, - float lacunarity, - float baseAmplitude, - float gain, - float transitionRatio = 2.0f) - { - baseWavelength = math.max(baseWavelength, 1.0e-6f); - sampleSpacing = math.max(sampleSpacing, 0.0f); - lacunarity = math.max(lacunarity, 1.01f); - transitionRatio = math.max(transitionRatio, 1.01f); - maximumOctaves = math.min(math.max(maximumOctaves, 0), 16); - var nyquistWavelength = sampleSpacing * 2.0f; - var wavelength = baseWavelength; - var active = 0; - var finalWeight = 0.0f; - var finest = 0.0f; - - for (var octave = 0; octave < maximumOctaves; octave++) - { - var weight = nyquistWavelength <= 0.0f - ? 1.0f - : math.smoothstep(nyquistWavelength, nyquistWavelength * transitionRatio, wavelength); - if (weight <= 0.0f) break; - active++; - finalWeight = weight; - finest = wavelength; - wavelength /= lacunarity; - } - - var omittedAmplitude = 0.0f; - var amplitude = math.abs(baseAmplitude); - for (var octave = 0; octave < maximumOctaves; octave++) - { - if (octave >= active) omittedAmplitude += amplitude; - else if (octave == active - 1) omittedAmplitude += amplitude * (1.0f - finalWeight); - amplitude *= math.abs(gain); - } - - return new ErosionBandSelection(active, active > 0 ? finalWeight : 0.0f, finest, omittedAmplitude); - } -} diff --git a/src/GameCult.Geometry/Planetary/PlanetaryField.cs b/src/GameCult.Geometry/Planetary/PlanetaryField.cs deleted file mode 100644 index 21f4f805..00000000 --- a/src/GameCult.Geometry/Planetary/PlanetaryField.cs +++ /dev/null @@ -1,220 +0,0 @@ -using System; -using CultMath; - -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this file, -// You can obtain one at https://mozilla.org/MPL/2.0/. - -namespace GameCult.Geometry; - -public readonly record struct PlanetaryQueryScale(float FootprintMeters, float MaximumUnresolvedHeight) -{ - public static PlanetaryQueryScale AtFootprint(float footprintMeters, float maximumUnresolvedHeight = float.PositiveInfinity) - => new PlanetaryQueryScale(footprintMeters, maximumUnresolvedHeight).Validate(); - - public PlanetaryQueryScale Validate() - { - if (!float.IsFinite(FootprintMeters) || FootprintMeters < 0) throw new ArgumentOutOfRangeException(nameof(FootprintMeters)); - if (float.IsNaN(MaximumUnresolvedHeight) || MaximumUnresolvedHeight < 0) throw new ArgumentOutOfRangeException(nameof(MaximumUnresolvedHeight)); - return this; - } -} - -public readonly record struct PlanetaryFieldDefinition( - ulong FieldVersion, - float Radius, - int Seed, - AdvancedErosionParameters Erosion) -{ - public static PlanetaryFieldDefinition Create( - ulong baseFieldVersion, - float radius, - int seed, - AdvancedErosionParameters erosion) - => new PlanetaryFieldDefinition(PlanetaryFieldIdentity.Compute(baseFieldVersion, radius, seed, erosion), radius, seed, erosion).Validate(); - - public PlanetaryFieldDefinition Validate() - { - if (FieldVersion == 0) throw new ArgumentOutOfRangeException(nameof(FieldVersion)); - if (!float.IsFinite(Radius) || Radius <= 0) throw new ArgumentOutOfRangeException(nameof(Radius)); - if (!float.IsFinite(Erosion.Scale) || Erosion.Scale <= 0) throw new ArgumentOutOfRangeException(nameof(Erosion)); - return this; - } -} - -public static class PlanetaryFieldIdentity -{ - public static ulong Compute(ulong baseFieldVersion, float radius, int seed, in AdvancedErosionParameters erosion) - { - if (baseFieldVersion == 0) throw new ArgumentOutOfRangeException(nameof(baseFieldVersion)); - const ulong offset = 14695981039346656037UL; - const ulong prime = 1099511628211UL; - var hash = offset; - Add(baseFieldVersion); - Add((uint)BitConverter.SingleToInt32Bits(radius)); Add((uint)seed); - Add((uint)BitConverter.SingleToInt32Bits(erosion.Scale)); Add((uint)BitConverter.SingleToInt32Bits(erosion.Strength)); - Add((uint)BitConverter.SingleToInt32Bits(erosion.GullyWeight)); Add((uint)BitConverter.SingleToInt32Bits(erosion.Detail)); - AddFloat4(erosion.Rounding); AddFloat4(erosion.Onset); - Add((uint)BitConverter.SingleToInt32Bits(erosion.AssumedSlope.x)); Add((uint)BitConverter.SingleToInt32Bits(erosion.AssumedSlope.y)); - Add((uint)BitConverter.SingleToInt32Bits(erosion.CellScale)); Add((uint)BitConverter.SingleToInt32Bits(erosion.Normalization)); - Add((uint)erosion.Octaves); Add((uint)BitConverter.SingleToInt32Bits(erosion.Lacunarity)); Add((uint)BitConverter.SingleToInt32Bits(erosion.Gain)); - return hash == 0 ? 1 : hash; - - void Add(ulong value) => hash = unchecked((hash ^ value) * prime); - void AddFloat4(float4 value) - { - Add((uint)BitConverter.SingleToInt32Bits(value.x)); Add((uint)BitConverter.SingleToInt32Bits(value.y)); - Add((uint)BitConverter.SingleToInt32Bits(value.z)); Add((uint)BitConverter.SingleToInt32Bits(value.w)); - } - } -} - -/// -/// Application-supplied base evidence at one unit spherical direction. -/// FieldValue and FieldGradient drive erosion. RadialDisplacement and -/// RadialGradient describe the uneroded surface in metres. -/// -public readonly record struct PlanetaryBaseFieldSample( - float RadialDisplacement, - float3 RadialGradient, - float FieldValue, - float3 FieldGradient, - float FadeTarget); - -public readonly record struct PlanetarySurfaceSample( - ulong FieldVersion, - float3 UnitDirection, - float Radius, - float RadialDisplacement, - float3 TangentGradient, - float3 SurfaceNormal, - float Slope, - float Ridge, - float Gully, - float FinestResolvedWavelength, - float UnresolvedHeightBound) -{ - public float SurfaceRadius => Radius + RadialDisplacement; - public float3 PlanetLocalPosition => UnitDirection * SurfaceRadius; - public bool Meets(PlanetaryQueryScale scale) => UnresolvedHeightBound <= scale.MaximumUnresolvedHeight; -} - -public static class PlanetaryField -{ - public static PlanetarySurfaceSample Sample( - in PlanetaryFieldDefinition definition, - float3 unitDirection, - in PlanetaryBaseFieldSample baseSample, - in PlanetaryQueryScale scale) - { - definition.Validate(); - scale.Validate(); - PlanetaryTopology.ValidateDirection(unitDirection); - PlanetaryTopology.ValidateFinite(baseSample.RadialGradient, nameof(baseSample)); - PlanetaryTopology.ValidateFinite(baseSample.FieldGradient, nameof(baseSample)); - if (!float.IsFinite(baseSample.RadialDisplacement) || !float.IsFinite(baseSample.FieldValue) || !float.IsFinite(baseSample.FadeTarget)) - throw new ArgumentOutOfRangeException(nameof(baseSample)); - - var direction = math.normalize(unitDirection); - var radialGradient = Tangent(direction, baseSample.RadialGradient); - var fieldGradient = Tangent(direction, baseSample.FieldGradient); - var p = definition.Erosion; - var band = ErosionFrequencyBands.Select( - p.Scale * p.CellScale, - scale.FootprintMeters, - p.Octaves, - p.Lacunarity, - p.Strength * p.Scale, - p.Gain); - - var powers = new float3( - Pow4(MathF.Abs(direction.x)), - Pow4(MathF.Abs(direction.y)), - Pow4(MathF.Abs(direction.z))); - var powerSum = math.max(powers.x + powers.y + powers.z, 1.0e-6f); - var weights = powers / powerSum; - var world = direction * definition.Radius; - var seed = definition.Seed; - var xy = AdvancedErosionFilter.Sample( - world.xy + new float2(713 + seed * 0.754877666f, -291 + seed * 0.569840296f), - new float3(baseSample.FieldValue, fieldGradient.x, fieldGradient.y), baseSample.FadeTarget, p, band); - var yz = AdvancedErosionFilter.Sample( - world.yz + new float2(-431 + seed * 0.438289027f, 887 + seed * 0.328438163f), - new float3(baseSample.FieldValue, fieldGradient.y, fieldGradient.z), baseSample.FadeTarget, p, band); - var zx = AdvancedErosionFilter.Sample( - new float2(world.z, world.x) + new float2(197 + seed * 0.219783071f, 557 + seed * 0.145898034f), - new float3(baseSample.FieldValue, fieldGradient.z, fieldGradient.x), baseSample.FadeTarget, p, band); - - var erosionHeight = xy.Delta.x * weights.z + yz.Delta.x * weights.x + zx.Delta.x * weights.y; - var erosionGradient = new float3( - xy.Delta.y * weights.z + zx.Delta.z * weights.y, - xy.Delta.z * weights.z + yz.Delta.y * weights.x, - yz.Delta.z * weights.x + zx.Delta.y * weights.y); - var powerGradientX = Tangent(direction, new float3(4 * direction.x * direction.x * direction.x / definition.Radius, 0, 0)); - var powerGradientY = Tangent(direction, new float3(0, 4 * direction.y * direction.y * direction.y / definition.Radius, 0)); - var powerGradientZ = Tangent(direction, new float3(0, 0, 4 * direction.z * direction.z * direction.z / definition.Radius)); - var sumGradient = powerGradientX + powerGradientY + powerGradientZ; - var weightGradientX = (powerGradientX * powerSum - sumGradient * powers.x) / (powerSum * powerSum); - var weightGradientY = (powerGradientY * powerSum - sumGradient * powers.y) / (powerSum * powerSum); - var weightGradientZ = (powerGradientZ * powerSum - sumGradient * powers.z) / (powerSum * powerSum); - erosionGradient += yz.Delta.x * weightGradientX + zx.Delta.x * weightGradientY + xy.Delta.x * weightGradientZ; - var gradient = Tangent(direction, radialGradient + erosionGradient); - var ridgeEvidence = xy.RidgeMap * weights.z + yz.RidgeMap * weights.x + zx.RidgeMap * weights.y; - var ridge = math.saturate(ridgeEvidence * 0.5f + 0.5f); - var gully = math.saturate(0.5f - ridgeEvidence * 0.5f); - var displacement = baseSample.RadialDisplacement + erosionHeight; - - return new( - definition.FieldVersion, - direction, - definition.Radius, - displacement, - gradient, - PlanetaryTopology.SurfaceNormal(direction, gradient), - math.length(gradient), - ridge, - gully, - band.FinestIncludedWavelength, - band.UnresolvedHeightBound); - } - - public static void SampleBatch( - in PlanetaryFieldDefinition definition, - ReadOnlySpan directions, - in PlanetaryQueryScale scale, - TSource source, - Span destination) - where TSource : IPlanetaryBaseField - { - if (destination.Length < directions.Length) throw new ArgumentException("Destination is shorter than directions.", nameof(destination)); - for (var i = 0; i < directions.Length; i++) - { - var direction = math.normalize(directions[i]); - var baseSample = source.Sample(direction); - destination[i] = Sample(definition, direction, baseSample, scale); - } - } - - public static PlanetarySurfaceSample SamplePosition( - in PlanetaryFieldDefinition definition, - double3 planetLocalPosition, - in PlanetaryQueryScale scale, - TSource source) - where TSource : IPlanetaryBaseField - { - if (!double.IsFinite(planetLocalPosition.x) || !double.IsFinite(planetLocalPosition.y) || !double.IsFinite(planetLocalPosition.z)) - throw new ArgumentOutOfRangeException(nameof(planetLocalPosition)); - var length = Math.Sqrt(planetLocalPosition.x * planetLocalPosition.x + planetLocalPosition.y * planetLocalPosition.y + planetLocalPosition.z * planetLocalPosition.z); - if (length < 1.0e-12) throw new ArgumentOutOfRangeException(nameof(planetLocalPosition)); - var direction = new float3((float)(planetLocalPosition.x / length), (float)(planetLocalPosition.y / length), (float)(planetLocalPosition.z / length)); - return Sample(definition, direction, source.Sample(direction), scale); - } - - private static float3 Tangent(float3 direction, float3 gradient) => gradient - direction * math.dot(gradient, direction); - private static float Pow4(float value) { var square = value * value; return square * square; } -} - -public interface IPlanetaryBaseField -{ - PlanetaryBaseFieldSample Sample(float3 unitDirection); -} diff --git a/src/GameCult.Geometry/Planetary/PlanetaryGpuPages.cs b/src/GameCult.Geometry/Planetary/PlanetaryGpuPages.cs deleted file mode 100644 index a6d7f4df..00000000 --- a/src/GameCult.Geometry/Planetary/PlanetaryGpuPages.cs +++ /dev/null @@ -1,60 +0,0 @@ -using System; -using CultMath; - -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this file, -// You can obtain one at https://mozilla.org/MPL/2.0/. - -namespace GameCult.Geometry; - -public readonly record struct PlanetaryGpuPageInput(float4 DirectionRadius, float4 Sampling); - -public readonly record struct PlanetaryGpuPageMetadata(float4 Address, float4 Layout, float4 Bounds, float4 State); - -public readonly record struct PlanetaryGpuPageContent( - PlanetaryPageLayout Layout, - PlanetaryGpuPageInput[] Inputs, - float SampleSpacing, - float ParentSampleSpacing); - -public static class PlanetaryGpuPageBuilder -{ - public static PlanetaryGpuPageContent BuildContent(PlanetaryPageLayout layout, float radius) - { - layout.Validate(); - if (!float.IsFinite(radius) || radius <= 0) throw new ArgumentOutOfRangeException(nameof(radius)); - var spacing = PlanetaryPageSampling.NominalAngularTexelSize(layout) * radius; - var parentSpacing = layout.Tile.Level == 0 - ? 0 - : PlanetaryPageSampling.NominalAngularTexelSize(new(layout.Tile.Parent(), layout.InteriorSize, layout.BorderSize)) * radius; - var inputs = new PlanetaryGpuPageInput[layout.StorageSize * layout.StorageSize]; - for (var y = 0; y < layout.StorageSize; y++) - for (var x = 0; x < layout.StorageSize; x++) - { - var direction = PlanetaryPageSampling.Direction(layout, x, y); - inputs[y * layout.StorageSize + x] = new( - new float4(direction, radius), - new float4(spacing, parentSpacing, 0, 0)); - } - return new(layout, inputs, spacing, parentSpacing); - } - - public static PlanetaryGpuPageMetadata Metadata( - in PlanetaryGpuPageContent content, - int outputOffset, - float blend, - float4 bounds = default) - { - content.Layout.Validate(); - if (content.Inputs == null || content.Inputs.Length != content.Layout.StorageSize * content.Layout.StorageSize) - throw new ArgumentException("GPU page content does not match its layout.", nameof(content)); - if (outputOffset < 0) throw new ArgumentOutOfRangeException(nameof(outputOffset)); - if (!float.IsFinite(blend) || blend is < 0 or > 1) throw new ArgumentOutOfRangeException(nameof(blend)); - var tile = content.Layout.Tile; - return new( - new float4((int)tile.Face, tile.Level, tile.X, tile.Y), - new float4(outputOffset, content.Layout.StorageSize, content.Layout.InteriorSize, content.Layout.BorderSize), - bounds, - new float4(1, blend, PlanetaryPageSampling.NominalAngularTexelSize(content.Layout), content.SampleSpacing)); - } -} diff --git a/src/GameCult.Geometry/Planetary/PlanetaryMapTileEncoding.cs b/src/GameCult.Geometry/Planetary/PlanetaryMapTileEncoding.cs deleted file mode 100644 index ed07e7ba..00000000 --- a/src/GameCult.Geometry/Planetary/PlanetaryMapTileEncoding.cs +++ /dev/null @@ -1,122 +0,0 @@ -using System; -using System.IO; -using System.Text; -using CultMath; - -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this file, -// You can obtain one at https://mozilla.org/MPL/2.0/. - -namespace GameCult.Geometry; - -public static class PlanetaryMapTileEncoding -{ - public const uint FormatVersion = 1; - private static readonly byte[] Magic = Encoding.ASCII.GetBytes("CMPT"); - - public static void Write(Stream stream, in PlanetarySurfaceMapTile tile, bool leaveOpen = false) - { - if (stream == null || !stream.CanWrite) throw new ArgumentException("A writable stream is required.", nameof(stream)); - tile.Key.Validate(); - if (tile.Samples == null || tile.Validity == null || tile.Samples.Length != tile.Validity.Length) - throw new ArgumentException("Tile samples and validity must have equal lengths.", nameof(tile)); - var expectedCount = tile.Key.Layout.StorageSize * tile.Key.Layout.StorageSize; - if (tile.Samples.Length != expectedCount) throw new ArgumentException("Tile payload does not match its layout.", nameof(tile)); - - using var writer = new BinaryWriter(stream, Encoding.UTF8, leaveOpen); - writer.Write(Magic); - writer.Write(FormatVersion); - writer.Write(tile.Key.FieldVersion); - writer.Write(tile.Key.ProjectionVersion); - writer.Write(tile.Key.LayerId); - var projection = tile.Key.Layout.Projection; - writer.Write((int)projection.Kind); - writer.Write(projection.CenterLongitude); - writer.Write(projection.CenterLatitude); - writer.Write(projection.Scale); - writer.Write(tile.Key.Layout.Level); - writer.Write(tile.Key.Layout.X); - writer.Write(tile.Key.Layout.Y); - writer.Write(tile.Key.Layout.InteriorSize); - writer.Write(tile.Key.Layout.BorderSize); - writer.Write(tile.Key.QueryScale.FootprintMeters); - writer.Write(tile.Key.QueryScale.MaximumUnresolvedHeight); - writer.Write(tile.Samples.Length); - for (var i = 0; i < tile.Samples.Length; i++) - { - writer.Write(tile.Validity[i]); - if (!tile.Validity[i]) continue; - WriteSample(writer, tile.Samples[i], tile.Key.FieldVersion); - } - } - - public static PlanetarySurfaceMapTile Read(Stream stream, bool leaveOpen = false) - { - if (stream == null || !stream.CanRead) throw new ArgumentException("A readable stream is required.", nameof(stream)); - using var reader = new BinaryReader(stream, Encoding.UTF8, leaveOpen); - var magic = reader.ReadBytes(Magic.Length); - if (magic.Length != Magic.Length || !magic.AsSpan().SequenceEqual(Magic)) throw new InvalidDataException("Not a GameCult.Geometry planetary tile."); - var version = reader.ReadUInt32(); - if (version != FormatVersion) throw new InvalidDataException($"Unsupported planetary tile version {version}."); - var fieldVersion = reader.ReadUInt64(); - var projectionVersion = reader.ReadUInt32(); - var layerId = reader.ReadUInt32(); - var kindValue = reader.ReadInt32(); - if (!Enum.IsDefined(typeof(PlanetaryProjectionKind), kindValue)) throw new InvalidDataException("Unknown planetary projection kind."); - var projection = new PlanetaryProjectionParameters( - (PlanetaryProjectionKind)kindValue, - reader.ReadDouble(), reader.ReadDouble(), reader.ReadDouble()).Validate(); - var layout = new PlanetaryMapTileLayout( - projection, - reader.ReadInt32(), reader.ReadInt32(), reader.ReadInt32(), - reader.ReadInt32(), reader.ReadInt32()).Validate(); - var scale = new PlanetaryQueryScale(reader.ReadSingle(), reader.ReadSingle()).Validate(); - var key = new PlanetaryMapTileKey(fieldVersion, projectionVersion, layerId, layout, scale).Validate(); - var count = reader.ReadInt32(); - if (count != layout.StorageSize * layout.StorageSize) throw new InvalidDataException("Planetary tile sample count does not match layout."); - var samples = new PlanetarySurfaceSample[count]; - var validity = new bool[count]; - for (var i = 0; i < count; i++) - { - validity[i] = reader.ReadBoolean(); - if (validity[i]) samples[i] = ReadSample(reader, fieldVersion); - } - if (stream.CanSeek && stream.Position != stream.Length) throw new InvalidDataException("Planetary tile has trailing bytes."); - return new(key, samples, validity); - } - - private static void WriteSample(BinaryWriter writer, PlanetarySurfaceSample sample, ulong fieldVersion) - { - if (sample.FieldVersion != fieldVersion) throw new ArgumentException("Sample field version does not match tile identity."); - Write(writer, sample.UnitDirection); - writer.Write(sample.Radius); - writer.Write(sample.RadialDisplacement); - Write(writer, sample.TangentGradient); - Write(writer, sample.SurfaceNormal); - writer.Write(sample.Slope); - writer.Write(sample.Ridge); - writer.Write(sample.Gully); - writer.Write(sample.FinestResolvedWavelength); - writer.Write(sample.UnresolvedHeightBound); - } - - private static PlanetarySurfaceSample ReadSample(BinaryReader reader, ulong fieldVersion) - { - var direction = ReadFloat3(reader); - var radius = reader.ReadSingle(); - var displacement = reader.ReadSingle(); - var gradient = ReadFloat3(reader); - var normal = ReadFloat3(reader); - return new( - fieldVersion, direction, radius, displacement, gradient, normal, - reader.ReadSingle(), reader.ReadSingle(), reader.ReadSingle(), - reader.ReadSingle(), reader.ReadSingle()); - } - - private static void Write(BinaryWriter writer, float3 value) - { - writer.Write(value.x); writer.Write(value.y); writer.Write(value.z); - } - - private static float3 ReadFloat3(BinaryReader reader) => new(reader.ReadSingle(), reader.ReadSingle(), reader.ReadSingle()); -} diff --git a/src/GameCult.Geometry/Planetary/PlanetaryPages.cs b/src/GameCult.Geometry/Planetary/PlanetaryPages.cs deleted file mode 100644 index 919a6320..00000000 --- a/src/GameCult.Geometry/Planetary/PlanetaryPages.cs +++ /dev/null @@ -1,192 +0,0 @@ -using System; -using CultMath; - -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this file, -// You can obtain one at https://mozilla.org/MPL/2.0/. - -namespace GameCult.Geometry; - -public readonly record struct PlanetaryPageLayout(PlanetaryTileAddress Tile, int InteriorSize, int BorderSize) -{ - public int StorageSize => checked(InteriorSize + BorderSize * 2); - - public PlanetaryPageLayout Validate() - { - if (InteriorSize < 2) throw new ArgumentOutOfRangeException(nameof(InteriorSize)); - if (BorderSize < 0 || BorderSize >= InteriorSize) throw new ArgumentOutOfRangeException(nameof(BorderSize)); - return this; - } -} - -public readonly record struct PlanetaryPageSummary( - float MinimumHeight, - float MaximumHeight, - float MaximumSlope, - float UnresolvedHeightBound, - float AngularTexelSize); - -public readonly record struct PlanetaryPageSample(float Height, float3 TangentGradient, float Ridge, float Gully); - -public static class PlanetaryPageSampling -{ - public static float3 Direction(PlanetaryPageLayout page, int storageX, int storageY) - { - page.Validate(); - if ((uint)storageX >= (uint)page.StorageSize || (uint)storageY >= (uint)page.StorageSize) - throw new ArgumentOutOfRangeException(nameof(storageX)); - var localU = (storageX - page.BorderSize) / (double)(page.InteriorSize - 1); - var localV = (storageY - page.BorderSize) / (double)(page.InteriorSize - 1); - return DirectionAtLocal(page, localU, localV); - } - - public static float3 DirectionAtLocal(PlanetaryPageLayout page, double localU, double localV) - { - page.Validate(); - if (!double.IsFinite(localU) || !double.IsFinite(localV)) throw new ArgumentOutOfRangeException(nameof(localU)); - var count = page.Tile.AxisTileCount; - return PlanetaryTopology.Direction(new( - page.Tile.Face, - -1.0 + 2.0 * (page.Tile.X + localU) / count, - -1.0 + 2.0 * (page.Tile.Y + localV) / count)); - } - - public static float AngularTexelSize(PlanetaryPageLayout page) - { - var center = page.BorderSize + (page.InteriorSize - 1) / 2; - var a = Direction(page, center, center); - var b = Direction(page, Math.Min(center + 1, page.StorageSize - 1), center); - return MathF.Acos(math.clamp(math.dot(a, b), -1.0f, 1.0f)); - } - - public static float NominalAngularTexelSize(PlanetaryPageLayout page) - { - page.Validate(); - return MathF.PI * 0.5f / (page.Tile.AxisTileCount * (page.InteriorSize - 1)); - } - - public static PlanetaryPageSample Bilinear( - ReadOnlySpan samples, - PlanetaryPageLayout page, - float2 local) - { - page.Validate(); - if (samples.Length != page.StorageSize * page.StorageSize) throw new ArgumentException("Page sample count does not match layout.", nameof(samples)); - var texel = local * (page.InteriorSize - 1) + page.BorderSize; - var x0 = Math.Clamp((int)MathF.Floor(texel.x), 0, page.StorageSize - 1); - var y0 = Math.Clamp((int)MathF.Floor(texel.y), 0, page.StorageSize - 1); - var x1 = Math.Min(x0 + 1, page.StorageSize - 1); - var y1 = Math.Min(y0 + 1, page.StorageSize - 1); - var tx = math.frac(texel.x); - var ty = math.frac(texel.y); - var a = Lerp(samples[y0 * page.StorageSize + x0], samples[y0 * page.StorageSize + x1], tx); - var b = Lerp(samples[y1 * page.StorageSize + x0], samples[y1 * page.StorageSize + x1], tx); - return Lerp(a, b, ty); - } - - public static PlanetaryPageSummary Summarize(ReadOnlySpan samples, PlanetaryPageLayout page, float unresolvedHeightBound) - { - page.Validate(); - if (samples.Length != page.StorageSize * page.StorageSize) throw new ArgumentException("Page sample count does not match layout.", nameof(samples)); - var min = float.PositiveInfinity; - var max = float.NegativeInfinity; - var maxSlope = 0.0f; - foreach (var sample in samples) - { - min = MathF.Min(min, sample.Height); - max = MathF.Max(max, sample.Height); - maxSlope = MathF.Max(maxSlope, math.length(sample.TangentGradient)); - } - return new(min, max, maxSlope, MathF.Max(unresolvedHeightBound, 0), AngularTexelSize(page)); - } - - private static PlanetaryPageSample Lerp(PlanetaryPageSample a, PlanetaryPageSample b, float t) => new( - math.lerp(a.Height, b.Height, t), - math.lerp(a.TangentGradient, b.TangentGradient, t), - math.lerp(a.Ridge, b.Ridge, t), - math.lerp(a.Gully, b.Gully, t)); -} - -public readonly record struct PlanetaryBakedPage( - PlanetaryPageLayout Layout, - PlanetaryPageSample[] Samples, - PlanetaryPageSummary Summary); - -public static class PlanetaryPageBaker -{ - public static PlanetaryBakedPage Bake( - in PlanetaryFieldDefinition field, - PlanetaryPageLayout layout, - TSource source) - where TSource : IPlanetaryBaseField - { - layout.Validate(); field.Validate(); - var spacing = PlanetaryPageSampling.NominalAngularTexelSize(layout) * field.Radius; - var parentSpacing = layout.Tile.Level == 0 - ? 0 - : PlanetaryPageSampling.NominalAngularTexelSize(new(layout.Tile.Parent(), layout.InteriorSize, layout.BorderSize)) * field.Radius; - var samples = new PlanetaryPageSample[layout.StorageSize * layout.StorageSize]; - var unresolved = 0.0f; - for (var y = 0; y < layout.StorageSize; y++) - for (var x = 0; x < layout.StorageSize; x++) - { - var direction = PlanetaryPageSampling.Direction(layout, x, y); - var baseSample = source.Sample(direction); - var child = PlanetaryField.Sample(field, direction, baseSample, PlanetaryQueryScale.AtFootprint(spacing)); - var height = child.RadialDisplacement; - var gradient = child.TangentGradient; - if (parentSpacing > 0) - { - var parent = PlanetaryField.Sample(field, direction, baseSample, PlanetaryQueryScale.AtFootprint(parentSpacing)); - height -= parent.RadialDisplacement; - gradient -= parent.TangentGradient; - } - samples[y * layout.StorageSize + x] = new(height, gradient, child.Ridge, child.Gully); - unresolved = MathF.Max(unresolved, child.UnresolvedHeightBound); - } - return new(layout, samples, PlanetaryPageSampling.Summarize(samples, layout, unresolved)); - } -} - -public readonly record struct PlanetaryResidentPage(PlanetaryBakedPage Page, float Blend) -{ - public PlanetaryResidentPage Validate() - { - if (!float.IsFinite(Blend) || Blend is < 0 or > 1) throw new ArgumentOutOfRangeException(nameof(Blend)); - Page.Layout.Validate(); - if (Page.Samples.Length != Page.Layout.StorageSize * Page.Layout.StorageSize) throw new ArgumentException("Page payload does not match layout."); - return this; - } -} - -public static class PlanetaryPageSetSampling -{ - public static bool TrySample(ReadOnlySpan pages, float3 direction, out PlanetaryPageSample sample) - { - PlanetaryTopology.ValidateDirection(direction); - var height = 0.0f; - var gradient = float3.zero; - var ridge = 0.0f; - var gully = 0.0f; - var found = false; - var deepestLevel = -1; - foreach (var resident in pages) - { - resident.Validate(); - if (!PlanetaryTopology.TryLocalCoordinate(direction, resident.Page.Layout.Tile, out var local)) continue; - var contribution = PlanetaryPageSampling.Bilinear(resident.Page.Samples, resident.Page.Layout, local); - var blend = math.saturate(resident.Blend); - height += contribution.Height * blend; - gradient += contribution.TangentGradient * blend; - if (resident.Page.Layout.Tile.Level >= deepestLevel) - { - ridge = found ? math.lerp(ridge, contribution.Ridge, blend) : contribution.Ridge; - gully = found ? math.lerp(gully, contribution.Gully, blend) : contribution.Gully; - deepestLevel = resident.Page.Layout.Tile.Level; - } - found = true; - } - sample = new(height, gradient, ridge, gully); - return found; - } -} diff --git a/src/GameCult.Geometry/Planetary/PlanetaryPatch.cs b/src/GameCult.Geometry/Planetary/PlanetaryPatch.cs deleted file mode 100644 index 76a3fe92..00000000 --- a/src/GameCult.Geometry/Planetary/PlanetaryPatch.cs +++ /dev/null @@ -1,50 +0,0 @@ -using System; -using System.Linq; -using CultMath; - -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this file, -// You can obtain one at https://mozilla.org/MPL/2.0/. - -namespace GameCult.Geometry; - -public readonly record struct PlanetaryPatchVertex(float2 LocalCoordinate, float3 UnitDirection); - -public readonly record struct PlanetaryPatchMesh( - PlanetaryCubeFace Face, - int CellsPerAxis, - PlanetaryPatchVertex[] Vertices, - int[] Indices); - -public static class PlanetaryPatch -{ - public static PlanetaryPatchMesh CreateFace(PlanetaryCubeFace face, int cellsPerAxis) - { - if (!Enum.IsDefined(typeof(PlanetaryCubeFace), face)) throw new ArgumentOutOfRangeException(nameof(face)); - if (cellsPerAxis < 1 || cellsPerAxis > 4096) throw new ArgumentOutOfRangeException(nameof(cellsPerAxis)); - var axisVertices = cellsPerAxis + 1; - var vertices = new PlanetaryPatchVertex[checked(axisVertices * axisVertices)]; - for (var y = 0; y < axisVertices; y++) - for (var x = 0; x < axisVertices; x++) - { - var local = new float2(x / (float)cellsPerAxis, y / (float)cellsPerAxis); - vertices[y * axisVertices + x] = new(local, PlanetaryTopology.Direction(new(face, local.x * 2 - 1, local.y * 2 - 1))); - } - var indices = new int[checked(cellsPerAxis * cellsPerAxis * 6)]; - var write = 0; - for (var y = 0; y < cellsPerAxis; y++) - for (var x = 0; x < cellsPerAxis; x++) - { - var lowerLeft = y * axisVertices + x; - var lowerRight = lowerLeft + 1; - var upperLeft = lowerLeft + axisVertices; - var upperRight = upperLeft + 1; - indices[write++] = lowerLeft; indices[write++] = lowerRight; indices[write++] = upperRight; - indices[write++] = lowerLeft; indices[write++] = upperRight; indices[write++] = upperLeft; - } - return new(face, cellsPerAxis, vertices, indices); - } - - public static PlanetaryPatchMesh[] CreateCubeSphere(int cellsPerAxis) - => ((PlanetaryCubeFace[])Enum.GetValues(typeof(PlanetaryCubeFace))).Select(face => CreateFace(face, cellsPerAxis)).ToArray(); -} diff --git a/src/GameCult.Geometry/Planetary/PlanetaryProjection.cs b/src/GameCult.Geometry/Planetary/PlanetaryProjection.cs deleted file mode 100644 index 5851d367..00000000 --- a/src/GameCult.Geometry/Planetary/PlanetaryProjection.cs +++ /dev/null @@ -1,327 +0,0 @@ -using System; -using CultMath; - -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this file, -// You can obtain one at https://mozilla.org/MPL/2.0/. - -namespace GameCult.Geometry; - -public enum PlanetaryProjectionKind -{ - Equirectangular, - WebMercator, - EqualEarth, - Orthographic, - AzimuthalEquidistant, - AzimuthalEqualArea, - CubeAtlas, - LocalTangent, -} - -public readonly record struct PlanetaryProjectionParameters( - PlanetaryProjectionKind Kind, - double CenterLongitude = 0, - double CenterLatitude = 0, - double Scale = 1) -{ - public PlanetaryProjectionParameters Validate() - { - if (!double.IsFinite(CenterLongitude) || !double.IsFinite(CenterLatitude) || !double.IsFinite(Scale) || Scale <= 0) - throw new ArgumentOutOfRangeException(nameof(Scale)); - if (CenterLatitude is < -Math.PI / 2 or > Math.PI / 2) throw new ArgumentOutOfRangeException(nameof(CenterLatitude)); - return this; - } -} - -public static class PlanetaryProjection -{ - private const double EqualEarthA1 = 1.340264; - private const double EqualEarthA2 = -0.081106; - private const double EqualEarthA3 = 0.000893; - private const double EqualEarthA4 = 0.003796; - private static readonly double EqualEarthM = Math.Sqrt(3) / 2; - private static readonly double EqualEarthXMax = Math.PI / (EqualEarthM * EqualEarthA1); - private static readonly double EqualEarthYMax = EqualEarthY(Math.Asin(EqualEarthM)); - public static readonly double WebMercatorMaximumLatitude = Math.Atan(Math.Sinh(Math.PI)); - - public static bool TryForward(float3 direction, PlanetaryProjectionParameters parameters, out double2 coordinate) - { - parameters.Validate(); - PlanetaryTopology.ValidateDirection(direction); - var d = math.normalize(direction); - if (parameters.Kind == PlanetaryProjectionKind.CubeAtlas) - { - var face = PlanetaryTopology.FaceCoordinate(d); - var index = (int)face.Face; - var column = index % 3; - var row = index / 3; - coordinate = new( - (-1 + (column + (face.U + 1) * 0.5) * (2.0 / 3.0)) / parameters.Scale, - (1 - (row + (face.V + 1) * 0.5)) / parameters.Scale); - return true; - } - - var longitude = Math.Atan2(d.y, d.x); - var latitude = Math.Asin(Math.Clamp(d.z, -1, 1)); - var deltaLongitude = WrapLongitude(longitude - parameters.CenterLongitude); - var centered = CenteredForward(deltaLongitude, latitude, parameters, out coordinate); - coordinate /= parameters.Scale; - return centered; - } - - public static bool TryInverse(double2 coordinate, PlanetaryProjectionParameters parameters, out float3 direction) - { - parameters.Validate(); - if (!double.IsFinite(coordinate.x) || !double.IsFinite(coordinate.y)) - { - direction = float3.zero; - return false; - } - coordinate *= parameters.Scale; - if (parameters.Kind == PlanetaryProjectionKind.CubeAtlas) - return TryInverseCubeAtlas(coordinate, out direction); - - if (!CenteredInverse(coordinate, parameters, out var deltaLongitude, out var latitude)) - { - direction = float3.zero; - return false; - } - var longitude = WrapLongitude(deltaLongitude + parameters.CenterLongitude); - var cosLatitude = Math.Cos(latitude); - direction = math.normalize(new float3( - (float)(cosLatitude * Math.Cos(longitude)), - (float)(cosLatitude * Math.Sin(longitude)), - (float)Math.Sin(latitude))); - return true; - } - - private static bool CenteredForward(double longitude, double latitude, PlanetaryProjectionParameters p, out double2 coordinate) - { - switch (p.Kind) - { - case PlanetaryProjectionKind.Equirectangular: - coordinate = new(longitude / Math.PI, latitude / (Math.PI * 0.5)); - return true; - case PlanetaryProjectionKind.WebMercator: - if (Math.Abs(latitude) > WebMercatorMaximumLatitude) { coordinate = double2.zero; return false; } - coordinate = new(longitude / Math.PI, Math.Asinh(Math.Tan(latitude)) / Math.PI); - return true; - case PlanetaryProjectionKind.EqualEarth: - { - var theta = Math.Asin(EqualEarthM * Math.Sin(latitude)); - var theta2 = theta * theta; - var theta6 = theta2 * theta2 * theta2; - var denominator = EqualEarthM * (EqualEarthA1 + 3 * EqualEarthA2 * theta2 + theta6 * (7 * EqualEarthA3 + 9 * EqualEarthA4 * theta2)); - coordinate = new(longitude * Math.Cos(theta) / denominator / EqualEarthXMax, EqualEarthY(theta) / EqualEarthYMax); - return true; - } - case PlanetaryProjectionKind.Orthographic: - case PlanetaryProjectionKind.AzimuthalEquidistant: - case PlanetaryProjectionKind.AzimuthalEqualArea: - case PlanetaryProjectionKind.LocalTangent: - return ForwardAzimuthal(longitude, latitude, p, out coordinate); - default: - throw new ArgumentOutOfRangeException(nameof(p)); - } - } - - private static bool CenteredInverse(double2 coordinate, PlanetaryProjectionParameters p, out double longitude, out double latitude) - { - longitude = latitude = 0; - switch (p.Kind) - { - case PlanetaryProjectionKind.Equirectangular: - if (Math.Abs(coordinate.y) > 1) return false; - longitude = coordinate.x * Math.PI; latitude = coordinate.y * Math.PI * 0.5; return Math.Abs(coordinate.x) <= 1; - case PlanetaryProjectionKind.WebMercator: - if (Math.Abs(coordinate.x) > 1 || Math.Abs(coordinate.y) > 1) return false; - longitude = coordinate.x * Math.PI; latitude = Math.Atan(Math.Sinh(coordinate.y * Math.PI)); return true; - case PlanetaryProjectionKind.EqualEarth: - { - if (Math.Abs(coordinate.x) > 1 || Math.Abs(coordinate.y) > 1) return false; - var targetY = coordinate.y * EqualEarthYMax; - var theta = targetY / EqualEarthA1; - for (var i = 0; i < 8; i++) - { - var theta2 = theta * theta; - var theta6 = theta2 * theta2 * theta2; - var f = EqualEarthY(theta) - targetY; - var derivative = EqualEarthA1 + 3 * EqualEarthA2 * theta2 + theta6 * (7 * EqualEarthA3 + 9 * EqualEarthA4 * theta2); - theta -= f / derivative; - } - var sinLatitude = Math.Sin(theta) / EqualEarthM; - if (Math.Abs(sinLatitude) > 1.0000000001) return false; - latitude = Math.Asin(Math.Clamp(sinLatitude, -1, 1)); - var thetaSquared = theta * theta; - var thetaSixth = thetaSquared * thetaSquared * thetaSquared; - var denominator = EqualEarthM * (EqualEarthA1 + 3 * EqualEarthA2 * thetaSquared + thetaSixth * (7 * EqualEarthA3 + 9 * EqualEarthA4 * thetaSquared)); - longitude = coordinate.x * EqualEarthXMax * denominator / Math.Cos(theta); - return Math.Abs(longitude) <= Math.PI + 1.0e-10; - } - case PlanetaryProjectionKind.Orthographic: - case PlanetaryProjectionKind.AzimuthalEquidistant: - case PlanetaryProjectionKind.AzimuthalEqualArea: - case PlanetaryProjectionKind.LocalTangent: - return InverseAzimuthal(coordinate, p, out longitude, out latitude); - default: - throw new ArgumentOutOfRangeException(nameof(p)); - } - } - - private static bool ForwardAzimuthal(double longitude, double latitude, PlanetaryProjectionParameters p, out double2 coordinate) - { - var sin0 = Math.Sin(p.CenterLatitude); var cos0 = Math.Cos(p.CenterLatitude); - var sin = Math.Sin(latitude); var cos = Math.Cos(latitude); var cosLon = Math.Cos(longitude); - var cosC = Math.Clamp(sin0 * sin + cos0 * cos * cosLon, -1, 1); - var rawX = cos * Math.Sin(longitude); - var rawY = cos0 * sin - sin0 * cos * cosLon; - double factor; - switch (p.Kind) - { - case PlanetaryProjectionKind.Orthographic: - if (cosC < 0) { coordinate = double2.zero; return false; } - factor = 1; break; - case PlanetaryProjectionKind.AzimuthalEquidistant: - { - var c = Math.Acos(cosC); factor = c < 1.0e-12 ? 1 : c / Math.Sin(c); factor /= Math.PI; break; - } - case PlanetaryProjectionKind.AzimuthalEqualArea: - factor = Math.Sqrt(2 / Math.Max(1 + cosC, 1.0e-20)) * 0.5; break; - case PlanetaryProjectionKind.LocalTangent: - if (cosC <= 0) { coordinate = double2.zero; return false; } - factor = 1 / cosC; break; - default: throw new ArgumentOutOfRangeException(nameof(p)); - } - coordinate = new(rawX * factor, rawY * factor); - return true; - } - - private static bool InverseAzimuthal(double2 coordinate, PlanetaryProjectionParameters p, out double longitude, out double latitude) - { - longitude = latitude = 0; - var rho = Math.Sqrt(coordinate.x * coordinate.x + coordinate.y * coordinate.y); - double c; - switch (p.Kind) - { - case PlanetaryProjectionKind.Orthographic: if (rho > 1) return false; c = Math.Asin(Math.Clamp(rho, 0, 1)); break; - case PlanetaryProjectionKind.AzimuthalEquidistant: if (rho > 1) return false; c = rho * Math.PI; break; - case PlanetaryProjectionKind.AzimuthalEqualArea: if (rho > 1) return false; c = 2 * Math.Asin(Math.Clamp(rho, 0, 1)); break; - case PlanetaryProjectionKind.LocalTangent: c = Math.Atan(rho); break; - default: throw new ArgumentOutOfRangeException(nameof(p)); - } - if (rho < 1.0e-12) { latitude = p.CenterLatitude; return true; } - var sinC = Math.Sin(c); var cosC = Math.Cos(c); - var sin0 = Math.Sin(p.CenterLatitude); var cos0 = Math.Cos(p.CenterLatitude); - latitude = Math.Asin(Math.Clamp(cosC * sin0 + coordinate.y * sinC * cos0 / rho, -1, 1)); - longitude = Math.Atan2(coordinate.x * sinC, rho * cos0 * cosC - coordinate.y * sin0 * sinC); - return true; - } - - private static bool TryInverseCubeAtlas(double2 coordinate, out float3 direction) - { - direction = float3.zero; - if (coordinate.x is < -1 or > 1 || coordinate.y is < -1 or > 1) return false; - var column = Math.Min((int)((coordinate.x + 1) * 1.5), 2); - var row = Math.Min((int)((1 - coordinate.y)), 1); - var u = ((coordinate.x + 1) * 1.5 - column) * 2 - 1; - var v = ((1 - coordinate.y) - row) * 2 - 1; - direction = PlanetaryTopology.Direction(new((PlanetaryCubeFace)(row * 3 + column), u, v)); - return true; - } - - private static double EqualEarthY(double theta) - { - var theta2 = theta * theta; - var theta6 = theta2 * theta2 * theta2; - return theta * (EqualEarthA1 + EqualEarthA2 * theta2 + theta6 * (EqualEarthA3 + EqualEarthA4 * theta2)); - } - - private static double WrapLongitude(double longitude) - { - longitude %= Math.PI * 2; - if (longitude > Math.PI) longitude -= Math.PI * 2; - if (longitude < -Math.PI) longitude += Math.PI * 2; - return longitude; - } -} - -public readonly record struct PlanetaryMapTileLayout( - PlanetaryProjectionParameters Projection, - int Level, - int X, - int Y, - int InteriorSize, - int BorderSize) -{ - public int StorageSize => checked(InteriorSize + BorderSize * 2); - - public PlanetaryMapTileLayout Validate() - { - Projection.Validate(); - if (Level is < 0 or > 30) throw new ArgumentOutOfRangeException(nameof(Level)); - var count = 1 << Level; - if ((uint)X >= (uint)count || (uint)Y >= (uint)count) throw new ArgumentOutOfRangeException(nameof(X)); - if (InteriorSize < 2 || BorderSize < 0 || BorderSize >= InteriorSize) throw new ArgumentOutOfRangeException(nameof(InteriorSize)); - return this; - } -} - -public static class PlanetaryMapTileSampling -{ - public static bool TryDirection(PlanetaryMapTileLayout tile, int storageX, int storageY, out float3 direction) - { - tile.Validate(); - if ((uint)storageX >= (uint)tile.StorageSize || (uint)storageY >= (uint)tile.StorageSize) throw new ArgumentOutOfRangeException(nameof(storageX)); - var count = 1 << tile.Level; - var localU = (storageX - tile.BorderSize) / (double)(tile.InteriorSize - 1); - var localV = (storageY - tile.BorderSize) / (double)(tile.InteriorSize - 1); - var map = new double2(-1 + 2 * (tile.X + localU) / count, 1 - 2 * (tile.Y + localV) / count); - return PlanetaryProjection.TryInverse(map, tile.Projection, out direction); - } -} - -public readonly record struct PlanetaryMapTileKey( - ulong FieldVersion, - uint ProjectionVersion, - uint LayerId, - PlanetaryMapTileLayout Layout, - PlanetaryQueryScale QueryScale) -{ - public PlanetaryMapTileKey Validate() - { - if (FieldVersion == 0 || ProjectionVersion == 0) throw new ArgumentOutOfRangeException(nameof(FieldVersion)); - Layout.Validate(); QueryScale.Validate(); - return this; - } -} - -public readonly record struct PlanetarySurfaceMapTile( - PlanetaryMapTileKey Key, - PlanetarySurfaceSample[] Samples, - bool[] Validity); - -public static class PlanetaryMapTileBaker -{ - public static PlanetarySurfaceMapTile Bake( - in PlanetaryFieldDefinition field, - PlanetaryMapTileKey key, - TSource source) - where TSource : IPlanetaryBaseField - { - field.Validate(); key.Validate(); - if (field.FieldVersion != key.FieldVersion) throw new ArgumentException("Tile field version does not match definition.", nameof(key)); - var count = key.Layout.StorageSize * key.Layout.StorageSize; - var samples = new PlanetarySurfaceSample[count]; - var validity = new bool[count]; - for (var y = 0; y < key.Layout.StorageSize; y++) - for (var x = 0; x < key.Layout.StorageSize; x++) - { - var index = y * key.Layout.StorageSize + x; - if (!PlanetaryMapTileSampling.TryDirection(key.Layout, x, y, out var direction)) continue; - samples[index] = PlanetaryField.Sample(field, direction, source.Sample(direction), key.QueryScale); - validity[index] = true; - } - return new(key, samples, validity); - } -} diff --git a/src/GameCult.Geometry/Planetary/PlanetaryQueries.cs b/src/GameCult.Geometry/Planetary/PlanetaryQueries.cs deleted file mode 100644 index b50b7785..00000000 --- a/src/GameCult.Geometry/Planetary/PlanetaryQueries.cs +++ /dev/null @@ -1,198 +0,0 @@ -using System; -using CultMath; - -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this file, -// You can obtain one at https://mozilla.org/MPL/2.0/. - -namespace GameCult.Geometry; - -public readonly record struct PlanetaryRadialBounds(float MinimumDisplacement, float MaximumDisplacement) -{ - public PlanetaryRadialBounds Validate() - { - if (!float.IsFinite(MinimumDisplacement) || !float.IsFinite(MaximumDisplacement) || MinimumDisplacement > MaximumDisplacement) - throw new ArgumentOutOfRangeException(nameof(MinimumDisplacement)); - return this; - } -} - -public readonly record struct PlanetaryRayHit( - double Distance, - double3 PlanetLocalPosition, - PlanetarySurfaceSample Surface); - -public readonly record struct PlanetaryPathSample(double Distance, PlanetarySurfaceSample Surface); - -public readonly record struct PlanetaryRegionSummary( - float MinimumDisplacement, - float MaximumDisplacement, - float MaximumSlope, - float MaximumUnresolvedHeight); - -public static class PlanetaryQueries -{ - public static bool TryIntersectRay( - in PlanetaryFieldDefinition field, - double3 rayOrigin, - double3 rayDirection, - double maximumDistance, - in PlanetaryQueryScale scale, - in PlanetaryRadialBounds bounds, - TSource source, - out PlanetaryRayHit hit) - where TSource : IPlanetaryBaseField - { - field.Validate(); scale.Validate(); bounds.Validate(); - ValidateFinite(rayOrigin, nameof(rayOrigin)); ValidateFinite(rayDirection, nameof(rayDirection)); - if (!double.IsFinite(maximumDistance) || maximumDistance <= 0) throw new ArgumentOutOfRangeException(nameof(maximumDistance)); - var directionLength = Length(rayDirection); - if (directionLength < 1.0e-15) throw new ArgumentOutOfRangeException(nameof(rayDirection)); - var direction = rayDirection / directionLength; - var outerRadius = field.Radius + bounds.MaximumDisplacement + scale.MaximumUnresolvedHeight; - if (!double.IsFinite(outerRadius)) outerRadius = field.Radius + bounds.MaximumDisplacement; - if (!TrySphereInterval(rayOrigin, direction, outerRadius, out var enter, out var leave)) { hit = default; return false; } - var start = Math.Max(enter, 0); - var end = Math.Min(leave, maximumDistance); - if (start > end) { hit = default; return false; } - - var previousDistance = start; - var previous = SignedSurfaceDistance(field, rayOrigin + direction * previousDistance, scale, source, out var previousSample); - if (previous <= 0) - { - hit = new(previousDistance, rayOrigin + direction * previousDistance, previousSample); - return true; - } - var minimumStep = Math.Max(scale.FootprintMeters * 0.5, field.Radius * 1.0e-7); - for (var iteration = 0; iteration < 512 && previousDistance < end; iteration++) - { - var nextDistance = Math.Min(previousDistance + Math.Max(previous * 0.5, minimumStep), end); - var next = SignedSurfaceDistance(field, rayOrigin + direction * nextDistance, scale, source, out var nextSample); - if (next <= 0) - { - var low = previousDistance; - var high = nextDistance; - var surface = nextSample; - for (var refine = 0; refine < 32; refine++) - { - var middle = (low + high) * 0.5; - var value = SignedSurfaceDistance(field, rayOrigin + direction * middle, scale, source, out var middleSample); - if (value > 0) low = middle; - else { high = middle; surface = middleSample; } - } - var position = rayOrigin + direction * high; - hit = new(high, position, surface); - return true; - } - if (nextDistance >= end) break; - previousDistance = nextDistance; - previous = next; - previousSample = nextSample; - } - hit = default; - return false; - } - - public static double SegmentClearance( - in PlanetaryFieldDefinition field, - double3 start, - double3 end, - double sampleSpacing, - in PlanetaryQueryScale scale, - TSource source) - where TSource : IPlanetaryBaseField - { - ValidateFinite(start, nameof(start)); ValidateFinite(end, nameof(end)); - if (!double.IsFinite(sampleSpacing) || sampleSpacing <= 0) throw new ArgumentOutOfRangeException(nameof(sampleSpacing)); - var delta = end - start; - var distance = Length(delta); - var steps = Math.Max((int)Math.Ceiling(distance / sampleSpacing), 1); - var minimum = double.PositiveInfinity; - for (var i = 0; i <= steps; i++) - { - var position = start + delta * (i / (double)steps); - minimum = Math.Min(minimum, SignedSurfaceDistance(field, position, scale, source, out _)); - } - return minimum; - } - - public static PlanetaryPathSample[] SampleGreatCircle( - in PlanetaryFieldDefinition field, - float3 startDirection, - float3 endDirection, - int segmentCount, - in PlanetaryQueryScale scale, - TSource source) - where TSource : IPlanetaryBaseField - { - if (segmentCount < 1) throw new ArgumentOutOfRangeException(nameof(segmentCount)); - PlanetaryTopology.ValidateDirection(startDirection); PlanetaryTopology.ValidateDirection(endDirection); - var a = math.normalize(startDirection); var b = math.normalize(endDirection); - var angle = MathF.Acos(math.clamp(math.dot(a, b), -1, 1)); - var sinAngle = MathF.Sin(angle); - var result = new PlanetaryPathSample[segmentCount + 1]; - for (var i = 0; i <= segmentCount; i++) - { - var t = i / (float)segmentCount; - var direction = sinAngle < 1.0e-6f - ? math.normalize(math.lerp(a, b, t)) - : math.normalize(a * (MathF.Sin((1 - t) * angle) / sinAngle) + b * (MathF.Sin(t * angle) / sinAngle)); - var sample = PlanetaryField.Sample(field, direction, source.Sample(direction), scale); - result[i] = new(angle * field.Radius * t, sample); - } - return result; - } - - public static PlanetaryRegionSummary Summarize( - in PlanetaryFieldDefinition field, - ReadOnlySpan directions, - in PlanetaryQueryScale scale, - TSource source) - where TSource : IPlanetaryBaseField - { - if (directions.IsEmpty) throw new ArgumentException("At least one direction is required.", nameof(directions)); - var min = float.PositiveInfinity; var max = float.NegativeInfinity; var slope = 0.0f; var unresolved = 0.0f; - foreach (var direction in directions) - { - var normalized = math.normalize(direction); - var sample = PlanetaryField.Sample(field, normalized, source.Sample(normalized), scale); - min = MathF.Min(min, sample.RadialDisplacement); - max = MathF.Max(max, sample.RadialDisplacement); - slope = MathF.Max(slope, sample.Slope); - unresolved = MathF.Max(unresolved, sample.UnresolvedHeightBound); - } - return new(min, max, slope, unresolved); - } - - private static double SignedSurfaceDistance( - in PlanetaryFieldDefinition field, - double3 position, - in PlanetaryQueryScale scale, - TSource source, - out PlanetarySurfaceSample sample) - where TSource : IPlanetaryBaseField - { - var radius = Length(position); - if (radius < 1.0e-15) { sample = default; return double.NegativeInfinity; } - var direction = new float3((float)(position.x / radius), (float)(position.y / radius), (float)(position.z / radius)); - sample = PlanetaryField.Sample(field, direction, source.Sample(direction), scale); - return radius - sample.SurfaceRadius; - } - - private static bool TrySphereInterval(double3 origin, double3 direction, double radius, out double enter, out double leave) - { - var b = Dot(origin, direction); - var c = Dot(origin, origin) - radius * radius; - var discriminant = b * b - c; - if (discriminant < 0) { enter = leave = 0; return false; } - var root = Math.Sqrt(discriminant); - enter = -b - root; leave = -b + root; return leave >= 0; - } - - private static double Dot(double3 a, double3 b) => a.x * b.x + a.y * b.y + a.z * b.z; - private static double Length(double3 value) => Math.Sqrt(Dot(value, value)); - private static void ValidateFinite(double3 value, string name) - { - if (!double.IsFinite(value.x) || !double.IsFinite(value.y) || !double.IsFinite(value.z)) throw new ArgumentOutOfRangeException(name); - } -} diff --git a/src/GameCult.Geometry/Planetary/PlanetaryRadialRefinement.cs b/src/GameCult.Geometry/Planetary/PlanetaryRadialRefinement.cs deleted file mode 100644 index a958bc1a..00000000 --- a/src/GameCult.Geometry/Planetary/PlanetaryRadialRefinement.cs +++ /dev/null @@ -1,34 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this file, -// You can obtain one at https://mozilla.org/MPL/2.0/. - -using CultMath; - -namespace GameCult.Geometry; - -public static class PlanetaryRadialRefinement -{ - public static float3 Step( - float3 position, - float3 rayDirection, - float3 center, - float targetRadius, - float minimumDerivative = 0.2f, - float maximumCorrection = 0.08f) - { - var local = position - center; - var radius = math.length(local); - var radial = local / math.max(radius, 0.0001f); - var error = radius - targetRadius; - var derivative = math.dot(rayDirection, radial); - var safeMinimum = math.max(minimumDerivative, 0.0001f); - var safeDerivative = math.abs(derivative) > safeMinimum - ? derivative - : derivative < 0.0f ? -safeMinimum : safeMinimum; - var correction = math.clamp( - -error / safeDerivative, - -math.abs(maximumCorrection), - math.abs(maximumCorrection)); - return position + rayDirection * correction; - } -} diff --git a/src/GameCult.Geometry/Planetary/PlanetaryResidency.cs b/src/GameCult.Geometry/Planetary/PlanetaryResidency.cs deleted file mode 100644 index 79acc317..00000000 --- a/src/GameCult.Geometry/Planetary/PlanetaryResidency.cs +++ /dev/null @@ -1,153 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using CultMath; - -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this file, -// You can obtain one at https://mozilla.org/MPL/2.0/. - -namespace GameCult.Geometry; - -public readonly record struct PlanetaryLodParameters( - int MaximumLevel, - int PageInteriorSize, - int PageBorderSize, - float ViewportHeightPixels, - float MinimumFootprintMeters) -{ - public PlanetaryLodParameters Validate() - { - if (MaximumLevel is < 0 or > PlanetaryTileAddress.MaxLevel) throw new ArgumentOutOfRangeException(nameof(MaximumLevel)); - _ = new PlanetaryPageLayout(new(PlanetaryCubeFace.PositiveX, 0, 0, 0), PageInteriorSize, PageBorderSize).Validate(); - if (!float.IsFinite(ViewportHeightPixels) || ViewportHeightPixels <= 0) throw new ArgumentOutOfRangeException(nameof(ViewportHeightPixels)); - if (!float.IsFinite(MinimumFootprintMeters) || MinimumFootprintMeters < 0) throw new ArgumentOutOfRangeException(nameof(MinimumFootprintMeters)); - return this; - } -} - -public static class PlanetaryLodSelector -{ - public static int SelectLevel( - in PlanetaryFieldDefinition field, - float cameraDistanceFromCenter, - in PlanetaryLodParameters parameters) - { - field.Validate(); parameters.Validate(); - if (!float.IsFinite(cameraDistanceFromCenter) || cameraDistanceFromCenter < 0) throw new ArgumentOutOfRangeException(nameof(cameraDistanceFromCenter)); - var altitude = MathF.Max(cameraDistanceFromCenter - field.Radius, 0.001f); - var footprint = MathF.Max(altitude / parameters.ViewportHeightPixels * 2, parameters.MinimumFootprintMeters); - var erosion = field.Erosion; - for (var level = 0; level <= parameters.MaximumLevel; level++) - { - var page = new PlanetaryPageLayout(new(PlanetaryCubeFace.PositiveX, level, 0, 0), parameters.PageInteriorSize, parameters.PageBorderSize); - var spacing = PlanetaryPageSampling.NominalAngularTexelSize(page) * field.Radius; - var unresolved = ErosionFrequencyBands.Select( - erosion.Scale * erosion.CellScale, spacing, erosion.Octaves, - erosion.Lacunarity, erosion.Strength * erosion.Scale, erosion.Gain).UnresolvedHeightBound; - if (unresolved <= footprint) return level; - } - return parameters.MaximumLevel; - } - - public static PlanetaryTileAddress[] SelectAncestorChain( - in PlanetaryFieldDefinition field, - float3 cameraDirection, - float cameraDistanceFromCenter, - in PlanetaryLodParameters parameters) - { - PlanetaryTopology.ValidateDirection(cameraDirection); - var level = SelectLevel(field, cameraDistanceFromCenter, parameters); - var result = new PlanetaryTileAddress[6 + level]; - for (var face = 0; face < 6; face++) result[face] = new((PlanetaryCubeFace)face, 0, 0, 0); - var leaf = PlanetaryTopology.TileAt(cameraDirection, level); - for (var currentLevel = level; currentLevel >= 1; currentLevel--) - { - result[6 + currentLevel - 1] = leaf; - leaf = leaf.Parent(); - } - return result; - } -} - -public readonly record struct PlanetaryResidentTile(PlanetaryTileAddress Tile, float Blend, bool Departing); - -public readonly record struct PlanetaryResidencySnapshot( - ulong ContentVersion, - ulong PresentationVersion, - PlanetaryResidentTile[] Tiles); - -/// Owns presentation residency only. It never owns field content. -public sealed class PlanetaryResidualResidency -{ - private readonly Dictionary residents = new(); - private readonly Dictionary arrivals = new(); - private readonly Dictionary departures = new(); - private float lastTime = float.NegativeInfinity; - - public void Reset() - { - residents.Clear(); arrivals.Clear(); departures.Clear(); lastTime = float.NegativeInfinity; - } - - public PlanetaryResidencySnapshot Update(ReadOnlySpan desired, float timeSeconds, float transitionSeconds) - { - if (!float.IsFinite(timeSeconds)) throw new ArgumentOutOfRangeException(nameof(timeSeconds)); - if (!float.IsFinite(transitionSeconds) || transitionSeconds < 0) throw new ArgumentOutOfRangeException(nameof(transitionSeconds)); - if (timeSeconds < lastTime) Reset(); - lastTime = timeSeconds; - - var desiredKeys = new HashSet(); - foreach (var tile in desired) - { - var key = tile.StableKey; - desiredKeys.Add(key); - residents[key] = tile; - if (!arrivals.ContainsKey(key)) arrivals[key] = timeSeconds; - departures.Remove(key); - } - - foreach (var pair in residents.ToArray()) - { - if (pair.Value.Level == 0 || desiredKeys.Contains(pair.Key)) continue; - if (!departures.ContainsKey(pair.Key)) - { - var startingBlend = transitionSeconds == 0 - ? 1 - : Math.Clamp((timeSeconds - arrivals[pair.Key]) / transitionSeconds, 0, 1); - departures[pair.Key] = (timeSeconds, startingBlend); - } - if (transitionSeconds > 0 && timeSeconds - departures[pair.Key].Time < transitionSeconds) continue; - residents.Remove(pair.Key); arrivals.Remove(pair.Key); departures.Remove(pair.Key); - } - - var ordered = residents - .OrderBy(pair => pair.Value.Level) - .ThenBy(pair => pair.Value.Face) - .ThenBy(pair => pair.Value.X) - .ThenBy(pair => pair.Value.Y) - .ToArray(); - var tiles = new PlanetaryResidentTile[ordered.Length]; - for (var i = 0; i < ordered.Length; i++) - { - var pair = ordered[i]; - var departing = departures.TryGetValue(pair.Key, out var departure); - var blend = pair.Value.Level == 0 || transitionSeconds == 0 - ? 1 - : departing - ? departure.StartingBlend * Math.Clamp(1 - (timeSeconds - departure.Time) / transitionSeconds, 0, 1) - : Math.Clamp((timeSeconds - arrivals[pair.Key]) / transitionSeconds, 0, 1); - tiles[i] = new(pair.Value, blend, departing); - } - return new(Hash(ordered.Select(pair => pair.Key)), Hash(tiles.Select(tile => tile.Tile.StableKey ^ (uint)BitConverter.SingleToInt32Bits(tile.Blend))), tiles); - } - - private static ulong Hash(IEnumerable values) - { - const ulong offset = 14695981039346656037UL; - const ulong prime = 1099511628211UL; - var hash = offset; - foreach (var value in values) hash = unchecked((hash ^ value) * prime); - return hash == 0 ? 1 : hash; - } -} diff --git a/src/GameCult.Geometry/Planetary/PlanetaryTopology.cs b/src/GameCult.Geometry/Planetary/PlanetaryTopology.cs deleted file mode 100644 index 18ff4f31..00000000 --- a/src/GameCult.Geometry/Planetary/PlanetaryTopology.cs +++ /dev/null @@ -1,196 +0,0 @@ -using System; -using CultMath; - -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this file, -// You can obtain one at https://mozilla.org/MPL/2.0/. - -namespace GameCult.Geometry; - -public enum PlanetaryCubeFace -{ - PositiveX, - NegativeX, - PositiveY, - NegativeY, - PositiveZ, - NegativeZ, -} - -public readonly record struct PlanetaryFaceCoordinate(PlanetaryCubeFace Face, double U, double V) -{ - public PlanetaryFaceCoordinate Validate(bool allowOutsideFace = false) - { - if (!double.IsFinite(U) || !double.IsFinite(V)) throw new ArgumentOutOfRangeException(nameof(U)); - if (!allowOutsideFace && (U is < -1.0 or > 1.0 || V is < -1.0 or > 1.0)) - throw new ArgumentOutOfRangeException(nameof(U), "Face coordinates must be in [-1, 1]."); - if (!Enum.IsDefined(typeof(PlanetaryCubeFace), Face)) throw new ArgumentOutOfRangeException(nameof(Face)); - return this; - } -} - -public readonly record struct PlanetaryTileAddress -{ - public const int MaxLevel = 30; - - public PlanetaryTileAddress(PlanetaryCubeFace face, int level, int x, int y) - { - if (!Enum.IsDefined(typeof(PlanetaryCubeFace), face)) throw new ArgumentOutOfRangeException(nameof(face)); - if (level is < 0 or > MaxLevel) throw new ArgumentOutOfRangeException(nameof(level)); - var count = 1 << level; - if ((uint)x >= (uint)count) throw new ArgumentOutOfRangeException(nameof(x)); - if ((uint)y >= (uint)count) throw new ArgumentOutOfRangeException(nameof(y)); - Face = face; - Level = level; - X = x; - Y = y; - } - - public PlanetaryCubeFace Face { get; } - public int Level { get; } - public int X { get; } - public int Y { get; } - public int AxisTileCount => 1 << Level; - - public PlanetaryTileAddress Parent() => Level == 0 - ? throw new InvalidOperationException("Root cube tile has no parent.") - : new(Face, Level - 1, X >> 1, Y >> 1); - - public PlanetaryTileAddress Child(int childX, int childY) - { - if ((uint)childX > 1) throw new ArgumentOutOfRangeException(nameof(childX)); - if ((uint)childY > 1) throw new ArgumentOutOfRangeException(nameof(childY)); - if (Level == MaxLevel) throw new InvalidOperationException($"Level {MaxLevel} cannot be subdivided."); - return new(Face, Level + 1, (X << 1) | childX, (Y << 1) | childY); - } - - public PlanetaryFaceCoordinate PositionAt(double localU, double localV) - { - if (!double.IsFinite(localU) || localU is < 0.0 or > 1.0) throw new ArgumentOutOfRangeException(nameof(localU)); - if (!double.IsFinite(localV) || localV is < 0.0 or > 1.0) throw new ArgumentOutOfRangeException(nameof(localV)); - var count = AxisTileCount; - return new(Face, -1.0 + 2.0 * (X + localU) / count, -1.0 + 2.0 * (Y + localV) / count); - } - - public ulong StableKey - { - get - { - const ulong offset = 14695981039346656037UL; - const ulong prime = 1099511628211UL; - var hash = offset; - hash = unchecked((hash ^ (uint)Face) * prime); - hash = unchecked((hash ^ (uint)Level) * prime); - hash = unchecked((hash ^ (uint)X) * prime); - hash = unchecked((hash ^ (uint)Y) * prime); - return hash == 0 ? 1 : hash; - } - } - public override string ToString() => $"{Face}/L{Level:D2}/{X}/{Y}"; -} - -public static class PlanetaryTopology -{ - private const float QuarterPi = MathF.PI * 0.25f; - - public static float3 Direction(PlanetaryFaceCoordinate coordinate) - { - coordinate.Validate(allowOutsideFace: true); - var u = MathF.Tan((float)coordinate.U * QuarterPi); - var v = MathF.Tan((float)coordinate.V * QuarterPi); - var cube = coordinate.Face switch - { - PlanetaryCubeFace.PositiveX => new float3(1, v, -u), - PlanetaryCubeFace.NegativeX => new float3(-1, v, u), - PlanetaryCubeFace.PositiveY => new float3(u, 1, -v), - PlanetaryCubeFace.NegativeY => new float3(u, -1, v), - PlanetaryCubeFace.PositiveZ => new float3(u, v, 1), - PlanetaryCubeFace.NegativeZ => new float3(-u, v, -1), - _ => throw new ArgumentOutOfRangeException(nameof(coordinate)), - }; - return math.normalize(cube); - } - - public static float3 NormalizedCubeDirection(PlanetaryFaceCoordinate coordinate) - { - coordinate.Validate(allowOutsideFace: true); - var u = (float)coordinate.U; - var v = (float)coordinate.V; - return math.normalize(CubeVector(coordinate.Face, u, v)); - } - - public static PlanetaryFaceCoordinate FaceCoordinate(float3 direction) - { - ValidateDirection(direction); - var d = math.normalize(direction); - var a = math.abs(d); - if (a.x >= a.y && a.x >= a.z) - return d.x >= 0 - ? FromCubeUv(PlanetaryCubeFace.PositiveX, -d.z / a.x, d.y / a.x) - : FromCubeUv(PlanetaryCubeFace.NegativeX, d.z / a.x, d.y / a.x); - if (a.y >= a.z) - return d.y >= 0 - ? FromCubeUv(PlanetaryCubeFace.PositiveY, d.x / a.y, -d.z / a.y) - : FromCubeUv(PlanetaryCubeFace.NegativeY, d.x / a.y, d.z / a.y); - return d.z >= 0 - ? FromCubeUv(PlanetaryCubeFace.PositiveZ, d.x / a.z, d.y / a.z) - : FromCubeUv(PlanetaryCubeFace.NegativeZ, -d.x / a.z, d.y / a.z); - } - - public static PlanetaryTileAddress TileAt(float3 direction, int level) - { - if (level is < 0 or > PlanetaryTileAddress.MaxLevel) throw new ArgumentOutOfRangeException(nameof(level)); - var face = FaceCoordinate(direction); - var count = 1 << level; - var x = Math.Clamp((int)((face.U * 0.5 + 0.5) * count), 0, count - 1); - var y = Math.Clamp((int)((face.V * 0.5 + 0.5) * count), 0, count - 1); - return new(face.Face, level, x, y); - } - - public static bool TryLocalCoordinate(float3 direction, PlanetaryTileAddress tile, out float2 local) - { - var face = FaceCoordinate(direction); - if (face.Face != tile.Face) { local = float2.zero; return false; } - var scaledU = (face.U * 0.5 + 0.5) * tile.AxisTileCount; - var scaledV = (face.V * 0.5 + 0.5) * tile.AxisTileCount; - local = new((float)(scaledU - tile.X), (float)(scaledV - tile.Y)); - return local.x is >= 0 and <= 1 && local.y is >= 0 and <= 1; - } - - public static float3 SurfaceNormal(float3 unitDirection, float3 worldDistanceTangentGradient) - { - ValidateDirection(unitDirection); - ValidateFinite(worldDistanceTangentGradient, nameof(worldDistanceTangentGradient)); - var direction = math.normalize(unitDirection); - var tangent = worldDistanceTangentGradient - direction * math.dot(worldDistanceTangentGradient, direction); - return math.normalize(direction - tangent); - } - - private static PlanetaryFaceCoordinate FromCubeUv(PlanetaryCubeFace face, float cubeU, float cubeV) => new( - face, - MathF.Atan(cubeU) / QuarterPi, - MathF.Atan(cubeV) / QuarterPi); - - private static float3 CubeVector(PlanetaryCubeFace face, float u, float v) => face switch - { - PlanetaryCubeFace.PositiveX => new float3(1, v, -u), - PlanetaryCubeFace.NegativeX => new float3(-1, v, u), - PlanetaryCubeFace.PositiveY => new float3(u, 1, -v), - PlanetaryCubeFace.NegativeY => new float3(u, -1, v), - PlanetaryCubeFace.PositiveZ => new float3(u, v, 1), - PlanetaryCubeFace.NegativeZ => new float3(-u, v, -1), - _ => throw new ArgumentOutOfRangeException(nameof(face)), - }; - - internal static void ValidateDirection(float3 direction) - { - ValidateFinite(direction, nameof(direction)); - if (math.length(direction) < 1.0e-12f) throw new ArgumentOutOfRangeException(nameof(direction)); - } - - internal static void ValidateFinite(float3 value, string name) - { - if (!float.IsFinite(value.x) || !float.IsFinite(value.y) || !float.IsFinite(value.z)) - throw new ArgumentOutOfRangeException(name); - } -} diff --git a/src/GameCult.Geometry/Planetary/SphericalErosion.cs b/src/GameCult.Geometry/Planetary/SphericalErosion.cs deleted file mode 100644 index b20e0f00..00000000 --- a/src/GameCult.Geometry/Planetary/SphericalErosion.cs +++ /dev/null @@ -1,114 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this file, -// You can obtain one at https://mozilla.org/MPL/2.0/. - -using CultMath; - -namespace GameCult.Geometry; - -public readonly record struct SphericalErosionParameters( - float Frequency, - float Amplitude, - float Lacunarity, - float Gain, - float SlopeStrength, - float Detail, - float Seed, - int Octaves) -{ - public static SphericalErosionParameters Default => new(18.0f, 0.035f, 2.03f, 0.52f, 6.0f, 1.35f, 0.0f, 5); -} - -public readonly record struct SphericalErosionSample(float HeightOffset, float Ridge, float Gully, float3 Gradient); - -/// -/// Stateless, point-evaluable erosion-like synthesis on a unit sphere. -/// The matching shader body is gamecult_geometry_spherical_erosion in GameCult.Geometry.hlsl. -/// Position and gradient are planet-space; the gradient is projected tangent -/// internally so cube faces and tile request order never enter the result. -/// -public static class SphericalErosion -{ - public static SphericalErosionSample Sample( - float3 unitPosition, - float baseHeight, - float3 baseGradient, - SphericalErosionParameters parameters) - { - var position = math.normalize(unitPosition); - var gradient = baseGradient - position * math.dot(baseGradient, position); - var slope = math.length(gradient); - var flow = slope > 1.0e-6f ? -gradient / slope : StableTangent(position); - var frequency = math.max(parameters.Frequency, 0.001f); - var amplitude = parameters.Amplitude; - var combiMask = EaseOut(math.saturate(slope * parameters.SlopeStrength)); - var fadeTarget = math.saturate(baseHeight * 0.5f + 0.5f) * 2.0f - 1.0f; - var heightOffset = 0.0f; - var ridge = 0.0f; - var gully = 0.0f; - - var octaveCount = math.min(math.max(parameters.Octaves, 0), 8); - for (var octave = 0; octave < octaveCount; octave++) - { - var phase = DirectionalCellWave(position * frequency, flow, parameters.Seed + octave * 19.19f); - var wave = math.cos(phase); - var waveSlope = math.sin(phase); - var faded = math.lerp(fadeTarget, wave, combiMask); - heightOffset += faded * amplitude; - ridge = math.max(ridge, math.saturate(faded)); - gully = math.max(gully, math.saturate(-faded)); - - var across = math.normalize(math.cross(position, flow)); - gradient += across * (math.sign(waveSlope) * amplitude * frequency * combiMask); - gradient -= position * math.dot(gradient, position); - slope = math.length(gradient); - flow = slope > 1.0e-6f ? -gradient / slope : StableTangent(position); - - var newMask = EaseOut(math.saturate(math.abs(waveSlope) * 1.35f)); - combiMask = PowInverse(combiMask, parameters.Detail) * newMask; - fadeTarget = faded; - frequency *= math.max(parameters.Lacunarity, 1.01f); - amplitude *= math.saturate(parameters.Gain); - } - - return new SphericalErosionSample(heightOffset, ridge, gully, gradient); - } - - private static float DirectionalCellWave(float3 position, float3 flow, float seed) - { - var cell = math.floor(position); - var local = math.frac(position); - var weights = local * local * (3.0f - 2.0f * local); - var x00 = CellWave(position, flow, cell + new float3(0, 0, 0), seed); - var x10 = CellWave(position, flow, cell + new float3(1, 0, 0), seed); - var x01 = CellWave(position, flow, cell + new float3(0, 1, 0), seed); - var x11 = CellWave(position, flow, cell + new float3(1, 1, 0), seed); - var y00 = math.lerp(x00, x10, weights.x); - var y10 = math.lerp(x01, x11, weights.x); - var z0 = math.lerp(y00, y10, weights.y); - var x00z = CellWave(position, flow, cell + new float3(0, 0, 1), seed); - var x10z = CellWave(position, flow, cell + new float3(1, 0, 1), seed); - var x01z = CellWave(position, flow, cell + new float3(0, 1, 1), seed); - var x11z = CellWave(position, flow, cell + new float3(1, 1, 1), seed); - var z1 = math.lerp(math.lerp(x00z, x10z, weights.x), math.lerp(x01z, x11z, weights.x), weights.y); - return math.lerp(z0, z1, weights.z); - } - - private static float CellWave(float3 position, float3 flow, float3 cell, float seed) - { - var pivot = cell + new float3( - math.hash(cell + seed + 11.7f), - math.hash(cell.yzx + seed + 37.1f), - math.hash(cell.zyx + seed + 73.9f)); - return math.dot(position - pivot, flow) * math.TAU + math.hash(cell + seed + 101.3f) * math.TAU; - } - - private static float3 StableTangent(float3 position) - { - var axis = math.abs(position.z) < 0.8f ? new float3(0, 0, 1) : new float3(0, 1, 0); - return math.normalize(math.cross(axis, position)); - } - - private static float EaseOut(float value) => 1.0f - (1.0f - value) * (1.0f - value); - private static float PowInverse(float value, float power) => 1.0f - math.pow(1.0f - math.saturate(value), math.max(power, 0.001f)); -} diff --git a/src/GameCult.Geometry/README.md b/src/GameCult.Geometry/README.md index 42c681cb..2d12dc25 100644 --- a/src/GameCult.Geometry/README.md +++ b/src/GameCult.Geometry/README.md @@ -1,8 +1,21 @@ # GameCult.Geometry -`GameCult.Geometry` owns spatial values, planetary fields and topology, -residual-page residency, map projections and tiles, geometry queries, and the -CultCache-native documents that coordinate geometry production. +The project is known as **CultGeometry**. `GameCult.Geometry` is its clean .NET +package and namespace identity. + +`GameCult.Geometry` owns reusable procedural geometry algorithms together with +the CultCache-native documents that carry their domains, build requests, +selected cuts, and mesh chunk artifacts. + +It is the schema authority for the shared `CultGeometry*` records. +`GameCult.Geometry.Csg` owns CSG production behavior and proves wire parity +against those records; it does not independently evolve their identities. + +Its first engine-neutral algorithm is `CultGeometryIsoSurface.Extract`, a +marching-tetrahedra extractor over regular scalar grids. It uses CultMath for +numeric primitives and emits `CultGeometryTriangleMesh`; Godot, Unity, native +workers, and other consumers adapt that neutral result into their own runtime +objects. See `ISOSURFACE-PROVENANCE.md` for the clean-port boundary. The package is deliberately substrate-shaped: @@ -20,8 +33,8 @@ The package is deliberately substrate-shaped: The small value layer is for cross-runtime query contracts, physics probes, native slices, and UI surfaces: -- `CultMath.float2` and `CultMath.float3`: whole-vector values for positions, - velocities, accelerations, and query inputs. SoA layouts may store them efficiently, but +- `CultVec2` and `CultVec3`: whole-vector values for positions, velocities, + accelerations, and query inputs. SoA layouts may store them efficiently, but the semantic API does not force callers to split x/y/z into unrelated fields. - `CultRect`: canonical XY viewport/query rectangle stored as min/max corners. - `CultCircle`: 2D influence/query brush with rect intersection helpers. @@ -33,10 +46,10 @@ runtime equivalents instead of bespoke `{ minX, minY, maxX, maxY }` payloads. ## Runtime Path -Geometry workers emit domain documents, selected-cut manifests, and chunk -artifacts as MessagePack CultCache documents. A Unity runtime, local worker, or -remote geometry process receives the same records through CultMesh/CultNet and -reads them back as `CultGeometry*` types. +CultGeometry CSG (`GameCult.Geometry.Csg`) emits domain documents, selected-cut +manifests, and chunk artifacts as MessagePack CultCache documents. A Godot +runtime, local worker, or remote geometry process receives the same records +through CultMesh/CultNet and reads them back as `CultGeometry*` types. ```csharp using GameCult.Caching; @@ -44,13 +57,17 @@ using GameCult.Geometry; using GameCult.Mesh; using GameCult.Networking; -var geometryDocuments = new CultNetDocumentRegistry(CultDocumentRegistry.Shared); +var geometryDocuments = new CultNetDocumentRegistry(CultDocumentRegistry.Shared) + .Register(CultNetDocumentBinding.ForDocument()) + .Register(CultNetDocumentBinding.ForDocument()) + .Register(CultNetDocumentBinding.ForDocument()) + .Register(CultNetDocumentBinding.ForDocument()); using var node = await CultMesh.CreateNodeAsync("ragnarok-geometry.ccmp", new CultMeshNodeOptions { DatabaseOptions = new CultNetDatabaseOptions { - RuntimeId = "unity-runtime", + RuntimeId = "godot-runtime", DocumentRegistry = geometryDocuments } }); @@ -60,33 +77,20 @@ using var chunks = node.Database .Subscribe(change => ApplyChunk(change.Document)); ``` -The generated CultDocument metadata owns geometry registration. Consumers pass -the shared registry into CultNet; they do not maintain a second manual binding -list that can drift when the schema cohort changes. - The record key is part of the contract. Rust and C# must compute the same key for the same geometry record, so stable fingerprints use the Rust canonical -field order and exact IEEE-754 component bits. If this drifts, the same chunk becomes two +field order and invariant float text. If this drifts, the same chunk becomes two different chunks depending on which runtime touched it first. That is not a cache miss; that is split-brain geometry. ## Documents -- `CultMath.float2`, `CultMath.float3`, `CultRect`, `CultCircle`, and - `CultSphere`: shared value primitives for query and view contracts. +- `CultVec2`, `CultVec3`, `CultRect`, `CultCircle`, `CultSphere`: shared value + primitives for query and view contracts. - `CultGeometryDomainDocument`: one hierarchical domain tree, suitable for - hierarchical feature DSL output. + Ragnarok/Fensalir-style feature DSL output. - `CultGeometryBuildRequest`: one LOD/frustum/budget request for workers. - `CultGeometrySelectedCutManifest`: the selected domain cut, deferred children, fallback rows, and diagnostic contribution rows. - `CultGeometryChunkArtifact`: render mesh, optional collider mesh, source domain/claim identities, build counters, and transition clipping seed. - -## Planetary geometry - -The planetary field, cube-sphere topology, pages, residency, projections, -queries, and versioned `CMPT` map tiles are described in -[`docs/planetary.md`](docs/planetary.md). Shader consumers include -`Shaders/GameCult.Geometry.hlsl`; Unity consumers use the -`org.gamecult.geometry` package, and browser tile consumers use -`packages/gamecult-geometry-ts`. diff --git a/src/GameCult.Geometry/Shaders/AdvancedErosionFilter.hlsl b/src/GameCult.Geometry/Shaders/AdvancedErosionFilter.hlsl deleted file mode 100644 index 219afc86..00000000 --- a/src/GameCult.Geometry/Shaders/AdvancedErosionFilter.hlsl +++ /dev/null @@ -1,147 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this file, -// You can obtain one at https://mozilla.org/MPL/2.0/. -// Mathematical procedure adapted from Rune Skovbo Johansen's Advanced Terrain -// Erosion Filter (2025): https://www.shadertoy.com/view/wXcfWn - -#ifndef GAMECULT_GEOMETRY_ADVANCED_EROSION_FILTER_HLSL -#define GAMECULT_GEOMETRY_ADVANCED_EROSION_FILTER_HLSL - -struct GameCultGeometryAdvancedErosionParameters -{ - float scale; float strength; float gully_weight; float detail; - float4 rounding; float4 onset; float2 assumed_slope; - float cell_scale; float normalization; int octaves; float lacunarity; float gain; -}; - -struct GameCultGeometryAdvancedErosionResult -{ - float3 delta; float magnitude; float ridge_map; float fade_target; -}; - -struct GameCultGeometryErosionBandSelection -{ - int active_octaves; float final_octave_weight; float finest_included_wavelength; float unresolved_height_bound; -}; - -GameCultGeometryErosionBandSelection gamecult_geometry_select_erosion_bands(float base_wavelength, float sample_spacing, int maximum_octaves, float lacunarity, float base_amplitude, float gain, float transition_ratio) -{ - base_wavelength=max(base_wavelength,1.0e-6); sample_spacing=max(sample_spacing,0.0); lacunarity=max(lacunarity,1.01); transition_ratio=max(transition_ratio,1.01); maximum_octaves=min(max(maximum_octaves,0),16); - float nyquist=sample_spacing*2.0, wavelength=base_wavelength, final_weight=0.0, finest=0.0; int active=0; - [loop] for(int octave=0; octave=active) omitted+=amplitude; else if(index==active-1) omitted+=amplitude*(1.0-final_weight); amplitude*=abs(gain); - } - GameCultGeometryErosionBandSelection result; result.active_octaves=active; result.final_octave_weight=active>0?final_weight:0.0; result.finest_included_wavelength=finest; result.unresolved_height_bound=omitted; return result; -} - -float gamecult_geometry_erosion_hash_unit(int2 cell, uint salt) -{ - uint value = (uint)cell.x * 0x8da6b343u ^ (uint)cell.y * 0xd8163841u ^ salt; - value ^= value >> 16; value *= 0x7feb352du; value ^= value >> 15; value *= 0x846ca68bu; value ^= value >> 16; - return (value & 0x00ffffffu) * (1.0 / 16777216.0); -} -float2 gamecult_geometry_erosion_hash2(float2 input) -{ - int2 cell = (int2)input; - return float2(gamecult_geometry_erosion_hash_unit(cell, 0x68bc21ebu), gamecult_geometry_erosion_hash_unit(cell, 0x02e5be93u)) * 2.0 - 1.0; -} - -float4 gamecult_geometry_phacelle_noise(float2 position, float2 normal_direction, float frequency, float offset_cycles, float normalization) -{ - float2 side_direction = float2(-normal_direction.y, normal_direction.x) * frequency * 6.28318530718; - float offset = offset_cycles * 6.28318530718; - float2 cell = floor(position); - float2 local = frac(position); - float2 phase = 0.0; - float weight_sum = 0.0; - [unroll] for (int x = -1; x <= 2; x++) - [unroll] for (int y = -1; y <= 2; y++) - { - float2 grid_offset = float2(x, y); - float2 grid_point = cell + grid_offset; - float2 random_offset = gamecult_geometry_erosion_hash2(grid_point) * 0.5; - float2 delta = local - grid_offset - random_offset; - float weight = max(0.0, exp(-dot(delta, delta) * 2.0) - 0.01111); - weight_sum += weight; - float wave = dot(delta, side_direction) + offset; - phase += float2(cos(wave), sin(wave)) * weight; - } - float2 interpolated = phase / max(weight_sum, 1.0e-20); - float magnitude = max(1.0 - saturate(normalization), length(interpolated)); - return float4(interpolated / max(magnitude, 1.0e-20), side_direction); -} - -float gamecult_geometry_erosion_pow_inverse(float value, float power) { return 1.0 - pow(1.0 - saturate(value), power); } -float gamecult_geometry_erosion_ease_out(float value) { float inverse = 1.0 - saturate(value); return 1.0 - inverse * inverse; } -float gamecult_geometry_erosion_smooth_start(float value, float smoothing) -{ - return smoothing <= 1.0e-10 || value >= smoothing ? value - 0.5 * smoothing : 0.5 * value * value / smoothing; -} -float2 gamecult_geometry_erosion_safe_normalize(float2 value) -{ - float value_length = length(value); - return abs(value_length) > 1.0e-10 ? value / value_length : value; -} - -GameCultGeometryAdvancedErosionResult gamecult_geometry_advanced_erosion_filter_banded(float2 position, float3 base_height_and_slope, float fade_target, GameCultGeometryAdvancedErosionParameters p, GameCultGeometryErosionBandSelection band) -{ - float strength = p.strength * p.scale; - fade_target = clamp(fade_target, -1.0, 1.0); - float3 input_value = base_height_and_slope; - float3 height_and_slope = input_value; - float frequency = 1.0 / max(p.scale * p.cell_scale, 1.0e-10); - float2 slope = height_and_slope.yz; - float slope_length = max(length(slope), 1.0e-10); - float magnitude = 0.0; - float rounding_multiplier = 1.0; - float input_rounding = lerp(p.rounding.y, p.rounding.x, saturate(fade_target + 0.5)) * p.rounding.z; - float combined_mask = gamecult_geometry_erosion_ease_out(gamecult_geometry_erosion_smooth_start(slope_length * p.onset.x, input_rounding * p.onset.x)); - float ridge_mask = gamecult_geometry_erosion_ease_out(slope_length * p.onset.z); - float ridge_fade = fade_target; - float2 gully_slope = lerp(slope, slope / slope_length * p.assumed_slope.x, p.assumed_slope.y); - - int octave_count=min(min(max(p.octaves,0),8),max(band.active_octaves,0)); - [loop] for (int octave = 0; octave < octave_count; octave++) - { - float4 phacelle = gamecult_geometry_phacelle_noise(position * frequency, gamecult_geometry_erosion_safe_normalize(gully_slope), p.cell_scale, 0.25, p.normalization); - float2 derivative_direction = phacelle.zw * -frequency; - float sloping = abs(phacelle.y); - gully_slope += sign(phacelle.y) * derivative_direction * strength * p.gully_weight; - float3 gullies = float3(phacelle.x, phacelle.y * derivative_direction.x, phacelle.y * derivative_direction.y); - float3 faded = lerp(float3(fade_target, 0.0, 0.0), gullies * p.gully_weight, combined_mask); - float octave_weight=octave==octave_count-1?saturate(band.final_octave_weight):1.0; - height_and_slope += faded * strength * octave_weight; - magnitude += strength * octave_weight; - fade_target = faded.x; - float octave_rounding = lerp(p.rounding.y, p.rounding.x, saturate(phacelle.x + 0.5)) * rounding_multiplier; - float new_mask = gamecult_geometry_erosion_ease_out(gamecult_geometry_erosion_smooth_start(sloping * p.onset.y, octave_rounding * p.onset.y)); - combined_mask = gamecult_geometry_erosion_pow_inverse(combined_mask, p.detail) * new_mask; - ridge_fade = lerp(ridge_fade, lerp(ridge_fade, gullies.x, ridge_mask), octave_weight); - ridge_mask *= lerp(1.0, gamecult_geometry_erosion_ease_out(sloping * p.onset.w), octave_weight); - strength *= p.gain; - frequency *= p.lacunarity; - rounding_multiplier *= p.rounding.w; - } - - GameCultGeometryAdvancedErosionResult result; - result.delta = height_and_slope - input_value; - result.magnitude = magnitude; - result.ridge_map = ridge_fade * (1.0 - ridge_mask); - result.fade_target = fade_target; - return result; -} - -GameCultGeometryAdvancedErosionResult gamecult_geometry_advanced_erosion_filter(float2 position, float3 base_height_and_slope, float fade_target, GameCultGeometryAdvancedErosionParameters p) -{ - GameCultGeometryErosionBandSelection band; band.active_octaves=p.octaves; band.final_octave_weight=1.0; band.finest_included_wavelength=0.0; band.unresolved_height_bound=0.0; - return gamecult_geometry_advanced_erosion_filter_banded(position, base_height_and_slope, fade_target, p, band); -} - -#endif diff --git a/src/GameCult.Geometry/Shaders/GameCult.Geometry.hlsl b/src/GameCult.Geometry/Shaders/GameCult.Geometry.hlsl deleted file mode 100644 index 00623b75..00000000 --- a/src/GameCult.Geometry/Shaders/GameCult.Geometry.hlsl +++ /dev/null @@ -1,14 +0,0 @@ -#ifndef GAMECULT_GEOMETRY_HLSL -#define GAMECULT_GEOMETRY_HLSL - -#if defined(GAMECULT_GEOMETRY_UNITY_PACKAGE) -#include "Packages/org.gamecult.cultmath/shaders/CultMath.hlsl" -#else -#include "CultMath/CultMath.hlsl" -#endif -#include "AdvancedErosionFilter.hlsl" -#include "Planetary.hlsl" -#include "SphericalErosion.hlsl" -#include "PlanetaryRadialRefinement.hlsl" - -#endif diff --git a/src/GameCult.Geometry/Shaders/GameCult.Geometry.hlsl.meta b/src/GameCult.Geometry/Shaders/GameCult.Geometry.hlsl.meta deleted file mode 100644 index d685ccce..00000000 --- a/src/GameCult.Geometry/Shaders/GameCult.Geometry.hlsl.meta +++ /dev/null @@ -1,7 +0,0 @@ -fileFormatVersion: 2 -guid: b253fa50ddd34bf3b69fb323fe6e00e8 -ShaderIncludeImporter: - externalObjects: {} - userData: - assetBundleName: - assetBundleVariant: diff --git a/src/GameCult.Geometry/Shaders/Planetary.hlsl b/src/GameCult.Geometry/Shaders/Planetary.hlsl deleted file mode 100644 index 8e16bfff..00000000 --- a/src/GameCult.Geometry/Shaders/Planetary.hlsl +++ /dev/null @@ -1,392 +0,0 @@ -#ifndef GAMECULT_GEOMETRY_PLANETARY_HLSL -#define GAMECULT_GEOMETRY_PLANETARY_HLSL - -static const float GAMECULT_GEOMETRY_PLANETARY_QUARTER_PI = 0.78539816339744830962; - -float3 gamecult_geometry_planetary_face_direction(int face, float2 coordinate) -{ - float2 tangent = tan(coordinate * GAMECULT_GEOMETRY_PLANETARY_QUARTER_PI); - float3 cube = face == 0 ? float3(1.0, tangent.y, -tangent.x) - : face == 1 ? float3(-1.0, tangent.y, tangent.x) - : face == 2 ? float3(tangent.x, 1.0, -tangent.y) - : face == 3 ? float3(tangent.x, -1.0, tangent.y) - : face == 4 ? float3(tangent.x, tangent.y, 1.0) - : float3(-tangent.x, tangent.y, -1.0); - return cultmath_normalize(cube); -} - -float2 gamecult_geometry_planetary_face_coordinate(float3 direction, out int face) -{ - float3 d = cultmath_normalize(direction); - float3 a = abs(d); - float2 cube_uv; - if (a.x >= a.y && a.x >= a.z) - { - if (d.x >= 0.0) { face = 0; cube_uv = float2(-d.z, d.y) / a.x; } - else { face = 1; cube_uv = float2(d.z, d.y) / a.x; } - } - else if (a.y >= a.z) - { - if (d.y >= 0.0) { face = 2; cube_uv = float2(d.x, -d.z) / a.y; } - else { face = 3; cube_uv = float2(d.x, d.z) / a.y; } - } - else - { - if (d.z >= 0.0) { face = 4; cube_uv = d.xy / a.z; } - else { face = 5; cube_uv = float2(-d.x, d.y) / a.z; } - } - return atan(cube_uv) / GAMECULT_GEOMETRY_PLANETARY_QUARTER_PI; -} - -bool gamecult_geometry_planetary_page_local(float3 direction, float4 tile_address, float active, out float2 local) -{ - int face; - float2 uv = gamecult_geometry_planetary_face_coordinate(direction, face); - local = 0.0; - if (active < 0.5 || abs((float)face - tile_address.x) > 0.25) return false; - float axis_tiles = exp2(tile_address.y); - float2 scaled = (uv * 0.5 + 0.5) * axis_tiles; - float2 canonical = floor(min(scaled, axis_tiles - 1.0e-5)); - if (any(abs(canonical - tile_address.zw) > 0.25)) return false; - local = scaled - tile_address.zw; - return all(local >= 0.0) && all(local <= 1.0); -} - -float3 gamecult_geometry_planetary_surface_normal(float3 direction, float3 world_distance_tangent_gradient) -{ - direction = cultmath_normalize(direction); - float3 tangent_gradient = world_distance_tangent_gradient - direction * dot(world_distance_tangent_gradient, direction); - return cultmath_normalize(direction - tangent_gradient); -} - -float2 gamecult_geometry_planetary_equirectangular_forward(float3 direction) -{ - direction = cultmath_normalize(direction); - return float2(atan2(direction.y, direction.x) / CULTMATH_PI, asin(clamp(direction.z, -1.0, 1.0)) / CULTMATH_HALF_PI); -} - -float3 gamecult_geometry_planetary_equirectangular_inverse(float2 coordinate) -{ - float longitude = coordinate.x * CULTMATH_PI; - float latitude = coordinate.y * CULTMATH_HALF_PI; - float cos_latitude = cos(latitude); - return cultmath_normalize(float3(cos_latitude * cos(longitude), cos_latitude * sin(longitude), sin(latitude))); -} - -static const float GAMECULT_GEOMETRY_EQUAL_EARTH_A1 = 1.340264; -static const float GAMECULT_GEOMETRY_EQUAL_EARTH_A2 = -0.081106; -static const float GAMECULT_GEOMETRY_EQUAL_EARTH_A3 = 0.000893; -static const float GAMECULT_GEOMETRY_EQUAL_EARTH_A4 = 0.003796; -static const float GAMECULT_GEOMETRY_EQUAL_EARTH_M = 0.8660254037844386; -static const float GAMECULT_GEOMETRY_EQUAL_EARTH_X_MAX = 2.706629; -static const float GAMECULT_GEOMETRY_EQUAL_EARTH_Y_MAX = 1.3173628; - -float gamecult_geometry_equal_earth_y(float theta) -{ - float theta2 = theta * theta; - float theta6 = theta2 * theta2 * theta2; - return theta * (GAMECULT_GEOMETRY_EQUAL_EARTH_A1 + GAMECULT_GEOMETRY_EQUAL_EARTH_A2 * theta2 + theta6 * (GAMECULT_GEOMETRY_EQUAL_EARTH_A3 + GAMECULT_GEOMETRY_EQUAL_EARTH_A4 * theta2)); -} - -float2 gamecult_geometry_planetary_equal_earth_forward(float3 direction) -{ - direction = cultmath_normalize(direction); - float longitude = atan2(direction.y, direction.x); - float latitude = asin(clamp(direction.z, -1.0, 1.0)); - float theta = asin(GAMECULT_GEOMETRY_EQUAL_EARTH_M * sin(latitude)); - float theta2 = theta * theta; - float theta6 = theta2 * theta2 * theta2; - float denominator = GAMECULT_GEOMETRY_EQUAL_EARTH_M * (GAMECULT_GEOMETRY_EQUAL_EARTH_A1 + 3.0 * GAMECULT_GEOMETRY_EQUAL_EARTH_A2 * theta2 + theta6 * (7.0 * GAMECULT_GEOMETRY_EQUAL_EARTH_A3 + 9.0 * GAMECULT_GEOMETRY_EQUAL_EARTH_A4 * theta2)); - return float2(longitude * cos(theta) / denominator / GAMECULT_GEOMETRY_EQUAL_EARTH_X_MAX, gamecult_geometry_equal_earth_y(theta) / GAMECULT_GEOMETRY_EQUAL_EARTH_Y_MAX); -} - -float3 gamecult_geometry_planetary_equal_earth_inverse(float2 coordinate) -{ - float target_y = coordinate.y * GAMECULT_GEOMETRY_EQUAL_EARTH_Y_MAX; - float theta = target_y / GAMECULT_GEOMETRY_EQUAL_EARTH_A1; - [unroll] for (int iteration = 0; iteration < 8; iteration++) - { - float theta2 = theta * theta; - float theta6 = theta2 * theta2 * theta2; - float derivative = GAMECULT_GEOMETRY_EQUAL_EARTH_A1 + 3.0 * GAMECULT_GEOMETRY_EQUAL_EARTH_A2 * theta2 + theta6 * (7.0 * GAMECULT_GEOMETRY_EQUAL_EARTH_A3 + 9.0 * GAMECULT_GEOMETRY_EQUAL_EARTH_A4 * theta2); - theta -= (gamecult_geometry_equal_earth_y(theta) - target_y) / derivative; - } - float latitude = asin(clamp(sin(theta) / GAMECULT_GEOMETRY_EQUAL_EARTH_M, -1.0, 1.0)); - float theta2 = theta * theta; - float theta6 = theta2 * theta2 * theta2; - float denominator = GAMECULT_GEOMETRY_EQUAL_EARTH_M * (GAMECULT_GEOMETRY_EQUAL_EARTH_A1 + 3.0 * GAMECULT_GEOMETRY_EQUAL_EARTH_A2 * theta2 + theta6 * (7.0 * GAMECULT_GEOMETRY_EQUAL_EARTH_A3 + 9.0 * GAMECULT_GEOMETRY_EQUAL_EARTH_A4 * theta2)); - float longitude = coordinate.x * GAMECULT_GEOMETRY_EQUAL_EARTH_X_MAX * denominator / cos(theta); - float cos_latitude = cos(latitude); - return cultmath_normalize(float3(cos_latitude * cos(longitude), cos_latitude * sin(longitude), sin(latitude))); -} - -float gamecult_geometry_planetary_wrap_longitude(float longitude) -{ - longitude = fmod(longitude, CULTMATH_TAU); - if (longitude > CULTMATH_PI) longitude -= CULTMATH_TAU; - if (longitude < -CULTMATH_PI) longitude += CULTMATH_TAU; - return longitude; -} - -float3 gamecult_geometry_planetary_lon_lat_direction(float longitude, float latitude) -{ - float cos_latitude = cos(latitude); - return cultmath_normalize(float3(cos_latitude * cos(longitude), cos_latitude * sin(longitude), sin(latitude))); -} - -float gamecult_geometry_planetary_asinh(float value) { return log(value + sqrt(value * value + 1.0)); } -float gamecult_geometry_planetary_sinh(float value) { return 0.5 * (exp(value) - exp(-value)); } - -bool gamecult_geometry_planetary_azimuthal_forward(float3 direction, float2 center_lon_lat, int kind, out float2 coordinate) -{ - direction = cultmath_normalize(direction); - float longitude = gamecult_geometry_planetary_wrap_longitude(atan2(direction.y, direction.x) - center_lon_lat.x); - float latitude = asin(clamp(direction.z, -1.0, 1.0)); - float sin0 = sin(center_lon_lat.y), cos0 = cos(center_lon_lat.y); - float sin_latitude = sin(latitude), cos_latitude = cos(latitude), cos_longitude = cos(longitude); - float cos_c = clamp(sin0 * sin_latitude + cos0 * cos_latitude * cos_longitude, -1.0, 1.0); - float2 raw = float2(cos_latitude * sin(longitude), cos0 * sin_latitude - sin0 * cos_latitude * cos_longitude); - float factor = 1.0; - if (kind == 3) - { - if (cos_c < 0.0) { coordinate = 0.0; return false; } - } - else if (kind == 4) - { - float c = acos(cos_c); factor = (c < 1.0e-6 ? 1.0 : c / sin(c)) / CULTMATH_PI; - } - else if (kind == 5) factor = sqrt(2.0 / max(1.0 + cos_c, 1.0e-20)) * 0.5; - else - { - if (cos_c <= 0.0) { coordinate = 0.0; return false; } - factor = 1.0 / cos_c; - } - coordinate = raw * factor; - return true; -} - -bool gamecult_geometry_planetary_azimuthal_inverse(float2 coordinate, float2 center_lon_lat, int kind, out float3 direction) -{ - float rho = length(coordinate); - float c; - if (kind == 3) { if (rho > 1.0) { direction = 0.0; return false; } c = asin(clamp(rho, 0.0, 1.0)); } - else if (kind == 4) { if (rho > 1.0) { direction = 0.0; return false; } c = rho * CULTMATH_PI; } - else if (kind == 5) { if (rho > 1.0) { direction = 0.0; return false; } c = 2.0 * asin(clamp(rho, 0.0, 1.0)); } - else c = atan(rho); - if (rho < 1.0e-6) { direction = gamecult_geometry_planetary_lon_lat_direction(center_lon_lat.x, center_lon_lat.y); return true; } - float sin_c = sin(c), cos_c = cos(c), sin0 = sin(center_lon_lat.y), cos0 = cos(center_lon_lat.y); - float latitude = asin(clamp(cos_c * sin0 + coordinate.y * sin_c * cos0 / rho, -1.0, 1.0)); - float delta_longitude = atan2(coordinate.x * sin_c, rho * cos0 * cos_c - coordinate.y * sin0 * sin_c); - direction = gamecult_geometry_planetary_lon_lat_direction(gamecult_geometry_planetary_wrap_longitude(delta_longitude + center_lon_lat.x), latitude); - return true; -} - -bool gamecult_geometry_planetary_cube_atlas_forward(float3 direction, out float2 coordinate) -{ - int face; - float2 face_coordinate = gamecult_geometry_planetary_face_coordinate(direction, face); - int column = face % 3; - int row = face / 3; - coordinate = float2(-1.0 + (column + (face_coordinate.x + 1.0) * 0.5) * (2.0 / 3.0), 1.0 - (row + (face_coordinate.y + 1.0) * 0.5)); - return true; -} - -bool gamecult_geometry_planetary_cube_atlas_inverse(float2 coordinate, out float3 direction) -{ - if (any(coordinate < -1.0) || any(coordinate > 1.0)) { direction = 0.0; return false; } - int column = min((int)((coordinate.x + 1.0) * 1.5), 2); - int row = min((int)(1.0 - coordinate.y), 1); - float u = ((coordinate.x + 1.0) * 1.5 - column) * 2.0 - 1.0; - float v = ((1.0 - coordinate.y) - row) * 2.0 - 1.0; - direction = gamecult_geometry_planetary_face_direction(row * 3 + column, float2(u, v)); - return true; -} - -bool gamecult_geometry_planetary_projection_forward(float3 direction, int kind, float3 center_lon_lat_scale, out float2 coordinate) -{ - float scale = max(center_lon_lat_scale.z, 1.0e-20); - if (kind == 6) - { - bool valid = gamecult_geometry_planetary_cube_atlas_forward(direction, coordinate); - coordinate /= scale; return valid; - } - if (kind >= 3) - { - bool valid = gamecult_geometry_planetary_azimuthal_forward(direction, center_lon_lat_scale.xy, kind, coordinate); - coordinate /= scale; return valid; - } - direction = cultmath_normalize(direction); - float longitude = gamecult_geometry_planetary_wrap_longitude(atan2(direction.y, direction.x) - center_lon_lat_scale.x); - float latitude = asin(clamp(direction.z, -1.0, 1.0)); - if (kind == 0) coordinate = float2(longitude / CULTMATH_PI, latitude / CULTMATH_HALF_PI); - else if (kind == 1) - { - const float maximum_latitude = 1.4844222297453324; - if (abs(latitude) > maximum_latitude) { coordinate = 0.0; return false; } - coordinate = float2(longitude / CULTMATH_PI, gamecult_geometry_planetary_asinh(tan(latitude)) / CULTMATH_PI); - } - else - { - float3 centered = gamecult_geometry_planetary_lon_lat_direction(longitude, latitude); - coordinate = gamecult_geometry_planetary_equal_earth_forward(centered); - } - coordinate /= scale; - return true; -} - -bool gamecult_geometry_planetary_projection_inverse(float2 coordinate, int kind, float3 center_lon_lat_scale, out float3 direction) -{ - coordinate *= max(center_lon_lat_scale.z, 1.0e-20); - if (kind == 6) return gamecult_geometry_planetary_cube_atlas_inverse(coordinate, direction); - if (kind >= 3) return gamecult_geometry_planetary_azimuthal_inverse(coordinate, center_lon_lat_scale.xy, kind, direction); - if (any(abs(coordinate) > 1.0)) { direction = 0.0; return false; } - float longitude, latitude; - if (kind == 0) { longitude = coordinate.x * CULTMATH_PI; latitude = coordinate.y * CULTMATH_HALF_PI; } - else if (kind == 1) { longitude = coordinate.x * CULTMATH_PI; latitude = atan(gamecult_geometry_planetary_sinh(coordinate.y * CULTMATH_PI)); } - else - { - float3 centered = gamecult_geometry_planetary_equal_earth_inverse(coordinate); - longitude = atan2(centered.y, centered.x); latitude = asin(clamp(centered.z, -1.0, 1.0)); - } - direction = gamecult_geometry_planetary_lon_lat_direction(gamecult_geometry_planetary_wrap_longitude(longitude + center_lon_lat_scale.x), latitude); - return true; -} - -struct GameCultGeometryPlanetaryFieldDefinition -{ - float radius; - int seed; - GameCultGeometryAdvancedErosionParameters erosion; -}; - -struct GameCultGeometryPlanetaryBaseFieldSample -{ - float radial_displacement; - float3 radial_gradient; - float field_value; - float3 field_gradient; - float fade_target; -}; - -struct GameCultGeometryPlanetarySurfaceSample -{ - float radial_displacement; - float3 tangent_gradient; - float3 surface_normal; - float slope; - float ridge; - float gully; - float finest_resolved_wavelength; - float unresolved_height_bound; -}; - -struct GameCultGeometryPlanetaryPageSample -{ - float4 height_gradient; - float2 masks; -}; - -GameCultGeometryPlanetaryPageSample gamecult_geometry_planetary_page_lerp( - GameCultGeometryPlanetaryPageSample a, - GameCultGeometryPlanetaryPageSample b, - GameCultGeometryPlanetaryPageSample c, - GameCultGeometryPlanetaryPageSample d, - float2 fraction) -{ - GameCultGeometryPlanetaryPageSample result; - result.height_gradient = lerp(lerp(a.height_gradient, b.height_gradient, fraction.x), lerp(c.height_gradient, d.height_gradient, fraction.x), fraction.y); - result.masks = lerp(lerp(a.masks, b.masks, fraction.x), lerp(c.masks, d.masks, fraction.x), fraction.y); - return result; -} - -GameCultGeometryPlanetaryPageSample gamecult_geometry_planetary_residual_sample( - GameCultGeometryPlanetarySurfaceSample child, - GameCultGeometryPlanetarySurfaceSample parent, - bool has_parent) -{ - GameCultGeometryPlanetaryPageSample result; - result.height_gradient = float4(child.radial_displacement, child.tangent_gradient); - if (has_parent) result.height_gradient -= float4(parent.radial_displacement, parent.tangent_gradient); - result.masks = float2(child.ridge, child.gully); - return result; -} - -float4 gamecult_geometry_planetary_summary_empty() -{ - return float4(3.402823466e+38, -3.402823466e+38, 0.0, 0.0); -} - -float4 gamecult_geometry_planetary_summary_accumulate(float4 summary, float4 height_gradient, float unresolved_height) -{ - summary.x = min(summary.x, height_gradient.x); - summary.y = max(summary.y, height_gradient.x); - summary.z = max(summary.z, length(height_gradient.yzw)); - summary.w = max(summary.w, max(unresolved_height, 0.0)); - return summary; -} - -float gamecult_geometry_planetary_pow4(float value) { float square = value * value; return square * square; } - -GameCultGeometryPlanetarySurfaceSample gamecult_geometry_planetary_field_sample( - GameCultGeometryPlanetaryFieldDefinition definition, - float3 unit_direction, - GameCultGeometryPlanetaryBaseFieldSample base_sample, - float footprint_meters) -{ - float3 direction = cultmath_normalize(unit_direction); - float3 radial_gradient = base_sample.radial_gradient - direction * dot(base_sample.radial_gradient, direction); - float3 field_gradient = base_sample.field_gradient - direction * dot(base_sample.field_gradient, direction); - GameCultGeometryAdvancedErosionParameters p = definition.erosion; - GameCultGeometryErosionBandSelection band = gamecult_geometry_select_erosion_bands( - p.scale * p.cell_scale, footprint_meters, p.octaves, p.lacunarity, - p.strength * p.scale, p.gain, 2.0); - float3 powers = float3( - gamecult_geometry_planetary_pow4(abs(direction.x)), - gamecult_geometry_planetary_pow4(abs(direction.y)), - gamecult_geometry_planetary_pow4(abs(direction.z))); - float power_sum = max(powers.x + powers.y + powers.z, 1.0e-6); - float3 weights = powers / power_sum; - float3 world = direction * definition.radius; - float seed = (float)definition.seed; - GameCultGeometryAdvancedErosionResult xy = gamecult_geometry_advanced_erosion_filter_banded( - world.xy + float2(713.0 + seed * 0.754877666, -291.0 + seed * 0.569840296), - float3(base_sample.field_value, field_gradient.x, field_gradient.y), base_sample.fade_target, p, band); - GameCultGeometryAdvancedErosionResult yz = gamecult_geometry_advanced_erosion_filter_banded( - world.yz + float2(-431.0 + seed * 0.438289027, 887.0 + seed * 0.328438163), - float3(base_sample.field_value, field_gradient.y, field_gradient.z), base_sample.fade_target, p, band); - GameCultGeometryAdvancedErosionResult zx = gamecult_geometry_advanced_erosion_filter_banded( - world.zx + float2(197.0 + seed * 0.219783071, 557.0 + seed * 0.145898034), - float3(base_sample.field_value, field_gradient.z, field_gradient.x), base_sample.fade_target, p, band); - float erosion_height = xy.delta.x * weights.z + yz.delta.x * weights.x + zx.delta.x * weights.y; - float3 erosion_gradient = float3( - xy.delta.y * weights.z + zx.delta.z * weights.y, - xy.delta.z * weights.z + yz.delta.y * weights.x, - yz.delta.z * weights.x + zx.delta.y * weights.y); - float3 power_gradient_x = float3(4.0 * direction.x * direction.x * direction.x / definition.radius, 0.0, 0.0); - float3 power_gradient_y = float3(0.0, 4.0 * direction.y * direction.y * direction.y / definition.radius, 0.0); - float3 power_gradient_z = float3(0.0, 0.0, 4.0 * direction.z * direction.z * direction.z / definition.radius); - power_gradient_x -= direction * dot(power_gradient_x, direction); - power_gradient_y -= direction * dot(power_gradient_y, direction); - power_gradient_z -= direction * dot(power_gradient_z, direction); - float3 power_sum_gradient = power_gradient_x + power_gradient_y + power_gradient_z; - float inverse_power_sum_squared = 1.0 / (power_sum * power_sum); - float3 weight_gradient_x = (power_gradient_x * power_sum - power_sum_gradient * powers.x) * inverse_power_sum_squared; - float3 weight_gradient_y = (power_gradient_y * power_sum - power_sum_gradient * powers.y) * inverse_power_sum_squared; - float3 weight_gradient_z = (power_gradient_z * power_sum - power_sum_gradient * powers.z) * inverse_power_sum_squared; - erosion_gradient += yz.delta.x * weight_gradient_x + zx.delta.x * weight_gradient_y + xy.delta.x * weight_gradient_z; - float3 gradient = radial_gradient + erosion_gradient; - gradient -= direction * dot(gradient, direction); - float ridge_evidence = xy.ridge_map * weights.z + yz.ridge_map * weights.x + zx.ridge_map * weights.y; - GameCultGeometryPlanetarySurfaceSample result; - result.radial_displacement = base_sample.radial_displacement + erosion_height; - result.tangent_gradient = gradient; - result.surface_normal = gamecult_geometry_planetary_surface_normal(direction, gradient); - result.slope = length(gradient); - result.ridge = saturate(ridge_evidence * 0.5 + 0.5); - result.gully = saturate(0.5 - ridge_evidence * 0.5); - result.finest_resolved_wavelength = band.finest_included_wavelength; - result.unresolved_height_bound = band.unresolved_height_bound; - return result; -} - -#endif diff --git a/src/GameCult.Geometry/Shaders/Planetary.hlsl.meta b/src/GameCult.Geometry/Shaders/Planetary.hlsl.meta deleted file mode 100644 index 009c2989..00000000 --- a/src/GameCult.Geometry/Shaders/Planetary.hlsl.meta +++ /dev/null @@ -1,7 +0,0 @@ -fileFormatVersion: 2 -guid: bb055ad5328888e479e90ae3fe62c3fe -ShaderIncludeImporter: - externalObjects: {} - userData: - assetBundleName: - assetBundleVariant: diff --git a/src/GameCult.Geometry/Shaders/PlanetaryRadialRefinement.hlsl b/src/GameCult.Geometry/Shaders/PlanetaryRadialRefinement.hlsl deleted file mode 100644 index 20f3e2e5..00000000 --- a/src/GameCult.Geometry/Shaders/PlanetaryRadialRefinement.hlsl +++ /dev/null @@ -1,27 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this file, -// You can obtain one at https://mozilla.org/MPL/2.0/. - -#ifndef GAMECULT_GEOMETRY_PLANETARY_RADIAL_REFINEMENT_HLSL -#define GAMECULT_GEOMETRY_PLANETARY_RADIAL_REFINEMENT_HLSL - -float3 gamecult_geometry_planetary_radial_refinement_step( - float3 position, - float3 ray_direction, - float3 center, - float target_radius, - float minimum_derivative, - float maximum_correction) -{ - float3 local=position-center; - float radius=length(local); - float3 radial=local/max(radius,0.0001); - float error=radius-target_radius; - float derivative=dot(ray_direction,radial); - float safe_minimum=max(minimum_derivative,0.0001); - float safe_derivative=abs(derivative)>safe_minimum?derivative:(derivative<0?-safe_minimum:safe_minimum); - float correction=clamp(-error/safe_derivative,-abs(maximum_correction),abs(maximum_correction)); - return position+ray_direction*correction; -} - -#endif diff --git a/src/GameCult.Geometry/Shaders/PlanetaryRadialRefinement.hlsl.meta b/src/GameCult.Geometry/Shaders/PlanetaryRadialRefinement.hlsl.meta deleted file mode 100644 index 28d629b1..00000000 --- a/src/GameCult.Geometry/Shaders/PlanetaryRadialRefinement.hlsl.meta +++ /dev/null @@ -1,7 +0,0 @@ -fileFormatVersion: 2 -guid: 356389371b854e12b9d419f7ee310d81 -ShaderIncludeImporter: - externalObjects: {} - userData: - assetBundleName: - assetBundleVariant: diff --git a/src/GameCult.Geometry/Shaders/SphericalErosion.hlsl b/src/GameCult.Geometry/Shaders/SphericalErosion.hlsl deleted file mode 100644 index 107ba24b..00000000 --- a/src/GameCult.Geometry/Shaders/SphericalErosion.hlsl +++ /dev/null @@ -1,80 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this file, -// You can obtain one at https://mozilla.org/MPL/2.0/. - -#ifndef GAMECULT_GEOMETRY_SPHERICAL_EROSION_HLSL -#define GAMECULT_GEOMETRY_SPHERICAL_EROSION_HLSL - -struct GameCultGeometrySphericalErosionParameters -{ - float frequency; float amplitude; float lacunarity; float gain; - float slope_strength; float detail; float seed; int octaves; -}; - -struct GameCultGeometrySphericalErosionSample -{ - float height_offset; float ridge; float gully; float3 gradient; -}; - -float gamecult_geometry_spherical_cell_wave(float3 position, float3 flow, float3 cell, float seed) -{ - float3 pivot = cell + float3(cultmath_hash(cell + seed + 11.7), cultmath_hash(cell.yzx + seed + 37.1), cultmath_hash(cell.zyx + seed + 73.9)); - return dot(position - pivot, flow) * 6.28318530718 + cultmath_hash(cell + seed + 101.3) * 6.28318530718; -} - -float gamecult_geometry_spherical_directional_wave(float3 position, float3 flow, float seed) -{ - float3 cell = floor(position); - float3 local = cultmath_frac(position); - float3 weights = local * local * (3.0 - 2.0 * local); - float z0 = cultmath_lerp( - cultmath_lerp(gamecult_geometry_spherical_cell_wave(position, flow, cell + float3(0,0,0), seed), gamecult_geometry_spherical_cell_wave(position, flow, cell + float3(1,0,0), seed), weights.x), - cultmath_lerp(gamecult_geometry_spherical_cell_wave(position, flow, cell + float3(0,1,0), seed), gamecult_geometry_spherical_cell_wave(position, flow, cell + float3(1,1,0), seed), weights.x), weights.y); - float z1 = cultmath_lerp( - cultmath_lerp(gamecult_geometry_spherical_cell_wave(position, flow, cell + float3(0,0,1), seed), gamecult_geometry_spherical_cell_wave(position, flow, cell + float3(1,0,1), seed), weights.x), - cultmath_lerp(gamecult_geometry_spherical_cell_wave(position, flow, cell + float3(0,1,1), seed), gamecult_geometry_spherical_cell_wave(position, flow, cell + float3(1,1,1), seed), weights.x), weights.y); - return cultmath_lerp(z0, z1, weights.z); -} - -float3 gamecult_geometry_spherical_stable_tangent(float3 position) -{ - float3 axis = abs(position.z) < 0.8 ? float3(0,0,1) : float3(0,1,0); - return cultmath_normalize(cross(axis, position)); -} - -GameCultGeometrySphericalErosionSample gamecult_geometry_spherical_erosion(float3 unit_position, float base_height, float3 base_gradient, GameCultGeometrySphericalErosionParameters p) -{ - float3 position = cultmath_normalize(unit_position); - float3 gradient = base_gradient - position * dot(base_gradient, position); - float slope = length(gradient); - float3 flow = slope > 1.0e-6 ? -gradient / slope : gamecult_geometry_spherical_stable_tangent(position); - float frequency = max(p.frequency, 0.001); - float amplitude = p.amplitude; - float combi_mask = 1.0 - pow(1.0 - cultmath_saturate(slope * p.slope_strength), 2.0); - float fade_target = cultmath_saturate(base_height * 0.5 + 0.5) * 2.0 - 1.0; - GameCultGeometrySphericalErosionSample result = (GameCultGeometrySphericalErosionSample)0; - [loop] for (int octave = 0; octave < min(max(p.octaves, 0), 8); octave++) - { - float phase = gamecult_geometry_spherical_directional_wave(position * frequency, flow, p.seed + octave * 19.19); - float wave = cos(phase); - float wave_slope = sin(phase); - float faded = cultmath_lerp(fade_target, wave, combi_mask); - result.height_offset += faded * amplitude; - result.ridge = max(result.ridge, cultmath_saturate(faded)); - result.gully = max(result.gully, cultmath_saturate(-faded)); - float3 across = cultmath_normalize(cross(position, flow)); - gradient += across * (sign(wave_slope) * amplitude * frequency * combi_mask); - gradient -= position * dot(gradient, position); - slope = length(gradient); - flow = slope > 1.0e-6 ? -gradient / slope : gamecult_geometry_spherical_stable_tangent(position); - float new_mask = 1.0 - pow(1.0 - cultmath_saturate(abs(wave_slope) * 1.35), 2.0); - combi_mask = (1.0 - pow(1.0 - cultmath_saturate(combi_mask), max(p.detail, 0.001))) * new_mask; - fade_target = faded; - frequency *= max(p.lacunarity, 1.01); - amplitude *= cultmath_saturate(p.gain); - } - result.gradient = gradient; - return result; -} - -#endif diff --git a/src/GameCult.Geometry/Shaders/SphericalErosion.hlsl.meta b/src/GameCult.Geometry/Shaders/SphericalErosion.hlsl.meta deleted file mode 100644 index 7b7ddc79..00000000 --- a/src/GameCult.Geometry/Shaders/SphericalErosion.hlsl.meta +++ /dev/null @@ -1,7 +0,0 @@ -fileFormatVersion: 2 -guid: 43d458fc5b6b441a941e37001df919e3 -ShaderIncludeImporter: - externalObjects: {} - userData: - assetBundleName: - assetBundleVariant: diff --git a/src/GameCult.Geometry/THIRD-PARTY-NOTICES.md b/src/GameCult.Geometry/THIRD-PARTY-NOTICES.md deleted file mode 100644 index 3388b601..00000000 --- a/src/GameCult.Geometry/THIRD-PARTY-NOTICES.md +++ /dev/null @@ -1,24 +0,0 @@ -# GameCult.Geometry source provenance - -## CultMath planetary geometry - -The C# files under `Planetary/` and the HLSL files under `Shaders/` are licensed -under the Mozilla Public License 2.0 and retain MPL-2.0 notices. CultMath owns -the numeric primitives used by these geometry systems; GameCult.Geometry owns -their spatial meaning. - -## Advanced Terrain Erosion Filter - -`Planetary/AdvancedErosionFilter.cs` and `Shaders/AdvancedErosionFilter.hlsl` -adapt the mathematical procedure from -Rune Skovbo Johansen's **Advanced Terrain Erosion Filter**. - -- Copyright 2025 Rune Skovbo Johansen. -- Upstream: -- Explanation: -- License: Mozilla Public License 2.0, - -The implementation includes C#/HLSL type lowering, bounded octave counts, -finite zero-rounding behavior, and a shared integer avalanche hash for CPU/GPU -cell-selection parity. The technique builds on earlier work credited upstream -to Fewes and Clay John; the hash form follows public shader math by Inigo Quilez. diff --git a/src/GameCult.Geometry/docs/planetary.md b/src/GameCult.Geometry/docs/planetary.md deleted file mode 100644 index 94eab553..00000000 --- a/src/GameCult.Geometry/docs/planetary.md +++ /dev/null @@ -1,135 +0,0 @@ -# Planetary Field Runtime - -GameCult.Geometry owns the deterministic, renderer-independent planetary field. The same -definition can be queried by a CPU simulation, baked into residual pages, -sampled through a map projection, or evaluated by HLSL. Render resources and -gameplay interpretation remain consumer responsibilities. - -## Authority - -- `PlanetaryFieldDefinition` identifies radius, seed, erosion parameters, and a - version that includes the consumer-owned base-field version. -- `IPlanetaryBaseField` supplies deterministic application-authored base - evidence without making Geometry depend on an application. -- `PlanetaryQueryScale` makes the physical footprint and accepted unresolved - error explicit. There is deliberately no scale-free authoritative - `HeightAt` call. -- `PlanetarySurfaceSample` carries displacement, tangent gradient, normal, - slope, ridge/gully evidence, resolved wavelength, and unresolved bound. -- pages, residency, patches, and map tiles are disposable lowerings of that - contract. - -The C# surface targets .NET and Unity-compatible `netstandard2.1`. The matching -HLSL lives in `Shaders/Planetary.hlsl`. Consumers include -`Shaders/GameCult.Geometry.hlsl`, which imports CultMath's numeric mirror before -the geometry kernels. - -## Point and batch queries - -```csharp -var erosion = AdvancedErosionParameters.Default with -{ - Scale = radius * 0.075f, - Strength = 0.12f, - Octaves = 7, -}; -var field = PlanetaryFieldDefinition.Create( - baseFieldVersion: 17, - radius, - seed: 42, - erosion); -var scale = PlanetaryQueryScale.AtFootprint( - footprintMeters: 10, - maximumUnresolvedHeight: 0.5f); - -PlanetaryBaseFieldSample basis = source.Sample(direction); -PlanetarySurfaceSample sample = PlanetaryField.Sample( - field, direction, basis, scale); -``` - -`PlanetaryField.SampleBatch` writes into a caller-owned span. `SamplePosition` -accepts `double3`, reduces the astronomical position to a stable unit direction, -and then evaluates the same float field contract. - -Higher-level CPU queries include bounded ray intersection, segment clearance, -great-circle profiles, and region summaries. Their answers remain field queries; -only a consuming simulation can turn them into committed game consequences. - -## Cube-sphere topology and pages - -`PlanetaryTopology` owns the tangent/QSC cube-sphere mapping, inverse face -coordinates, tile selection, local page coordinates, and radial-graph surface -normal. `PlanetaryPageSampling` owns bordered directions, nominal spacing, -bilinear sampling, and summaries. - -`PlanetaryPageBaker` evaluates a root page at its physical sample spacing. A -child stores `child filtered field - parent filtered field`. A -`PlanetaryPageSetSampling` query adds every containing ancestor contribution -and applies transition weights only to residuals. - -`PlanetaryLodSelector` selects six roots plus one camera-facing ancestor chain -from the unresolved-height error. `PlanetaryResidualResidency` owns arrival and -departure presentation weights, content and presentation versions, and nothing -about field truth. - -## Patches - -`PlanetaryPatch.CreateFace` and `CreateCubeSphere` produce renderer-neutral -coarse meshes from the canonical topology. HLSL calls -`gamecult_geometry_planetary_face_direction` for procedural vertices and -`gamecult_geometry_planetary_radial_refinement_step` for bounded pixel refinement. - -Renderers still own cameras, clip space, depth, materials, buffers, dispatch, -and draw submission. - -`PlanetaryGpuPageBuilder` owns the renderer-neutral page input and metadata -payload. Host adapters may convert its `float4` values to native vector types, -but do not recalculate directions, spacing, addressing, or transition state. - -The `GameCult.Geometry.Unity` assembly in `org.gamecult.geometry` converts a canonical face patch into a -Unity `Mesh` and converts the shared GPU payload into Unity-native structs. It -owns no field parameters, page selection, or simulation state. The package's -Planetary Field Viewer sample renders the shared HLSL field and reports matching -CPU queries. Unity render code remains responsible for allocating/binding -graphics buffers and dispatching application-specific base-field entry points. -The host Unity project must set `-langversion:latest` in `Assets/csc.rsp`. - -## Maps - -`PlanetaryProjection` supplies forward and inverse transforms for: - -- equirectangular; -- Web Mercator with an explicit polar cutoff; -- Equal Earth; -- orthographic; -- azimuthal equidistant and equal-area; -- cube atlas; -- local tangent/gnomonic views. - -`PlanetaryMapTileBaker` maps each valid tile sample through the inverse -projection and canonical field query. `PlanetaryMapTileKey` includes field, -projection, layer, layout, and query-scale identity. Palettes, contours, labels, -and strategic overlays do not enter GameCult.Geometry's canonical field. - -`PlanetaryMapTileEncoding` writes the resulting evidence as a versioned -little-endian `CMPT` binary asset. -`packages/gamecult-geometry-ts/src/planetary-tile.ts` decodes that exact format -into browser-native values without defining another terrain generator. -Cross-runtime fixtures are generated by -`tools/GameCult.Geometry.PlanetaryTileFixture`. -Typed CultMesh documents should carry the tile key and asset reference; the -binary payload is derived CDN/cache data, not committed game state. - -## Numeric contract - -Integer hashes and published field versions are exact. Basic arithmetic follows -CultMath's matched C#/HLSL semantics. Hardware transcendental functions such as -`atan` and `tan` use measured tolerances rather than fictional bit identity. -Every filtered terrain query carries an unresolved-height bound so consumers can -reject insufficient fidelity explicitly. - -Current D3D12 parity evidence covers the advanced erosion kernel, QSC topology, -and forward/inverse transforms for every supported map projection, including -center and scale parameters. Production page and patch shaders use the common -functions and retain their seam, residual, summary, and radial-hit integration -probes in Fensalir. diff --git a/src/GameCult.Mesh.Quic.Native/CultMeshNativeQuicRealtimeTransport.cs b/src/GameCult.Mesh.Quic.Native/CultMeshNativeQuicRealtimeTransport.cs index d687d5b6..cb356044 100644 --- a/src/GameCult.Mesh.Quic.Native/CultMeshNativeQuicRealtimeTransport.cs +++ b/src/GameCult.Mesh.Quic.Native/CultMeshNativeQuicRealtimeTransport.cs @@ -42,11 +42,11 @@ namespace GameCult.Mesh.Quic.Native public async Task ConnectAsync( CultMeshTransportCandidate candidate, - CultMeshEndpointId endpointId, + CultMeshSessionTarget target, CancellationToken cancellationToken = default) { if (candidate == null) throw new ArgumentNullException(nameof(candidate)); - if (endpointId == null) throw new ArgumentNullException(nameof(endpointId)); + if (target == null) throw new ArgumentNullException(nameof(target)); if (!TryParseEndpoint(candidate.Endpoint, out var host, out var port, out var pin)) throw new NotSupportedException($"Native QUIC connector does not support '{candidate.Endpoint}'."); @@ -72,7 +72,9 @@ namespace GameCult.Mesh.Quic.Native var transport = new CultMeshNativeQuicRealtimeTransport( candidate.Endpoint, handle, - _options.PollInterval); + _options.PollInterval, + target, + candidate.Generation); try { await transport.WaitUntilConnectedAsync(_options.HandshakeTimeout, cancellationToken).ConfigureAwait(false); @@ -122,23 +124,42 @@ namespace GameCult.Mesh.Quic.Native } } - internal sealed class CultMeshNativeQuicRealtimeTransport : ICultMeshRealtimeTransport + internal sealed class CultMeshNativeQuicRealtimeTransport : ICultMeshRealtimeTransport, ICultMeshVerifiedTransport { private readonly object _gate = new object(); private readonly TimeSpan _pollInterval; private IntPtr _handle; + private readonly CultMeshSessionTarget _target; + private readonly string _routeGeneration; private bool _disposed; - public CultMeshNativeQuicRealtimeTransport(string endpoint, IntPtr handle, TimeSpan pollInterval) + public CultMeshNativeQuicRealtimeTransport( + string endpoint, + IntPtr handle, + TimeSpan pollInterval, + CultMeshSessionTarget target, + string routeGeneration) { Endpoint = endpoint; _handle = handle; _pollInterval = pollInterval; + _target = target; + _routeGeneration = routeGeneration ?? string.Empty; } public string TransportId => "msquic-native-realtime"; public string Endpoint { get; } + public bool IsVerifiedFor( + string verseId, + string authorityRuntimeId, + string protocolId, + string routeGeneration) => + string.Equals(_target.VerseId, verseId, StringComparison.Ordinal) && + string.Equals(_target.AuthorityRuntimeId, authorityRuntimeId, StringComparison.Ordinal) && + string.Equals(CultMeshProtocols.RealtimeState.Value, protocolId, StringComparison.Ordinal) && + string.Equals(_routeGeneration, routeGeneration, StringComparison.Ordinal); + public Task SendAsync(CultMeshRealtimeFrame frame, CancellationToken cancellationToken = default) => Task.FromException(new NotSupportedException( "The Unity-compatible native QUIC connector currently owns provider-to-client state only.")); diff --git a/src/GameCult.Mesh.Quic/CultMeshQuicRealtimeTransport.cs b/src/GameCult.Mesh.Quic/CultMeshQuicRealtimeTransport.cs index a0198a32..91af6d7c 100644 --- a/src/GameCult.Mesh.Quic/CultMeshQuicRealtimeTransport.cs +++ b/src/GameCult.Mesh.Quic/CultMeshQuicRealtimeTransport.cs @@ -22,7 +22,7 @@ public sealed class CultMeshQuicRealtimeConnectorOptions /// /// Optional provider-aware certificate validator. Standard TLS validation is used when omitted. /// - public Func? + public Func? ValidateProviderCertificate { get; set; } } @@ -55,11 +55,11 @@ public sealed class CultMeshQuicRealtimeTransportConnector : ICultMeshRealtimeTr /// public async Task ConnectAsync( CultMeshTransportCandidate candidate, - CultMeshEndpointId endpointId, + CultMeshSessionTarget target, CancellationToken cancellationToken = default) { if (candidate == null) throw new ArgumentNullException(nameof(candidate)); - if (endpointId == null) throw new ArgumentNullException(nameof(endpointId)); + if (target == null) throw new ArgumentNullException(nameof(target)); if (!QuicConnection.IsSupported) throw new PlatformNotSupportedException("QUIC is unavailable. Install a supported MsQuic runtime."); if (!TryParseEndpoint(candidate.Endpoint, out var host, out var port)) @@ -84,14 +84,15 @@ public sealed class CultMeshQuicRealtimeTransportConnector : ICultMeshRealtimeTr { var providerCertificate = certificate as X509Certificate2 ?? (certificate == null ? null : new X509Certificate2(certificate)); - return validator?.Invoke(endpointId, providerCertificate, chain, errors) ?? + return validator?.Invoke(CultMeshRuntimeId.Parse(target.AuthorityRuntimeId), providerCertificate, chain, errors) ?? errors == SslPolicyErrors.None || MatchesAdvertisedCertificatePin(candidate.Endpoint, providerCertificate); } } }, cancellationToken).ConfigureAwait(false); - return new CultMeshQuicRealtimeTransport(candidate.Endpoint, connection); + return new CultMeshQuicRealtimeTransport( + candidate.Endpoint, connection, target: target, routeGeneration: candidate.Generation); } private static bool TryParseEndpoint(string? endpoint, out string host, out int port) @@ -308,7 +309,7 @@ public sealed class CultMeshQuicRealtimeServer : IAsyncDisposable } } -internal sealed class CultMeshQuicRealtimeTransport : ICultMeshRealtimeTransport +internal sealed class CultMeshQuicRealtimeTransport : ICultMeshRealtimeTransport, ICultMeshVerifiedTransport { private readonly QuicConnection _connection; private readonly CancellationTokenSource _shutdown = new(); @@ -318,6 +319,8 @@ internal sealed class CultMeshQuicRealtimeTransport : ICultMeshRealtimeTransport private readonly SemaphoreSlim _reliableSendGate = new(1, 1); private readonly CultMeshRealtimeInbox _outbound = new(); private readonly Action? _observer; + private readonly CultMeshSessionTarget? _target; + private readonly string _routeGeneration; private readonly Task _acceptLoop; private readonly Task _outboundLoop; private readonly Task _completion; @@ -328,11 +331,15 @@ internal sealed class CultMeshQuicRealtimeTransport : ICultMeshRealtimeTransport public CultMeshQuicRealtimeTransport( string endpoint, QuicConnection connection, - Action? observer = null) + Action? observer = null, + CultMeshSessionTarget? target = null, + string routeGeneration = "") { Endpoint = endpoint; _connection = connection; _observer = observer; + _target = target; + _routeGeneration = routeGeneration ?? string.Empty; _acceptLoop = AcceptStreamsAsync(_shutdown.Token); _outboundLoop = SendPublishedFramesAsync(_shutdown.Token); _completion = Task.WhenAll(_acceptLoop, _outboundLoop); @@ -342,6 +349,17 @@ internal sealed class CultMeshQuicRealtimeTransport : ICultMeshRealtimeTransport public string Endpoint { get; } public Task Completion => _completion; + public bool IsVerifiedFor( + string verseId, + string authorityRuntimeId, + string protocolId, + string routeGeneration) => + _target != null && + string.Equals(_target.VerseId, verseId, StringComparison.Ordinal) && + string.Equals(_target.AuthorityRuntimeId, authorityRuntimeId, StringComparison.Ordinal) && + string.Equals(CultMeshProtocols.RealtimeState.Value, protocolId, StringComparison.Ordinal) && + string.Equals(_routeGeneration, routeGeneration, StringComparison.Ordinal); + internal bool TryPublishLatest(CultMeshRealtimeFrame frame) { if (_disposed || frame.Delivery != CultMeshRealtimeDelivery.LatestOnly) return false; diff --git a/src/GameCult.Mesh/CultMesh.cs b/src/GameCult.Mesh/CultMesh.cs index fbb08f22..e7436db2 100644 --- a/src/GameCult.Mesh/CultMesh.cs +++ b/src/GameCult.Mesh/CultMesh.cs @@ -432,7 +432,7 @@ namespace GameCult.Mesh /// public Task FlushAsync(bool soft = false) { - return _host.FlushAsync(soft); + return _host.FlushAsync(); } /// @@ -493,56 +493,6 @@ namespace GameCult.Mesh return Document(recordKey, verse.Context, documentId, sources, routeHint); } - /// - /// Creates a managed reactive document mirror over one distributed record hosted by this node. - /// - public Task> ReactiveDocumentAsync( - CultRecordKey key, - CultMeshVerseContext context, - CultMeshReactiveDocumentOptions? options = null) - where TDocument : class - { - return Document(key, context).ReactiveAsync(options); - } - - /// - /// Creates a managed reactive document mirror over one distributed record hosted by this node. - /// - public Task> ReactiveDocumentAsync( - CultRecordKey key, - CultMeshVerse verse, - CultMeshReactiveDocumentOptions? options = null) - where TDocument : class - { - if (verse == null) throw new ArgumentNullException(nameof(verse)); - return ReactiveDocumentAsync(key, verse.Context, options); - } - - /// - /// Creates a managed reactive document mirror over one distributed record hosted by this node. - /// - public Task> ReactiveDocumentAsync( - string recordKey, - CultMeshVerseContext context, - CultMeshReactiveDocumentOptions? options = null) - where TDocument : class - { - return ReactiveDocumentAsync(new CultRecordKey(recordKey), context, options); - } - - /// - /// Creates a managed reactive document mirror over one distributed record hosted by this node. - /// - public Task> ReactiveDocumentAsync( - string recordKey, - CultMeshVerse verse, - CultMeshReactiveDocumentOptions? options = null) - where TDocument : class - { - if (verse == null) throw new ArgumentNullException(nameof(verse)); - return ReactiveDocumentAsync(recordKey, verse.Context, options); - } - /// public void Dispose() { @@ -1167,9 +1117,18 @@ namespace GameCult.Mesh string pointerId, string? sourceId = null, string? schemaId = null, - CultMeshRouteHint? routeHint = null) + CultMeshRouteHint? routeHint = null, + string? bindingName = null, + string? documentId = null, + string? fieldPath = null, + string? valueKind = null, + string? accessMode = null, + string? authority = null, + string? writeCommand = null) { - return new CultMeshStateBindingDescriptor(targetProp, pointerId, sourceId, schemaId, routeHint); + return new CultMeshStateBindingDescriptor( + targetProp, pointerId, sourceId, schemaId, routeHint, + bindingName, documentId, fieldPath, valueKind, accessMode, authority, writeCommand); } /// @@ -1189,7 +1148,14 @@ namespace GameCult.Mesh string? sourceId = null, string? schemaId = null, string? routeKind = null, - string? routeDescription = null) + string? routeDescription = null, + string? bindingName = null, + string? documentId = null, + string? fieldPath = null, + string? valueKind = null, + string? accessMode = null, + string? authority = null, + string? writeCommand = null) { return new CultMeshStateBindingRecord( targetProp, @@ -1197,7 +1163,14 @@ namespace GameCult.Mesh sourceId, schemaId, routeKind, - routeDescription); + routeDescription, + bindingName, + documentId, + fieldPath, + valueKind, + accessMode, + authority, + writeCommand); } /// @@ -1547,19 +1520,23 @@ namespace GameCult.Mesh var sourceList = sources?.ToArray() ?? new[] { ProjectionSource(key.Value, descriptor.SchemaId, "CultCache record") }; var route = routeHint ?? new CultMeshRouteHint(CultMeshLocalityKind.InProcess, "CultCache document"); - return Document( + var feed = LiveFeed( ResolveDocumentId(documentId, key), - context, - _ => Task.FromResult(ReadRequired(cache, key)), - _ => cache.WatchRecord(key) + (_parameters, _context) => Task.FromResult(ReadRequired(cache, key)), + (_parameters, _context) => cache.WatchRecord(key) .Where(change => change.Document != null) .Select(change => change.Document!), - async value => - { - await cache.UpsertAsync(value, new CultRecordHandle(key)).ConfigureAwait(false); - }, sourceList, route); + return new CultMeshDocumentHandle( + BindLiveFeed(context, feed), + async value => await cache.UpsertAsync(value, new CultRecordHandle(key)).ConfigureAwait(false), + () => cache.WatchRecord(key), + () => + { + var (document, sequence) = cache.GetWithSequence(key); + return (RequireDocument(document, key), sequence); + }); } /// @@ -2010,21 +1987,22 @@ namespace GameCult.Mesh var sourceList = sources?.ToArray() ?? new[] { ProjectionSource(key.Value, descriptor.SchemaId, "CultNet database record") }; var route = routeHint ?? new CultMeshRouteHint(CultMeshLocalityKind.Automatic, "CultNet database document"); - return Document( + var feed = LiveFeed( ResolveDocumentId(documentId, key), - context, - async _ => await ReadDatabaseDocumentRequiredAsync(database, key).ConfigureAwait(false), - _ => WatchDatabaseRecordAs(database, key), - async value => - { - await PutDatabaseDocumentAsync(database, key, value, predicted: false).ConfigureAwait(false); - }, - async value => - { - await PutDatabaseDocumentAsync(database, key, value, predicted: true).ConfigureAwait(false); - }, + (_parameters, _context) => ReadDatabaseDocumentRequiredAsync(database, key), + (_parameters, _context) => WatchDatabaseRecordAs(database, key), sourceList, route); + Func? replace = database.CanWriteAuthoritatively(key) + ? value => PutDatabaseDocumentAsync(database, key, value, predicted: false) + : null; + Func? submitPrediction = database.CanSubmitPrediction(key) + ? value => PutDatabaseDocumentAsync(database, key, value, predicted: true) + : null; + return new CultMeshDocumentHandle( + BindLiveFeed(context, feed), + replace, + submitPrediction); } /// @@ -3397,13 +3375,15 @@ namespace GameCult.Mesh } private static TDocument ReadRequired(CultCache cache, CultRecordKey key) + where TDocument : class => + RequireDocument(cache.Get(key), key); + + private static TDocument RequireDocument(object? untyped, CultRecordKey key) where TDocument : class { - var document = cache.Get(key); - if (document != null) + if (untyped is TDocument document) return document; - var untyped = cache.Get(key); if (untyped != null && IsSameCultDocumentSchema(untyped.GetType())) return ConvertUntypedDocument(untyped); diff --git a/src/GameCult.Mesh/CultMeshAuthorityProof.cs b/src/GameCult.Mesh/CultMeshAuthorityProof.cs new file mode 100644 index 00000000..d4b2d0c8 --- /dev/null +++ b/src/GameCult.Mesh/CultMeshAuthorityProof.cs @@ -0,0 +1,349 @@ +using System; +using System.Buffers.Binary; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Security.Cryptography; +using System.Text; +using GameCult.Networking; + +namespace GameCult.Mesh +{ + public enum CultMeshAuthorityTrustMode + { + AuthenticatedRemote, + LocalDevelopment + } + + public sealed class CultMeshEcdsaP256PublicKey + { + public CultMeshEcdsaP256PublicKey(string keyId, string x, string y) + { + KeyId = Require(keyId, nameof(keyId)); + X = RequireCoordinate(x, nameof(x)); + Y = RequireCoordinate(y, nameof(y)); + } + + public string KeyId { get; } + public string X { get; } + public string Y { get; } + + internal ECDsa CreateVerifier() + { + var verifier = ECDsa.Create(); + verifier.ImportParameters(new ECParameters + { + Curve = ECCurve.NamedCurves.nistP256, + Q = new ECPoint { X = Convert.FromBase64String(X), Y = Convert.FromBase64String(Y) } + }); + return verifier; + } + + public static CultMeshEcdsaP256PublicKey From(string keyId, ECDsa key) + { + if (key == null) throw new ArgumentNullException(nameof(key)); + var parameters = key.ExportParameters(false); + return new CultMeshEcdsaP256PublicKey( + keyId, + Convert.ToBase64String(parameters.Q.X ?? throw new InvalidOperationException("P-256 key has no X coordinate.")), + Convert.ToBase64String(parameters.Q.Y ?? throw new InvalidOperationException("P-256 key has no Y coordinate."))); + } + + private static string RequireCoordinate(string value, string name) + { + var required = Require(value, name); + byte[] bytes; + try { bytes = Convert.FromBase64String(required); } + catch (FormatException error) { throw new ArgumentException("P-256 coordinates must be base64.", name, error); } + if (bytes.Length != 32) throw new ArgumentException("P-256 coordinates must contain 32 bytes.", name); + return required; + } + + private static string Require(string value, string name) => + string.IsNullOrWhiteSpace(value) ? throw new ArgumentException("Value must be non-empty.", name) : value.Trim(); + } + + public sealed class CultMeshRouteCertificate + { + public CultMeshRouteCertificate( + CultMeshEcdsaP256PublicKey providerKey, + string odinKeyId, + long issuedAtUnixMilliseconds, + long expiresAtUnixMilliseconds, + string signature) + { + ProviderKey = providerKey ?? throw new ArgumentNullException(nameof(providerKey)); + OdinKeyId = string.IsNullOrWhiteSpace(odinKeyId) + ? throw new ArgumentException("Odin key identity is required.", nameof(odinKeyId)) + : odinKeyId.Trim(); + if (issuedAtUnixMilliseconds < 0) throw new ArgumentOutOfRangeException(nameof(issuedAtUnixMilliseconds)); + if (expiresAtUnixMilliseconds <= issuedAtUnixMilliseconds) throw new ArgumentOutOfRangeException(nameof(expiresAtUnixMilliseconds)); + IssuedAtUnixMilliseconds = issuedAtUnixMilliseconds; + ExpiresAtUnixMilliseconds = expiresAtUnixMilliseconds; + Signature = signature?.Trim() ?? string.Empty; + } + + public CultMeshEcdsaP256PublicKey ProviderKey { get; } + public string OdinKeyId { get; } + public long IssuedAtUnixMilliseconds { get; } + public long ExpiresAtUnixMilliseconds { get; } + public string Signature { get; } + } + + public sealed class CultMeshAuthorityTrustPolicy + { + private readonly Dictionary _odinRoots; + + public CultMeshAuthorityTrustPolicy( + CultMeshAuthorityTrustMode mode, + IEnumerable? odinRoots = null) + { + Mode = mode; + _odinRoots = (odinRoots ?? Array.Empty()) + .ToDictionary(key => key.KeyId, StringComparer.Ordinal); + } + + public CultMeshAuthorityTrustMode Mode { get; } + public IReadOnlyCollection OdinRoots => _odinRoots.Values; + + /// Fails closed unless the route is valid under this consumer-owned policy. + public void Validate(string verseId, CultMeshAuthorityRoute route, DateTimeOffset now) => + Verify(verseId, route, now); + + internal CultMeshVerifiedAuthorityRoute Verify( + string verseId, + CultMeshAuthorityRoute route, + DateTimeOffset now) + { + if (route == null) throw new ArgumentNullException(nameof(route)); + var certificate = route.Certificate; + if (certificate == null || string.IsNullOrWhiteSpace(certificate.Signature)) + { + if (Mode == CultMeshAuthorityTrustMode.LocalDevelopment && IsLoopback(route.Endpoint)) + return CultMeshVerifiedAuthorityRoute.Local(route); + throw Failure(route, "Remote CultMesh routes require an Odin-signed authority certificate."); + } + if (!IsProtected(route.Endpoint) && + !(Mode == CultMeshAuthorityTrustMode.LocalDevelopment && IsLoopback(route.Endpoint))) + throw Failure(route, "Authenticated remote CultMesh routes require TLS or QUIC channel protection."); + if (!_odinRoots.TryGetValue(certificate.OdinKeyId, out var root)) + throw Failure(route, $"Odin key '{certificate.OdinKeyId}' is not trusted by this consumer."); + var nowMs = now.ToUnixTimeMilliseconds(); + if (nowMs < certificate.IssuedAtUnixMilliseconds || nowMs >= certificate.ExpiresAtUnixMilliseconds) + throw Failure(route, "The Odin route certificate is not currently valid."); + byte[] signature; + try { signature = Convert.FromBase64String(certificate.Signature); } + catch (FormatException error) { throw Failure(route, "The Odin route signature is not base64.", error); } + if (signature.Length != 64) throw Failure(route, "The Odin route signature is not IEEE P1363 P-256."); + using var verifier = root.CreateVerifier(); + if (!verifier.VerifyData( + CultMeshAuthorityProof.CanonicalRoute(verseId, route), + signature, + HashAlgorithmName.SHA256)) + throw Failure(route, "The Odin route certificate signature is invalid."); + return CultMeshVerifiedAuthorityRoute.Authenticated(route, certificate.ProviderKey); + } + + private static bool IsLoopback(string endpoint) + { + if (!Uri.TryCreate(endpoint, UriKind.Absolute, out var uri)) return false; + return uri.IsLoopback || + string.Equals(uri.Host, "localhost", StringComparison.OrdinalIgnoreCase) || + string.Equals(uri.Host, "127.0.0.1", StringComparison.Ordinal) || + string.Equals(uri.Host, "::1", StringComparison.Ordinal); + } + + private static bool IsProtected(string endpoint) + { + if (!Uri.TryCreate(endpoint, UriKind.Absolute, out var uri)) return false; + return string.Equals(uri.Scheme, "wss", StringComparison.OrdinalIgnoreCase) || + string.Equals(uri.Scheme, "https", StringComparison.OrdinalIgnoreCase) || + uri.Scheme.IndexOf("quic", StringComparison.OrdinalIgnoreCase) >= 0; + } + + private static CultMeshSessionException Failure(CultMeshAuthorityRoute route, string message, Exception? inner = null) => + new CultMeshSessionException(new CultMeshSessionFailure( + CultMeshSessionFailureReason.Authentication, + message, + route.Endpoint), inner); + } + + internal sealed class CultMeshVerifiedAuthorityRoute + { + private CultMeshVerifiedAuthorityRoute( + CultMeshAuthorityRoute route, + CultMeshEcdsaP256PublicKey? providerKey, + bool localDevelopment) + { + Route = route; + ProviderKey = providerKey; + IsLocalDevelopment = localDevelopment; + } + + public CultMeshAuthorityRoute Route { get; } + public CultMeshEcdsaP256PublicKey? ProviderKey { get; } + public bool IsLocalDevelopment { get; } + public static CultMeshVerifiedAuthorityRoute Authenticated(CultMeshAuthorityRoute route, CultMeshEcdsaP256PublicKey key) => new(route, key, false); + public static CultMeshVerifiedAuthorityRoute Local(CultMeshAuthorityRoute route) => new(route, null, true); + } + + public sealed class CultMeshSessionProofSigner + { + private readonly ECDsa _key; + + public CultMeshSessionProofSigner(CultMeshAuthorityRoute route, ECDsa providerPrivateKey) + { + Route = route ?? throw new ArgumentNullException(nameof(route)); + _key = providerPrivateKey ?? throw new ArgumentNullException(nameof(providerPrivateKey)); + var certificate = route.Certificate ?? throw new ArgumentException("A session proof signer requires a certified route.", nameof(route)); + ProviderKeyId = certificate.ProviderKey.KeyId; + var exported = CultMeshEcdsaP256PublicKey.From(ProviderKeyId, _key); + if (!string.Equals(exported.X, certificate.ProviderKey.X, StringComparison.Ordinal) || + !string.Equals(exported.Y, certificate.ProviderKey.Y, StringComparison.Ordinal)) + throw new ArgumentException("Provider private key does not match the certified route key.", nameof(providerPrivateKey)); + } + + public CultMeshAuthorityRoute Route { get; } + public string ProviderKeyId { get; } + + public string Sign(CultMeshSessionOpenMessage request) + { + var bytes = CultMeshAuthorityProof.CanonicalSession(request, Route.Endpoint); + return Convert.ToBase64String(CultMeshAuthorityProof.SignP256(_key, bytes)); + } + } + + public static class CultMeshAuthorityProof + { + private const string RouteDomain = "gamecult.cultmesh.route-certificate.v1"; + private const string SessionDomain = "gamecult.cultmesh.session-proof.v1"; + + public static CultMeshAuthorityRoute CreateSignedRoute( + string verseId, + string authorityRuntimeId, + string endpoint, + IEnumerable protocolIds, + int priority, + string generation, + CultMeshEcdsaP256PublicKey providerKey, + string odinKeyId, + DateTimeOffset issuedAt, + DateTimeOffset expiresAt, + ECDsa odinPrivateKey) + { + if (odinPrivateKey == null) throw new ArgumentNullException(nameof(odinPrivateKey)); + var unsignedCertificate = new CultMeshRouteCertificate( + providerKey, + odinKeyId, + issuedAt.ToUnixTimeMilliseconds(), + expiresAt.ToUnixTimeMilliseconds(), + string.Empty); + var unsignedRoute = new CultMeshAuthorityRoute( + authorityRuntimeId, endpoint, protocolIds, priority, generation, unsignedCertificate); + var signature = Convert.ToBase64String(SignP256( + odinPrivateKey, + CanonicalRoute(verseId, unsignedRoute))); + return new CultMeshAuthorityRoute( + authorityRuntimeId, + endpoint, + protocolIds, + priority, + generation, + new CultMeshRouteCertificate( + providerKey, + odinKeyId, + issuedAt.ToUnixTimeMilliseconds(), + expiresAt.ToUnixTimeMilliseconds(), + signature)); + } + + /// Verifies both the Odin route certificate and provider nonce proof. + public static bool VerifySessionProof( + CultMeshSessionOpenMessage request, + CultMeshSessionAcceptedMessage response, + string verseId, + CultMeshAuthorityRoute route, + CultMeshAuthorityTrustPolicy trust, + DateTimeOffset now) + { + if (request == null) throw new ArgumentNullException(nameof(request)); + if (response == null) throw new ArgumentNullException(nameof(response)); + if (trust == null) throw new ArgumentNullException(nameof(trust)); + return VerifySession(request, response, trust.Verify(verseId, route, now)); + } + + internal static byte[] CanonicalRoute(string verseId, CultMeshAuthorityRoute route) + { + var certificate = route.Certificate ?? throw new InvalidOperationException("Route certificate is required."); + return Canonical( + RouteDomain, + verseId, + route.AuthorityRuntimeId, + route.Endpoint, + string.Join("\u001f", route.ProtocolIds.OrderBy(value => value, StringComparer.Ordinal)), + route.Priority.ToString(System.Globalization.CultureInfo.InvariantCulture), + route.Generation, + certificate.ProviderKey.KeyId, + certificate.ProviderKey.X, + certificate.ProviderKey.Y, + certificate.OdinKeyId, + certificate.IssuedAtUnixMilliseconds.ToString(System.Globalization.CultureInfo.InvariantCulture), + certificate.ExpiresAtUnixMilliseconds.ToString(System.Globalization.CultureInfo.InvariantCulture)); + } + + internal static byte[] CanonicalSession(CultMeshSessionOpenMessage request, string endpoint) => Canonical( + SessionDomain, + request.ClientNonce ?? string.Empty, + request.MessageId ?? string.Empty, + request.SourceRuntimeId ?? string.Empty, + request.VerseId ?? string.Empty, + request.AuthorityRuntimeId ?? string.Empty, + request.ProtocolId ?? string.Empty, + endpoint ?? string.Empty, + request.RouteGeneration ?? string.Empty); + + internal static bool VerifySession( + CultMeshSessionOpenMessage request, + CultMeshSessionAcceptedMessage response, + CultMeshVerifiedAuthorityRoute route) + { + if (route.IsLocalDevelopment) return true; + if (route.ProviderKey == null || + !string.Equals(response.ProviderKeyId, route.ProviderKey.KeyId, StringComparison.Ordinal) || + !string.Equals(response.ClientNonce, request.ClientNonce, StringComparison.Ordinal) || + string.IsNullOrWhiteSpace(response.ProviderSignature)) + return false; + byte[] signature; + try { signature = Convert.FromBase64String(response.ProviderSignature); } + catch (FormatException) { return false; } + if (signature.Length != 64) return false; + using var verifier = route.ProviderKey.CreateVerifier(); + return verifier.VerifyData( + CanonicalSession(request, route.Route.Endpoint), + signature, + HashAlgorithmName.SHA256); + } + + internal static byte[] SignP256(ECDsa key, byte[] payload) + { + var signature = key.SignData(payload, HashAlgorithmName.SHA256); + if (signature.Length != 64) + throw new CryptographicException("ECDSA provider did not emit IEEE P1363 P-256 signature bytes."); + return signature; + } + + private static byte[] Canonical(params string[] values) + { + using var stream = new MemoryStream(); + var length = new byte[4]; + foreach (var value in values) + { + var bytes = Encoding.UTF8.GetBytes(value ?? string.Empty); + BinaryPrimitives.WriteInt32BigEndian(length, bytes.Length); + stream.Write(length, 0, length.Length); + stream.Write(bytes, 0, bytes.Length); + } + return stream.ToArray(); + } + } +} diff --git a/src/GameCult.Mesh/CultMeshBodyDemand.cs b/src/GameCult.Mesh/CultMeshBodyDemand.cs index 55fa3807..2ab90617 100644 --- a/src/GameCult.Mesh/CultMeshBodyDemand.cs +++ b/src/GameCult.Mesh/CultMeshBodyDemand.cs @@ -225,9 +225,11 @@ namespace GameCult.Mesh { internal CultMeshBodyDemandPlan( string bodyId, + long generation, IReadOnlyList consumers) { BodyId = bodyId; + Generation = generation; Consumers = consumers; RequiresSharedMemory = consumers.Any(value => value.Transport == CultMeshBodyTransportKind.SharedMemory); RequiresSharedFileMapping = consumers.Any(value => value.Transport == CultMeshBodyTransportKind.SharedFileMapping); @@ -235,6 +237,11 @@ namespace GameCult.Mesh } public string BodyId { get; } + /// + /// Monotone generation of the exact routed consumer set. Publishers must rotate any + /// retained capability before writing after this value changes. + /// + public long Generation { get; } public bool HasConsumers => Consumers.Count > 0; public bool RequiresSharedMemory { get; } public bool RequiresSharedFileMapping { get; } @@ -271,6 +278,8 @@ namespace GameCult.Mesh private readonly object _gate = new(); private readonly Dictionary _demands = new(StringComparer.Ordinal); + private readonly Dictionary _bodyGenerations = + new(StringComparer.Ordinal); private bool _disposed; public CultMeshBodyDemandTracker(CultNetDatabaseSubscriptionServer subscriptions) @@ -298,7 +307,10 @@ namespace GameCult.Mesh .OrderBy(value => value.ConsumerRuntimeId, StringComparer.Ordinal) .ThenBy(value => value.SubscriptionId, StringComparer.Ordinal) .ToArray(); - return new CultMeshBodyDemandPlan(bodyId, routes); + return new CultMeshBodyDemandPlan( + bodyId, + _bodyGenerations.TryGetValue(bodyId, out var generation) ? generation : 0, + routes); } } @@ -310,6 +322,7 @@ namespace GameCult.Mesh _disposed = true; if (_subscriptions != null) _subscriptions.DemandChanged -= OnDemandChanged; _demands.Clear(); + _bodyGenerations.Clear(); } } @@ -322,11 +335,63 @@ namespace GameCult.Mesh lock (_gate) { if (_disposed) return; + _demands.TryGetValue(key, out var previous); + var affectedBodyIds = (previous?.BodyIds ?? Array.Empty()) + .Concat(demand.BodyIds) + .Where(value => !string.IsNullOrWhiteSpace(value)) + .Distinct(StringComparer.Ordinal) + .ToArray(); + var before = affectedBodyIds.ToDictionary( + bodyId => bodyId, + RoutedConsumerSignature, + StringComparer.Ordinal); if (demand.Active) _demands[key] = demand; else _demands.Remove(key); + foreach (var bodyId in affectedBodyIds) + { + var after = RoutedConsumerSignature(bodyId); + if (string.Equals(before[bodyId], after, StringComparison.Ordinal)) continue; + _bodyGenerations[bodyId] = checked( + (_bodyGenerations.TryGetValue(bodyId, out var generation) ? generation : 0) + 1); + } } } + /// + /// Executes one synchronous capability commit only while the exact routed consumer + /// generation remains current. Demand transitions serialize against the callback, so + /// either this work linearizes before the transition or it is rejected without running. + /// + public bool TryExecuteAtGeneration(string bodyId, long generation, Action action) + { + if (string.IsNullOrWhiteSpace(bodyId)) throw new ArgumentException("Body identity is required.", nameof(bodyId)); + if (generation < 0) throw new ArgumentOutOfRangeException(nameof(generation)); + if (action == null) throw new ArgumentNullException(nameof(action)); + lock (_gate) + { + ThrowIfDisposed(); + var current = _bodyGenerations.TryGetValue(bodyId, out var value) ? value : 0; + if (current != generation) return false; + action(); + return true; + } + } + + private string RoutedConsumerSignature(string bodyId) => string.Join( + "\u001e", + _demands.Values + .Where(value => value.BodyIds.Contains(bodyId, StringComparer.Ordinal)) + .Select(Route) + .Where(value => value != null) + .Cast() + .OrderBy(value => value.ConsumerRuntimeId, StringComparer.Ordinal) + .ThenBy(value => value.SubscriptionId, StringComparer.Ordinal) + .Select(value => string.Join( + "\u001f", + value.ConsumerRuntimeId, + value.SubscriptionId, + value.Transport.ToString()))); + private void OnDemandChanged(CultNetDatabaseSubscriptionDemand demand) => Observe(demand); private static CultMeshBodyConsumerRoute? Route(CultNetDatabaseSubscriptionDemand demand) diff --git a/src/GameCult.Mesh/CultMeshBodySessions.cs b/src/GameCult.Mesh/CultMeshBodySessions.cs index cc45277e..ee946201 100644 --- a/src/GameCult.Mesh/CultMeshBodySessions.cs +++ b/src/GameCult.Mesh/CultMeshBodySessions.cs @@ -278,15 +278,15 @@ namespace GameCult.Mesh public sealed class CultMeshSessionBodyProvider { private readonly CultMeshSessionManager _sessions; - private readonly CultMeshEndpointId _endpointId; + private readonly CultMeshSessionTarget _target; private readonly CultMeshSessionBodyProviderOptions _options; public CultMeshSessionBodyProvider(string providerId, CultMeshSessionManager sessions, - CultMeshEndpointId endpointId, CultMeshSessionBodyProviderOptions? options = null) + CultMeshSessionTarget target, CultMeshSessionBodyProviderOptions? options = null) { ProviderId = string.IsNullOrWhiteSpace(providerId) ? throw new ArgumentException("Provider identity is required.", nameof(providerId)) : providerId; _sessions = sessions ?? throw new ArgumentNullException(nameof(sessions)); - _endpointId = endpointId ?? throw new ArgumentNullException(nameof(endpointId)); + _target = target ?? throw new ArgumentNullException(nameof(target)); _options = options ?? new CultMeshSessionBodyProviderOptions(); if (_options.ResponseTimeout <= TimeSpan.Zero) throw new ArgumentOutOfRangeException(nameof(options)); } @@ -300,7 +300,7 @@ namespace GameCult.Mesh if (descriptor == null) throw new ArgumentNullException(nameof(descriptor)); if (descriptor.TransportKind != CultMeshBodyTransportKind.Network) throw new NotSupportedException("Direct body provider requires a network descriptor."); - var session = await _sessions.ConnectAsync(_endpointId, CultMeshProtocols.Bodies, cancellationToken).ConfigureAwait(false); + var session = await _sessions.ConnectAsync(_target, CultMeshProtocols.Bodies, cancellationToken).ConfigureAwait(false); var messageId = Guid.NewGuid().ToString("N"); var completion = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); using var responseSubscription = session.OnCultNet(response => diff --git a/src/GameCult.Mesh/CultMeshClient.cs b/src/GameCult.Mesh/CultMeshClient.cs index 12ccd470..5db03ca5 100644 --- a/src/GameCult.Mesh/CultMeshClient.cs +++ b/src/GameCult.Mesh/CultMeshClient.cs @@ -6,6 +6,7 @@ using System.Threading; using System.Threading.Tasks; using GameCult.Caching; using GameCult.Networking; +using MessagePack; using R3; namespace GameCult.Mesh @@ -26,8 +27,8 @@ namespace GameCult.Mesh public IReadOnlyList? Connectors { get; set; } /// - /// Gets or sets streaming content connectors. TCP content delivery is used by default; - /// applications may replace it or explicitly add legacy RUDP. + /// Gets or sets streaming content connectors. HTTPS and loopback TCP content + /// delivery are used by default; applications may replace them or explicitly add legacy RUDP. /// public IReadOnlyList? ContentConnectors { get; set; } @@ -40,6 +41,102 @@ namespace GameCult.Mesh /// Gets or sets the deadline before an unanswered subscription intent is replayed. public TimeSpan SubscriptionResponseTimeout { get; set; } = TimeSpan.FromSeconds(2); + + /// Gets or sets the deadline for one correlated typed operation response. + public TimeSpan OperationResponseTimeout { get; set; } = TimeSpan.FromSeconds(10); + } + + /// + /// Carries one provider-authored operation response. Transport delivery does not by itself + /// establish that the provider accepted or committed the requested gameplay change. + /// + public sealed class CultMeshOperationResult + { + internal CultMeshOperationResult(CultNetOperationResponseMessage envelope, TResponse value) + { + MessageId = envelope.MessageId; + ServiceId = envelope.ServiceId; + Operation = envelope.Operation; + Status = envelope.Status; + PayloadSchema = envelope.PayloadSchema; + SourceRuntimeId = envelope.SourceRuntimeId; + Diagnostics = envelope.Diagnostics ?? Array.Empty(); + Value = value; + } + + /// Gets the durable caller-supplied idempotency key. + public string MessageId { get; } + /// Gets the provider service that answered. + public string ServiceId { get; } + /// Gets the operation that answered. + public string Operation { get; } + /// Gets the provider-authored operation status. + public string Status { get; } + /// Gets the portable response payload schema. + public string PayloadSchema { get; } + /// Gets the responding runtime identity, when supplied. + public string? SourceRuntimeId { get; } + /// Gets provider or transport diagnostics carried by the response. + public IReadOnlyList Diagnostics { get; } + /// Gets the decoded typed response payload. + public TResponse Value { get; } + } + + /// Correlated framework-level failure returned before a typed domain reply exists. + public sealed class CultMeshRemoteOperationException : Exception + { + internal CultMeshRemoteOperationException( + CultNetOperationResponseMessage envelope, + CultNetOperationFailure failure) + : base(failure?.Message ?? "CultMesh operation failed.") + { + Status = envelope?.Status ?? "error"; + Code = failure?.Code ?? "operation-error"; + Diagnostics = envelope?.Diagnostics ?? Array.Empty(); + } + + /// Gets the provider-authored failure status. + public string Status { get; } + + /// Gets the stable machine-readable failure code. + public string Code { get; } + + /// Gets provider diagnostics carried by the response. + public IReadOnlyList Diagnostics { get; } + } + + /// Owns one shared live document subscription until disposed. + public sealed class CultMeshDocumentLease : IDisposable where TDocument : class + { + private Action? _release; + + internal CultMeshDocumentLease(CultMeshDocumentHandle handle, Action release) + { + Handle = handle ?? throw new ArgumentNullException(nameof(handle)); + _release = release ?? throw new ArgumentNullException(nameof(release)); + } + + /// Gets the live typed document handle owned by this lease. + public CultMeshDocumentHandle Handle { get; } + + public void Dispose() => Interlocked.Exchange(ref _release, null)?.Invoke(); + } + + /// Owns one shared live collection subscription until disposed. + public sealed class CultMeshCollectionLease : IDisposable where TDocument : class + { + private Action? _release; + + internal CultMeshCollectionLease(CultMeshCollectionHandle handle, Action release) + { + Handle = handle ?? throw new ArgumentNullException(nameof(handle)); + _release = release ?? throw new ArgumentNullException(nameof(release)); + } + + /// Gets the live typed collection handle owned by this lease. + public CultMeshCollectionHandle Handle { get; } + + public void Dispose() => Interlocked.Exchange(ref _release, null)?.Invoke(); } /// @@ -51,9 +148,9 @@ namespace GameCult.Mesh private readonly CultMeshDiscoveryService _discovery; private readonly CultMeshSessionManager _sessions; private readonly TimeSpan _subscriptionResponseTimeout; - private readonly ConcurrentDictionary>> _documents = new(StringComparer.Ordinal); - private readonly ConcurrentDictionary>> _collections = new(StringComparer.Ordinal); - private readonly ConcurrentBag _documentResources = new(); + private readonly TimeSpan _operationResponseTimeout; + private readonly ConcurrentDictionary _documents = new(StringComparer.Ordinal); + private readonly ConcurrentDictionary _collections = new(StringComparer.Ordinal); private bool _disposed; public CultMeshClient(CultMeshClientOptions options) @@ -61,6 +158,8 @@ namespace GameCult.Mesh if (options == null) throw new ArgumentNullException(nameof(options)); if (options.SubscriptionResponseTimeout <= TimeSpan.Zero) throw new ArgumentOutOfRangeException(nameof(options.SubscriptionResponseTimeout)); + if (options.OperationResponseTimeout <= TimeSpan.Zero) + throw new ArgumentOutOfRangeException(nameof(options.OperationResponseTimeout)); var endpoints = options.RendezvousEndpoints .Where(endpoint => !string.IsNullOrWhiteSpace(endpoint)) .Distinct(StringComparer.Ordinal) @@ -85,195 +184,410 @@ namespace GameCult.Mesh }, options.ContentConnectors ?? new ICultMeshContentTransportConnector[] { + new CultMeshHttpsContentTransportConnector(), new CultMeshTcpContentTransportConnector() }, options.RealtimeConnectors, options.Sessions); _subscriptionResponseTimeout = options.SubscriptionResponseTimeout; + _operationResponseTimeout = options.OperationResponseTimeout; } - /// Connects to a stable endpoint identity using one application protocol. + /// Connects to one stable Verse/provider target using one application protocol. public Task ConnectAsync( - string endpointId, + CultMeshSessionTarget target, CultMeshProtocolId protocol, CancellationToken cancellationToken = default) { ThrowIfDisposed(); - return _sessions.ConnectAsync(CultMeshEndpointId.Parse(endpointId), protocol, cancellationToken); + return _sessions.ConnectAsync(target, protocol, cancellationToken); } - /// Connects the reusable realtime state plane by stable endpoint identity. + /// Connects the reusable realtime state plane for one stable Verse/provider target. public Task ConnectRealtimeAsync( - string endpointId, + CultMeshSessionTarget target, CancellationToken cancellationToken = default) { ThrowIfDisposed(); - return _sessions.ConnectRealtimeAsync(CultMeshEndpointId.Parse(endpointId), cancellationToken); - } - - /// Connects to a stable endpoint identity using one application protocol. - public Task ConnectAsync( - CultMeshEndpointId endpointId, - CultMeshProtocolId protocol, - CancellationToken cancellationToken = default) - { - ThrowIfDisposed(); - return _sessions.ConnectAsync(endpointId, protocol, cancellationToken); + return _sessions.ConnectRealtimeAsync(target, cancellationToken); } /// Creates a verified-transfer provider over this client's reusable content session owner. public CultMeshSessionContentProvider ContentProvider( string providerId, - string endpointId) + CultMeshSessionTarget target) { ThrowIfDisposed(); return new CultMeshSessionContentProvider( providerId, _sessions, - CultMeshEndpointId.Parse(endpointId)); + target); } /// Creates a direct body provider over this client's reusable negotiated session owner. public CultMeshSessionBodyProvider BodyProvider( string providerId, - string endpointId, + CultMeshSessionTarget target, CultMeshSessionBodyProviderOptions? options = null) { ThrowIfDisposed(); return new CultMeshSessionBodyProvider( providerId, _sessions, - CultMeshEndpointId.Parse(endpointId), + target, options); } - /// Opens one live typed document by stable provider identity and record key. - public async Task> DocumentAsync( - string endpointId, + /// Leases one shared live typed document by stable Verse/provider identity and record key. + public async Task> LeaseDocumentAsync( + CultMeshSessionTarget target, string recordKey, CancellationToken cancellationToken = default) where TDocument : class { ThrowIfDisposed(); - if (string.IsNullOrWhiteSpace(endpointId)) throw new ArgumentException("Endpoint identity is required.", nameof(endpointId)); + if (target == null) throw new ArgumentNullException(nameof(target)); if (string.IsNullOrWhiteSpace(recordKey)) throw new ArgumentException("Record key is required.", nameof(recordKey)); - var key = endpointId.Trim() + "\u001f" + typeof(TDocument).AssemblyQualifiedName + "\u001f" + recordKey.Trim(); - var lazy = _documents.GetOrAdd(key, _ => new Lazy>( - async () => await OpenDocumentAsync(endpointId.Trim(), recordKey.Trim()).ConfigureAwait(false), - LazyThreadSafetyMode.ExecutionAndPublication)); - try + var key = target.SessionKey + "\u001f" + typeof(TDocument).AssemblyQualifiedName + "\u001f" + recordKey.Trim(); + while (true) { - var binding = (RemoteDocumentBinding)await AwaitForCallerAsync(lazy.Value, cancellationToken) - .ConfigureAwait(false); - return binding.Handle; - } - catch - { - _documents.TryRemove(key, out _); - throw; + var resource = _documents.GetOrAdd(key, _ => new SharedResource( + async owner => await OpenDocumentAsync(target, recordKey.Trim(), owner).ConfigureAwait(false))); + if (!resource.TryAcquire()) + { + RemoveExact(_documents, key, resource); + continue; + } + try + { + var binding = (RemoteDocumentBinding)await AwaitForCallerAsync(resource.Value, cancellationToken) + .ConfigureAwait(false); + return new CultMeshDocumentLease( + binding.Handle, + () => Release(_documents, key, resource)); + } + catch + { + Release(_documents, key, resource); + throw; + } } } - /// Opens a live collection of one typed schema by stable provider identity. - public Task> CollectionAsync( - string endpointId, + /// Leases a shared live collection by stable Verse/provider identity. + public Task> LeaseCollectionAsync( + CultMeshSessionTarget target, CancellationToken cancellationToken) where TDocument : class => - CollectionAsync(endpointId, includeInitialSnapshot: true, cancellationToken); + LeaseCollectionAsync(target, includeInitialSnapshot: true, cancellationToken); - /// Opens a live collection and optionally skips replaying its initial snapshot. - public async Task> CollectionAsync( - string endpointId, + /// Leases a live collection and optionally skips replaying its initial snapshot. + public async Task> LeaseCollectionAsync( + CultMeshSessionTarget target, bool includeInitialSnapshot = true, CancellationToken cancellationToken = default) where TDocument : class { ThrowIfDisposed(); - if (string.IsNullOrWhiteSpace(endpointId)) throw new ArgumentException("Endpoint identity is required.", nameof(endpointId)); - var identity = endpointId.Trim(); + if (target == null) throw new ArgumentNullException(nameof(target)); + var identity = target.SessionKey; var key = identity + "\u001f" + typeof(TDocument).AssemblyQualifiedName + "\u001f" + includeInitialSnapshot; - var lazy = _collections.GetOrAdd(key, _ => new Lazy>( - async () => await OpenCollectionAsync(identity, includeInitialSnapshot).ConfigureAwait(false), - LazyThreadSafetyMode.ExecutionAndPublication)); - try + while (true) { - var binding = (RemoteCollectionBinding)await AwaitForCallerAsync(lazy.Value, cancellationToken) - .ConfigureAwait(false); - return binding.Handle; - } - catch - { - _collections.TryRemove(key, out _); - throw; + var resource = _collections.GetOrAdd(key, _ => new SharedResource( + async owner => await OpenCollectionAsync(target, includeInitialSnapshot, owner).ConfigureAwait(false))); + if (!resource.TryAcquire()) + { + RemoveExact(_collections, key, resource); + continue; + } + try + { + var binding = (RemoteCollectionBinding)await AwaitForCallerAsync(resource.Value, cancellationToken) + .ConfigureAwait(false); + return new CultMeshCollectionLease( + binding.Handle, + () => Release(_collections, key, resource)); + } + catch + { + Release(_collections, key, resource); + throw; + } } } + /// + /// Submits one typed document to a provider by stable identity. Submission is uncommitted input; + /// only a provider-authored receipt or resulting state can establish acceptance. + /// + public async Task SubmitDocumentAsync( + CultMeshSessionTarget target, + string recordKey, + TDocument document, + string sourceRuntimeId, + string sourceRole, + CancellationToken cancellationToken = default) + where TDocument : class + { + ThrowIfDisposed(); + if (target == null) throw new ArgumentNullException(nameof(target)); + if (string.IsNullOrWhiteSpace(recordKey)) throw new ArgumentException("Record key is required.", nameof(recordKey)); + if (document == null) throw new ArgumentNullException(nameof(document)); + if (string.IsNullOrWhiteSpace(sourceRuntimeId)) throw new ArgumentException("Source runtime identity is required.", nameof(sourceRuntimeId)); + if (string.IsNullOrWhiteSpace(sourceRole)) throw new ArgumentException("Source role is required.", nameof(sourceRole)); + + var cacheRegistry = CultMesh.CreateCultCacheDocumentRegistry(typeof(TDocument)); + var networkRegistry = CultMesh.CreateCultNetDocumentRegistry(new[] { typeof(TDocument) }, cacheRegistry); + var message = networkRegistry.CreateRawDocumentPutMessage( + "cultmesh-client-submit-" + Guid.NewGuid().ToString("N"), + new CultRecordHandle(new CultRecordKey(recordKey.Trim())), + document, + new CultNetDocumentMessageOptions + { + SourceRuntimeId = sourceRuntimeId.Trim(), + SourceRole = sourceRole.Trim() + }); + var session = await ConnectAsync(target, CultMeshProtocols.Documents, cancellationToken) + .ConfigureAwait(false); + cancellationToken.ThrowIfCancellationRequested(); + session.OpenSchemaClient().SendCultNet(message); + } + + /// + /// Invokes one typed provider operation through the reusable identity-owned session. + /// The caller supplies a durable idempotency key; the returned status and payload remain + /// provider-authored evidence rather than client-side authority over canonical state. + /// + public async Task> InvokeAsync( + CultMeshSessionTarget target, + string serviceId, + string operation, + string requestSchema, + string responseSchema, + TRequest request, + string sourceRuntimeId, + string idempotencyKey, + CancellationToken cancellationToken = default) + { + ThrowIfDisposed(); + if (target == null) throw new ArgumentNullException(nameof(target)); + if (string.IsNullOrWhiteSpace(serviceId)) throw new ArgumentException("Service identity is required.", nameof(serviceId)); + if (string.IsNullOrWhiteSpace(operation)) throw new ArgumentException("Operation identity is required.", nameof(operation)); + if (string.IsNullOrWhiteSpace(requestSchema)) throw new ArgumentException("Request schema is required.", nameof(requestSchema)); + if (string.IsNullOrWhiteSpace(responseSchema)) throw new ArgumentException("Response schema is required.", nameof(responseSchema)); + if (request is null) throw new ArgumentNullException(nameof(request)); + if (string.IsNullOrWhiteSpace(sourceRuntimeId)) throw new ArgumentException("Source runtime identity is required.", nameof(sourceRuntimeId)); + if (string.IsNullOrWhiteSpace(idempotencyKey)) throw new ArgumentException("A durable idempotency key is required.", nameof(idempotencyKey)); + + var expectedService = serviceId.Trim(); + var expectedOperation = operation.Trim(); + var expectedResponseSchema = responseSchema.Trim(); + var messageId = idempotencyKey.Trim(); + var message = new CultNetOperationRequestMessage + { + MessageId = messageId, + ServiceId = expectedService, + Operation = expectedOperation, + PayloadSchema = requestSchema.Trim(), + PayloadEncoding = "messagepack-base64", + Payload = Convert.ToBase64String(MessagePackSerializer.Serialize(request, CultNetSchemaMessageSerialization.Options)), + SourceRuntimeId = sourceRuntimeId.Trim(), + TargetRuntimeId = target.AuthorityRuntimeId + }; + using var timeout = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + timeout.CancelAfter(_operationResponseTimeout); + CultNetOperationResponseMessage envelope; + try + { + while (true) + { + var session = await ConnectOnlineAsync(target, timeout.Token).ConfigureAwait(false); + var completion = new TaskCompletionSource( + TaskCreationOptions.RunContinuationsAsynchronously); + var pathChanged = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var generation = session.PhysicalGeneration; + using var responseSubscription = session.OnCultNet(response => + { + if (string.Equals(response.MessageId, messageId, StringComparison.Ordinal)) + completion.TrySetResult(response); + }); + using var stateSubscription = session.WatchState().Subscribe(state => + { + if (state.Status != CultMeshSessionStatus.Online || session.PhysicalGeneration != generation) + pathChanged.TrySetResult(true); + }); + using var cancellation = timeout.Token.Register(() => completion.TrySetCanceled(timeout.Token)); + if (session.State.Status != CultMeshSessionStatus.Online || + session.PhysicalGeneration != generation) + continue; + + session.SendCultNet(message); + var finished = await Task.WhenAny(completion.Task, pathChanged.Task).ConfigureAwait(false); + if (finished == completion.Task) + { + envelope = await completion.Task.ConfigureAwait(false); + break; + } + // The operation has a durable idempotency key. A route loss before a + // correlated response replays the same request after replacement finality. + } + } + catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested) + { + throw new TimeoutException( + $"CultMesh operation '{expectedService}/{expectedOperation}' did not answer idempotency key '{messageId}' " + + $"within {_operationResponseTimeout}."); + } + cancellationToken.ThrowIfCancellationRequested(); + if (!string.Equals(envelope.ServiceId, expectedService, StringComparison.Ordinal) || + !string.Equals(envelope.Operation, expectedOperation, StringComparison.Ordinal)) + throw new InvalidOperationException("CultMesh operation response did not match the requested service and operation."); + if (!string.Equals(envelope.SourceRuntimeId, target.AuthorityRuntimeId, StringComparison.Ordinal)) + throw new CultMeshSessionException(new CultMeshSessionFailure( + CultMeshSessionFailureReason.Authority, + $"CultMesh operation response came from runtime '{envelope.SourceRuntimeId ?? "(missing)"}', " + + $"not selected authority '{target.AuthorityRuntimeId}'.")); + if (string.Equals(envelope.PayloadSchema, CultNetOperationServer.FailureSchemaId, StringComparison.Ordinal)) + { + if (!string.Equals(envelope.PayloadEncoding, "messagepack-base64", StringComparison.Ordinal)) + throw new InvalidOperationException( + $"CultMesh operation failure returned unsupported payload encoding '{envelope.PayloadEncoding}'."); + CultNetOperationFailure failure; + try + { + failure = MessagePackSerializer.Deserialize( + Convert.FromBase64String(envelope.Payload), + CultNetSchemaMessageSerialization.Options); + } + catch (Exception error) when (error is FormatException || error is MessagePackSerializationException) + { + throw new InvalidOperationException("CultMesh operation failure payload was malformed.", error); + } + throw new CultMeshRemoteOperationException(envelope, failure); + } + if (!string.Equals(envelope.PayloadSchema, expectedResponseSchema, StringComparison.Ordinal)) + throw new InvalidOperationException( + $"CultMesh operation returned payload schema '{envelope.PayloadSchema}', expected '{expectedResponseSchema}'."); + if (!string.Equals(envelope.PayloadEncoding, "messagepack-base64", StringComparison.Ordinal)) + throw new InvalidOperationException( + $"CultMesh operation returned unsupported payload encoding '{envelope.PayloadEncoding}'."); + if (string.IsNullOrWhiteSpace(envelope.Status)) + throw new InvalidOperationException("CultMesh operation response omitted its provider-authored status."); + + TResponse value; + try + { + value = MessagePackSerializer.Deserialize( + Convert.FromBase64String(envelope.Payload), + CultNetSchemaMessageSerialization.Options); + } + catch (FormatException error) + { + throw new InvalidOperationException("CultMesh operation response payload was not valid base64.", error); + } + if (value is null) throw new InvalidOperationException("CultMesh operation response decoded to null."); + return new CultMeshOperationResult(envelope, value); + } + + private async Task ConnectOnlineAsync( + CultMeshSessionTarget target, + CancellationToken cancellationToken) + { + while (true) + { + var session = await ConnectAsync(target, CultMeshProtocols.Documents, cancellationToken) + .ConfigureAwait(false); + if (session.State.Status == CultMeshSessionStatus.Online) + return session; + + var changed = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + using var subscription = session.WatchState().Subscribe(state => + { + if (state.Status == CultMeshSessionStatus.Online || state.Status == CultMeshSessionStatus.Offline) + changed.TrySetResult(true); + }); + using var cancellation = cancellationToken.Register(() => changed.TrySetCanceled(cancellationToken)); + if (session.State.Status != CultMeshSessionStatus.Reconnecting) + continue; + await changed.Task.ConfigureAwait(false); + if (session.State.Status == CultMeshSessionStatus.Online) + return session; + } + } + + /// Gets the number of currently leased shared document subscriptions. + public int ActiveDocumentResourceCount => _documents.Count; + + /// Gets the number of currently leased shared collection subscriptions. + public int ActiveCollectionResourceCount => _collections.Count; + /// Reads one typed record once through the reusable document session. public async Task ReadAsync( - string endpointId, + CultMeshSessionTarget target, string recordKey, CancellationToken cancellationToken = default) where TDocument : class { ThrowIfDisposed(); - if (string.IsNullOrWhiteSpace(endpointId)) throw new ArgumentException("Endpoint identity is required.", nameof(endpointId)); + if (target == null) throw new ArgumentNullException(nameof(target)); if (string.IsNullOrWhiteSpace(recordKey)) throw new ArgumentException("Record key is required.", nameof(recordKey)); - var documents = await ReadManyAsync(endpointId, new[] { recordKey }, cancellationToken) + var documents = await ReadManyAsync(target, new[] { recordKey }, cancellationToken) .ConfigureAwait(false); return documents.FirstOrDefault() ?? throw new InvalidOperationException( - $"CultMesh endpoint '{endpointId}' did not publish {typeof(TDocument).FullName} record '{recordKey}'."); + $"CultMesh target '{target}' did not publish {typeof(TDocument).FullName} record '{recordKey}'."); } /// Reads one typed record once with an explicit response deadline. public async Task ReadAsync( - string endpointId, + CultMeshSessionTarget target, string recordKey, TimeSpan responseTimeout, CancellationToken cancellationToken = default) where TDocument : class { var documents = await ReadManyCoreAsync( - endpointId, + target, new[] { recordKey }, responseTimeout, cancellationToken) .ConfigureAwait(false); return documents.FirstOrDefault() ?? throw new InvalidOperationException( - $"CultMesh endpoint '{endpointId}' did not publish {typeof(TDocument).FullName} record '{recordKey}'."); + $"CultMesh target '{target}' did not publish {typeof(TDocument).FullName} record '{recordKey}'."); } /// Reads typed records once in one request through the reusable document session. public async Task> ReadManyAsync( - string endpointId, + CultMeshSessionTarget target, IReadOnlyList recordKeys, CancellationToken cancellationToken = default) where TDocument : class => await ReadManyCoreAsync( - endpointId, + target, recordKeys, TimeSpan.FromSeconds(10), cancellationToken) .ConfigureAwait(false); private async Task> ReadManyCoreAsync( - string endpointId, + CultMeshSessionTarget target, IReadOnlyList recordKeys, TimeSpan responseTimeout, CancellationToken cancellationToken) where TDocument : class { ThrowIfDisposed(); - if (string.IsNullOrWhiteSpace(endpointId)) throw new ArgumentException("Endpoint identity is required.", nameof(endpointId)); + if (target == null) throw new ArgumentNullException(nameof(target)); if (recordKeys == null || recordKeys.Count == 0 || recordKeys.Any(string.IsNullOrWhiteSpace)) throw new ArgumentException("At least one non-empty record key is required.", nameof(recordKeys)); var cacheRegistry = CultMesh.CreateCultCacheDocumentRegistry(typeof(TDocument)); var networkRegistry = CultMesh.CreateCultNetDocumentRegistry(new[] { typeof(TDocument) }, cacheRegistry); using var snapshot = await CultMeshSnapshotSession.ConnectAsync( _sessions, - CultMeshEndpointId.Parse(endpointId), - CultMeshProtocols.Content, + target, + CultMeshProtocols.Documents, new CultMeshSnapshotRequestOptions { ConnectTimeout = TimeSpan.FromSeconds(5), @@ -293,7 +607,7 @@ namespace GameCult.Mesh { if (_disposed) return; _disposed = true; - foreach (var resource in _documentResources) resource.Dispose(); + foreach (var resource in _documents.Values.Concat(_collections.Values).Distinct()) resource.Dispose(); _documents.Clear(); _collections.Clear(); _sessions.Dispose(); @@ -305,12 +619,15 @@ namespace GameCult.Mesh if (_disposed) throw new ObjectDisposedException(nameof(CultMeshClient)); } - private async Task OpenDocumentAsync(string endpointId, string recordKey) + private async Task OpenDocumentAsync( + CultMeshSessionTarget target, + string recordKey, + SharedResource owner) where TDocument : class { - var session = await ConnectAsync(endpointId, CultMeshProtocols.Documents).ConfigureAwait(false); - var binding = new RemoteDocumentBinding(session, endpointId, recordKey, _subscriptionResponseTimeout); - _documentResources.Add(binding); + var session = await ConnectAsync(target, CultMeshProtocols.Documents).ConfigureAwait(false); + var binding = new RemoteDocumentBinding(session, target, recordKey, _subscriptionResponseTimeout); + owner.Attach(binding); try { await binding.StartAsync().ConfigureAwait(false); @@ -318,17 +635,21 @@ namespace GameCult.Mesh } catch { + owner.Detach(binding); binding.Dispose(); throw; } } - private async Task OpenCollectionAsync(string endpointId, bool includeInitialSnapshot) + private async Task OpenCollectionAsync( + CultMeshSessionTarget target, + bool includeInitialSnapshot, + SharedResource owner) where TDocument : class { - var session = await ConnectAsync(endpointId, CultMeshProtocols.Documents).ConfigureAwait(false); - var binding = new RemoteCollectionBinding(session, endpointId, includeInitialSnapshot, _subscriptionResponseTimeout); - _documentResources.Add(binding); + var session = await ConnectAsync(target, CultMeshProtocols.Documents).ConfigureAwait(false); + var binding = new RemoteCollectionBinding(session, target, includeInitialSnapshot, _subscriptionResponseTimeout); + owner.Attach(binding); try { await binding.StartAsync().ConfigureAwait(false); @@ -336,6 +657,7 @@ namespace GameCult.Mesh } catch { + owner.Detach(binding); binding.Dispose(); throw; } @@ -353,6 +675,99 @@ namespace GameCult.Mesh return await shared.ConfigureAwait(false); } + private static void Release( + ConcurrentDictionary resources, + string key, + SharedResource resource) + { + if (!resource.Release()) return; + RemoveExact(resources, key, resource); + resource.Dispose(); + } + + private static void RemoveExact( + ConcurrentDictionary resources, + string key, + SharedResource resource) + { + ((ICollection>)resources) + .Remove(new KeyValuePair(key, resource)); + } + + private sealed class SharedResource : IDisposable + { + private readonly object _gate = new(); + private readonly Lazy> _value; + private IDisposable? _attached; + private int _leases; + private bool _retired; + + public SharedResource(Func> open) + { + if (open == null) throw new ArgumentNullException(nameof(open)); + _value = new Lazy>(() => open(this), LazyThreadSafetyMode.ExecutionAndPublication); + } + + public Task Value => _value.Value; + + public bool TryAcquire() + { + lock (_gate) + { + if (_retired) return false; + _leases++; + return true; + } + } + + public bool Release() + { + lock (_gate) + { + if (_leases <= 0) return false; + _leases--; + if (_leases != 0) return false; + _retired = true; + return true; + } + } + + public void Attach(IDisposable resource) + { + if (resource == null) throw new ArgumentNullException(nameof(resource)); + lock (_gate) + { + if (_retired) + { + resource.Dispose(); + throw new ObjectDisposedException(nameof(SharedResource)); + } + if (_attached != null) throw new InvalidOperationException("CultMesh shared resource already has a live binding."); + _attached = resource; + } + } + + public void Detach(IDisposable resource) + { + lock (_gate) + { + if (ReferenceEquals(_attached, resource)) _attached = null; + } + } + + public void Dispose() + { + IDisposable? attached; + lock (_gate) + { + _retired = true; + attached = _attached; + _attached = null; + } + attached?.Dispose(); + } + } + private sealed class RemoteDocumentBinding : IDisposable where TDocument : class { private readonly CultMeshSession _session; @@ -371,14 +786,14 @@ namespace GameCult.Mesh public RemoteDocumentBinding( CultMeshSession session, - string endpointId, + CultMeshSessionTarget target, string recordKey, TimeSpan responseTimeout) { _session = session; _responseTimeout = responseTimeout; _recordKey = recordKey; - _subscriptionId = "cultmesh-document:" + endpointId + ":" + recordKey; + _subscriptionId = "cultmesh-document:" + target.SessionKey + ":" + recordKey; var cacheRegistry = CultMesh.CreateCultCacheDocumentRegistry(typeof(TDocument)); var networkRegistry = CultMesh.CreateCultNetDocumentRegistry(new[] { typeof(TDocument) }, cacheRegistry); _cache = new CultCache(cacheRegistry); @@ -386,8 +801,8 @@ namespace GameCult.Mesh Handle = CultMesh.Document( _cache, new CultRecordKey(recordKey), - CultMesh.Verse(endpointId, "cultmesh-client").Context, - routeHint: new CultMeshRouteHint(CultMeshLocalityKind.Network, endpointId)); + CultMesh.Verse(target.VerseId, "cultmesh-client").Context, + routeHint: new CultMeshRouteHint(CultMeshLocalityKind.Network, target.AuthorityRuntimeId)); } public CultMeshDocumentHandle Handle { get; } @@ -475,13 +890,13 @@ namespace GameCult.Mesh public RemoteCollectionBinding( CultMeshSession session, - string endpointId, + CultMeshSessionTarget target, bool includeInitialSnapshot, TimeSpan responseTimeout) { _session = session; _responseTimeout = responseTimeout; - _subscriptionId = "cultmesh-collection:" + endpointId + ":" + typeof(TDocument).FullName; + _subscriptionId = "cultmesh-collection:" + target.SessionKey + ":" + typeof(TDocument).FullName; _includeInitialSnapshot = includeInitialSnapshot; var cacheRegistry = CultMesh.CreateCultCacheDocumentRegistry(typeof(TDocument)); var networkRegistry = CultMesh.CreateCultNetDocumentRegistry(new[] { typeof(TDocument) }, cacheRegistry); @@ -489,7 +904,7 @@ namespace GameCult.Mesh _subscription = new CultNetDatabaseSubscriptionClient(session.OpenSchemaClient(), _cache, networkRegistry); Handle = CultMesh.Collection( _cache, - routeHint: new CultMeshRouteHint(CultMeshLocalityKind.Network, endpointId)); + routeHint: new CultMeshRouteHint(CultMeshLocalityKind.Network, target.AuthorityRuntimeId)); } public CultMeshCollectionHandle Handle { get; } @@ -587,7 +1002,7 @@ namespace GameCult.Mesh new CultMeshVerseCatalogRequestMessage { VerseIds = query.VerseIds.Count == 0 ? null : query.VerseIds.ToArray(), - TransportVersion = "cultmesh.v0" + TransportVersion = _options.TransportVersion }).ConfigureAwait(false); cancellationToken.ThrowIfCancellationRequested(); var observedAt = _options.Clock.UtcNow; diff --git a/src/GameCult.Mesh/CultMeshContentSessions.cs b/src/GameCult.Mesh/CultMeshContentSessions.cs index 51570c54..890dbfa8 100644 --- a/src/GameCult.Mesh/CultMeshContentSessions.cs +++ b/src/GameCult.Mesh/CultMeshContentSessions.cs @@ -122,31 +122,58 @@ namespace GameCult.Mesh public async Task ConnectAsync( CultMeshTransportCandidate candidate, - CultMeshEndpointId endpointId, + CultMeshSessionTarget target, CancellationToken cancellationToken = default) { if (!CanConnect(candidate)) throw new NotSupportedException($"Legacy RUDP content connector does not support '{candidate.Endpoint}'."); var client = await _schemaConnector.ConnectAsync(candidate, CultMeshProtocols.Content, cancellationToken) .ConfigureAwait(false); - return new LegacyRudpContentTransport(candidate.Endpoint, client, _options); + try + { + await CultMeshSessionIdentityClient.VerifyAsync( + client, + "cultmesh-content-client", + target, + CultMeshProtocols.Content, + candidate, + null, + _options.Clock.UtcNow, + _options.ResponseTimeout, + cancellationToken).ConfigureAwait(false); + return new LegacyRudpContentTransport( + candidate.Endpoint, client, _options, target, candidate.Generation); + } + catch + { + client.Dispose(); + throw; + } } - private sealed class LegacyRudpContentTransport : ICultMeshContentTransport + private sealed class LegacyRudpContentTransport : ICultMeshContentTransport, ICultMeshVerifiedTransport { private readonly ICultNetSchemaClient _client; private readonly CultMeshLegacyRudpContentTransportOptions _options; + private readonly string _verseId; + private readonly string _authorityRuntimeId; + private readonly string _routeGeneration; private readonly System.Collections.Concurrent.ConcurrentDictionary> _pending = new(StringComparer.Ordinal); private bool _disposed; public LegacyRudpContentTransport( string endpoint, ICultNetSchemaClient client, - CultMeshLegacyRudpContentTransportOptions options) + CultMeshLegacyRudpContentTransportOptions options, + CultMeshSessionTarget target, + string routeGeneration) { Endpoint = endpoint; _client = client; _options = options; + _verseId = target.VerseId; + _authorityRuntimeId = target.AuthorityRuntimeId; + _routeGeneration = routeGeneration ?? string.Empty; _client.OnCultNet(OnResponse); _client.OnCultNet(OnError); } @@ -154,6 +181,16 @@ namespace GameCult.Mesh public string TransportId => "legacy-rudp-content"; public string Endpoint { get; } + public bool IsVerifiedFor( + string verseId, + string authorityRuntimeId, + string protocolId, + string routeGeneration) => + string.Equals(_verseId, verseId, StringComparison.Ordinal) && + string.Equals(_authorityRuntimeId, authorityRuntimeId, StringComparison.Ordinal) && + string.Equals(CultMeshProtocols.Content.Value, protocolId, StringComparison.Ordinal) && + string.Equals(_routeGeneration, routeGeneration, StringComparison.Ordinal); + public async Task CopyChunkToAsync( CultMeshCdnChunkRef chunk, Stream destination, diff --git a/src/GameCult.Mesh/CultMeshContentTransports.cs b/src/GameCult.Mesh/CultMeshContentTransports.cs index f1814b2c..891c221a 100644 --- a/src/GameCult.Mesh/CultMeshContentTransports.cs +++ b/src/GameCult.Mesh/CultMeshContentTransports.cs @@ -36,7 +36,7 @@ namespace GameCult.Mesh bool CanConnect(CultMeshTransportCandidate candidate); Task ConnectAsync( CultMeshTransportCandidate candidate, - CultMeshEndpointId endpointId, + CultMeshSessionTarget target, CancellationToken cancellationToken = default); } @@ -52,18 +52,18 @@ namespace GameCult.Mesh private bool _disposed; internal CultMeshContentSession( - CultMeshEndpointId endpointId, + CultMeshSessionTarget target, ICultMeshContentTransport transport, CultMeshSessionState state, Action onTransportFailure) { - EndpointId = endpointId ?? throw new ArgumentNullException(nameof(endpointId)); + Target = target ?? throw new ArgumentNullException(nameof(target)); _transport = transport ?? throw new ArgumentNullException(nameof(transport)); State = state ?? throw new ArgumentNullException(nameof(state)); _onTransportFailure = onTransportFailure ?? throw new ArgumentNullException(nameof(onTransportFailure)); } - public CultMeshEndpointId EndpointId { get; } + public CultMeshSessionTarget Target { get; } public CultMeshSessionState State { get; private set; } public string TransportId { get { lock (_gate) return _transport.TransportId; } } @@ -138,18 +138,18 @@ namespace GameCult.Mesh public sealed class CultMeshSessionContentProvider : ICultMeshContentProvider { private readonly CultMeshSessionManager _sessions; - private readonly CultMeshEndpointId _endpointId; + private readonly CultMeshSessionTarget _target; public CultMeshSessionContentProvider( string providerId, CultMeshSessionManager sessions, - CultMeshEndpointId endpointId) + CultMeshSessionTarget target) { ProviderId = string.IsNullOrWhiteSpace(providerId) ? throw new ArgumentException("Provider identity is required.", nameof(providerId)) : providerId; _sessions = sessions ?? throw new ArgumentNullException(nameof(sessions)); - _endpointId = endpointId ?? throw new ArgumentNullException(nameof(endpointId)); + _target = target ?? throw new ArgumentNullException(nameof(target)); } public string ProviderId { get; } @@ -159,7 +159,7 @@ namespace GameCult.Mesh Stream destination, CancellationToken cancellationToken = default) { - var session = await _sessions.ConnectContentAsync(_endpointId, cancellationToken).ConfigureAwait(false); + var session = await _sessions.ConnectContentAsync(_target, cancellationToken).ConfigureAwait(false); await session.CopyChunkToAsync(chunk, destination, cancellationToken).ConfigureAwait(false); } } diff --git a/src/GameCult.Mesh/CultMeshDiscoveryService.cs b/src/GameCult.Mesh/CultMeshDiscoveryService.cs index 964477e3..3f108e03 100644 --- a/src/GameCult.Mesh/CultMeshDiscoveryService.cs +++ b/src/GameCult.Mesh/CultMeshDiscoveryService.cs @@ -33,16 +33,25 @@ namespace GameCult.Mesh public sealed class CultMeshDiscoveryQuery { - public CultMeshDiscoveryQuery(string endpointId, IEnumerable? verseIds = null) + /// Stable lookup identity used for caching and diagnostics, never a physical route. + /// Optional Verse filter. + /// Optional provider runtime that each accepted Verse must advertise. + public CultMeshDiscoveryQuery( + string endpointId, + IEnumerable? verseIds = null, + string? authorityRuntimeId = null) { EndpointId = Require(endpointId, nameof(endpointId)); VerseIds = verseIds?.Where(value => !string.IsNullOrWhiteSpace(value)) .Distinct(StringComparer.Ordinal).OrderBy(value => value, StringComparer.Ordinal).ToArray() ?? Array.Empty(); + AuthorityRuntimeId = string.IsNullOrWhiteSpace(authorityRuntimeId) ? null : authorityRuntimeId.Trim(); } public string EndpointId { get; } public IReadOnlyList VerseIds { get; } + /// Gets the optional provider runtime constraint for authority-sensitive resolution. + public string? AuthorityRuntimeId { get; } private static string Require(string value, string name) => string.IsNullOrWhiteSpace(value) ? throw new ArgumentException("Value must be non-empty.", name) : value; @@ -161,6 +170,7 @@ namespace GameCult.Mesh private readonly CultMeshDiscoveryServiceOptions _options; private readonly ConcurrentDictionary>> _inFlight = new(StringComparer.Ordinal); private readonly ConcurrentDictionary _states = new(StringComparer.Ordinal); + private readonly ConcurrentDictionary _invalidated = new(StringComparer.Ordinal); private readonly Subject _updates = new(); private long _diagnosticSequence; private bool _disposed; @@ -183,6 +193,19 @@ namespace GameCult.Mesh return state == null ? null : EvaluateAt(state, _options.Clock.UtcNow); } + /// + /// Invalidates one resolved identity so a transport failure must consult the + /// configured rendezvous sources again instead of retrying a fresh-but-dead route. + /// + public void Invalidate(CultMeshDiscoveryQuery query) + { + if (query == null) throw new ArgumentNullException(nameof(query)); + ThrowIfDisposed(); + var key = QueryKey(query); + _states.TryRemove(key, out _); + _invalidated[key] = 0; + } + public async Task ResolveAsync( CultMeshDiscoveryQuery query, CancellationToken cancellationToken = default) @@ -205,9 +228,11 @@ namespace GameCult.Mesh } private static string QueryKey(CultMeshDiscoveryQuery query) => - query.VerseIds.Count == 0 + query.AuthorityRuntimeId == null && query.VerseIds.Count == 0 ? query.EndpointId - : query.EndpointId + "\u001f" + string.Join("\u001f", query.VerseIds); + : string.Join( + "\u001f", + new[] { query.EndpointId, query.AuthorityRuntimeId ?? "" }.Concat(query.VerseIds)); private static async Task AwaitForCallerAsync(Task shared, CancellationToken cancellationToken) { @@ -241,6 +266,9 @@ namespace GameCult.Mesh var failures = results.Where(result => result.Error != null).Select(result => result.SourceId).ToArray(); var observations = results.SelectMany(result => result.Observations) .Where(observation => query.VerseIds.Count == 0 || query.VerseIds.Contains(observation.Candidate.VerseId, StringComparer.Ordinal)) + .Where(observation => query.AuthorityRuntimeId == null || + observation.Candidate.AuthorityRoutes.Any(route => + string.Equals(route.AuthorityRuntimeId, query.AuthorityRuntimeId, StringComparison.Ordinal))) .ToArray(); foreach (var rejected in observations.Where(observation => observation.Trust == CultMeshDiscoveryTrust.Rejected)) @@ -315,6 +343,7 @@ namespace GameCult.Mesh private async Task LoadPreviousAsync(string queryKey, CancellationToken cancellationToken) { if (_states.TryGetValue(queryKey, out var current)) return current; + if (_invalidated.TryRemove(queryKey, out _)) return null; if (_options.Store == null) return null; var stored = await _options.Store.LoadAsync(queryKey, cancellationToken).ConfigureAwait(false); if (stored != null) _states[queryKey] = stored; diff --git a/src/GameCult.Mesh/CultMeshFrameBodies.cs b/src/GameCult.Mesh/CultMeshFrameBodies.cs index 5c4d26d0..dbf2e235 100644 --- a/src/GameCult.Mesh/CultMeshFrameBodies.cs +++ b/src/GameCult.Mesh/CultMeshFrameBodies.cs @@ -3,6 +3,7 @@ using System.Buffers; using System.Collections.Generic; using System.IO; using System.IO.MemoryMappedFiles; +using System.Runtime.InteropServices; using System.Security.Cryptography; using System.Text; using System.Threading; @@ -12,6 +13,27 @@ using Microsoft.Win32.SafeHandles; namespace GameCult.Mesh { + /// Reports bounded shared-memory frame publication and explicit copy telemetry. + public sealed class CultMeshFrameBodyPublisherStats + { + internal CultMeshFrameBodyPublisherStats( + ulong publishedFrames, + ulong blockedWrites, + ulong unavoidableCopyCount) + { + PublishedFrames = publishedFrames; + BlockedWrites = blockedWrites; + UnavoidableCopyCount = unavoidableCopyCount; + } + + /// Gets the number of atomically committed generations. + public ulong PublishedFrames { get; } + /// Gets the number of write reservations refused by bounded slots. + public ulong BlockedWrites { get; } + /// Gets copies explicitly declared by copy-fallback publication. + public ulong UnavoidableCopyCount { get; } + } + // Owner: this publisher owns mapping lifetime, logical generation, and slot commit. // Consumers own only read leases; mappings, capabilities, and slot selection remain CultMesh state. public sealed class CultMeshFrameBodyPublisher : IDisposable @@ -32,8 +54,12 @@ namespace GameCult.Mesh private readonly Mutex _mutex; private readonly TimeSpan _leaseDuration; private readonly string _token; + private readonly string? _mappingPath; private int _cursor; private long _nextSequence; + private ulong _publishedFrames; + private ulong _blockedWrites; + private ulong _unavoidableCopyCount; private CultMeshFrameBodyWriteLease? _activeWrite; private bool _disposed; @@ -59,7 +85,21 @@ namespace GameCult.Mesh _leaseDuration = leaseDuration ?? TimeSpan.FromSeconds(5); if (_leaseDuration <= TimeSpan.Zero) throw new ArgumentOutOfRangeException(nameof(leaseDuration)); _token = CapabilityPrefix + RandomToken(); - _mapping = MemoryMappedFile.CreateNew(MappingName(_token), checked(HeaderSize + CultMeshFrameRegion.SlotCount * slotByteLength)); + var mappingByteLength = checked(HeaderSize + CultMeshFrameRegion.SlotCount * slotByteLength); + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + _mapping = MemoryMappedFile.CreateNew(MappingName(_token), mappingByteLength); + } + else + { + _mappingPath = MappingPath(_token); + _mapping = MemoryMappedFile.CreateFromFile( + _mappingPath, + FileMode.CreateNew, + null, + mappingByteLength, + MemoryMappedFileAccess.ReadWrite); + } _control = _mapping.CreateViewAccessor(0, HeaderSize, MemoryMappedFileAccess.ReadWrite); _control.Write(0, slotByteLength); _control.Write(4, CultMeshFrameRegion.SlotCount); @@ -88,7 +128,7 @@ namespace GameCult.Mesh using (write) { body.CopyTo(write.Span); - descriptor = write.Commit(body.Length, nowUtc); + descriptor = write.Commit(body.Length, nowUtc, unavoidableCopyCount: 1); return true; } } @@ -103,6 +143,7 @@ namespace GameCult.Mesh ThrowIfDisposed(); if (_activeWrite != null) { + _blockedWrites++; lease = null!; return false; } @@ -124,6 +165,7 @@ namespace GameCult.Mesh }); if (slot < 0) { + _blockedWrites++; lease = null!; return false; } @@ -149,7 +191,8 @@ namespace GameCult.Mesh internal CultMeshBodyDescriptor Commit( CultMeshFrameBodyWriteLease lease, int byteLength, - DateTimeOffset nowUtc) + DateTimeOffset nowUtc, + int unavoidableCopyCount) { lock (_gate) { @@ -158,6 +201,8 @@ namespace GameCult.Mesh throw new InvalidOperationException("The CultMesh frame write lease is stale or is not owned by this publisher."); if (byteLength < 0 || byteLength > SlotByteLength) throw new ArgumentOutOfRangeException(nameof(byteLength)); + if (unavoidableCopyCount < 0) + throw new ArgumentOutOfRangeException(nameof(unavoidableCopyCount)); var semanticHash = CultMeshBodyDescriptorValidator.ComputeSemanticHash(lease.Span[..byteLength]); WithMutex(() => { @@ -174,6 +219,8 @@ namespace GameCult.Mesh }); var descriptor = Descriptor(lease.SlotIndex, byteLength, lease.Sequence, semanticHash, nowUtc); _nextSequence++; + _publishedFrames++; + _unavoidableCopyCount += (ulong)Math.Max(0, unavoidableCopyCount); _activeWrite = null; lease.Complete(); return descriptor; @@ -194,6 +241,19 @@ namespace GameCult.Mesh } } + /// Gets current publication, backpressure, and explicit fallback-copy counters. + public CultMeshFrameBodyPublisherStats Stats() + { + lock (_gate) + { + ThrowIfDisposed(); + return new CultMeshFrameBodyPublisherStats( + _publishedFrames, + _blockedWrites, + _unavoidableCopyCount); + } + } + private CultMeshBodyDescriptor Descriptor( int slot, int byteLength, @@ -219,7 +279,7 @@ namespace GameCult.Mesh Mutex mutex; try { - mapping = MemoryMappedFile.OpenExisting(MappingName(token), MemoryMappedFileRights.ReadWrite); + mapping = OpenMapping(token); mutex = Mutex.OpenExisting(MutexName(token)); } catch (Exception error) when (error is FileNotFoundException || error is WaitHandleCannotBeOpenedException) @@ -263,7 +323,7 @@ namespace GameCult.Mesh { try { - using var mapping = MemoryMappedFile.OpenExisting(MappingName(token), MemoryMappedFileRights.ReadWrite); + using var mapping = OpenMapping(token); using var mutex = Mutex.OpenExisting(MutexName(token)); WithMutex(mutex, () => { @@ -287,6 +347,12 @@ namespace GameCult.Mesh _control.Dispose(); _mapping.Dispose(); _mutex.Dispose(); + if (_mappingPath != null) + { + try { File.Delete(_mappingPath); } + catch (IOException) { } + catch (UnauthorizedAccessException) { } + } } } @@ -299,6 +365,11 @@ namespace GameCult.Mesh } internal static long SlotOffset(int slot, int slotByteLength) => HeaderSize + (long)slot * slotByteLength; internal static string MappingName(string token) => "cultmesh-map-" + token; + internal static string MappingPath(string token) => Path.Combine(Path.GetTempPath(), MappingName(token) + ".map"); + internal static MemoryMappedFile OpenMapping(string token) => + RuntimeInformation.IsOSPlatform(OSPlatform.Windows) + ? MemoryMappedFile.OpenExisting(MappingName(token), MemoryMappedFileRights.ReadWrite) + : MemoryMappedFile.CreateFromFile(MappingPath(token), FileMode.Open, null, 0, MemoryMappedFileAccess.ReadWrite); internal static string MutexName(string token) => "cultmesh-lock-" + token; internal static void ParseCapability(string capability, out string token, out int slot) { @@ -349,9 +420,7 @@ namespace GameCult.Mesh CultMeshFrameBodyPublisher.ParseCapability(descriptor.CapabilityToken, out _token, out _); try { - _mapping = MemoryMappedFile.OpenExisting( - CultMeshFrameBodyPublisher.MappingName(_token), - MemoryMappedFileRights.ReadWrite); + _mapping = CultMeshFrameBodyPublisher.OpenMapping(_token); _mutex = Mutex.OpenExisting(CultMeshFrameBodyPublisher.MutexName(_token)); } catch (Exception error) when (error is FileNotFoundException || error is WaitHandleCannotBeOpenedException) @@ -459,9 +528,7 @@ namespace GameCult.Mesh }; try { - var mapping = MemoryMappedFile.OpenExisting( - CultMeshFrameBodyPublisher.MappingName(_token), - MemoryMappedFileRights.ReadWrite); + var mapping = CultMeshFrameBodyPublisher.OpenMapping(_token); lease = new CultMeshMappedBodyLease( descriptor, mapping, @@ -514,9 +581,12 @@ namespace GameCult.Mesh public Memory Memory => _memory.Memory; public Span Span => Memory.Span; - public CultMeshBodyDescriptor Commit(int byteLength, DateTimeOffset nowUtc) => + public CultMeshBodyDescriptor Commit( + int byteLength, + DateTimeOffset nowUtc, + int unavoidableCopyCount = 0) => (_owner ?? throw new ObjectDisposedException(nameof(CultMeshFrameBodyWriteLease))) - .Commit(this, byteLength, nowUtc); + .Commit(this, byteLength, nowUtc, unavoidableCopyCount); internal void Complete() { diff --git a/src/GameCult.Mesh/CultMeshHttpsContentTransport.cs b/src/GameCult.Mesh/CultMeshHttpsContentTransport.cs new file mode 100644 index 00000000..d6427470 --- /dev/null +++ b/src/GameCult.Mesh/CultMeshHttpsContentTransport.cs @@ -0,0 +1,124 @@ +using System; +using System.IO; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; + +namespace GameCult.Mesh +{ + /// + /// Opens immutable content over an Odin-certified HTTPS route. Provider + /// identity comes from the verified route and TLS endpoint; the transfer + /// service remains responsible for checking the advertised chunk hash. + /// + public sealed class CultMeshHttpsContentTransportConnector : ICultMeshContentTransportConnector + { + private readonly Func _createHandler; + + public CultMeshHttpsContentTransportConnector(Func? createHandler = null) + { + _createHandler = createHandler ?? (() => new HttpClientHandler()); + } + + public string ConnectorId => "https-content"; + public int Priority => 0; + + public bool CanConnect(CultMeshTransportCandidate candidate) => + candidate != null && + Uri.TryCreate(candidate.Endpoint, UriKind.Absolute, out var endpoint) && + string.Equals(endpoint.Scheme, Uri.UriSchemeHttps, StringComparison.OrdinalIgnoreCase); + + public Task ConnectAsync( + CultMeshTransportCandidate candidate, + CultMeshSessionTarget target, + CancellationToken cancellationToken = default) + { + if (candidate == null) throw new ArgumentNullException(nameof(candidate)); + if (target == null) throw new ArgumentNullException(nameof(target)); + if (!CanConnect(candidate)) + throw new NotSupportedException($"HTTPS content connector does not support '{candidate.Endpoint}'."); + cancellationToken.ThrowIfCancellationRequested(); + return Task.FromResult(new CultMeshHttpsContentTransport( + candidate.Endpoint, + new HttpClient(_createHandler(), disposeHandler: true), + target, + candidate.Generation)); + } + } + + internal sealed class CultMeshHttpsContentTransport : ICultMeshContentTransport, ICultMeshVerifiedTransport + { + private readonly HttpClient _client; + private readonly CultMeshSessionTarget _target; + private readonly string _routeGeneration; + private bool _disposed; + + public CultMeshHttpsContentTransport( + string endpoint, + HttpClient client, + CultMeshSessionTarget target, + string routeGeneration) + { + Endpoint = endpoint ?? throw new ArgumentNullException(nameof(endpoint)); + _client = client ?? throw new ArgumentNullException(nameof(client)); + _target = target ?? throw new ArgumentNullException(nameof(target)); + _routeGeneration = routeGeneration ?? string.Empty; + } + + public string TransportId => "https-content"; + public string Endpoint { get; } + + public bool IsVerifiedFor( + string verseId, + string authorityRuntimeId, + string protocolId, + string routeGeneration) => + string.Equals(_target.VerseId, verseId, StringComparison.Ordinal) && + string.Equals(_target.AuthorityRuntimeId, authorityRuntimeId, StringComparison.Ordinal) && + string.Equals(CultMeshProtocols.Content.Value, protocolId, StringComparison.Ordinal) && + string.Equals(_routeGeneration, routeGeneration, StringComparison.Ordinal); + + public async Task CopyChunkToAsync( + CultMeshCdnChunkRef chunk, + Stream destination, + CancellationToken cancellationToken = default) + { + if (_disposed) throw new ObjectDisposedException(nameof(CultMeshHttpsContentTransport)); + if (chunk == null) throw new ArgumentNullException(nameof(chunk)); + if (destination == null) throw new ArgumentNullException(nameof(destination)); + if (!destination.CanWrite) throw new ArgumentException("Content destination must be writable.", nameof(destination)); + + var requestUri = BuildChunkUri(Endpoint, chunk); + using var response = await _client.GetAsync( + requestUri, + HttpCompletionOption.ResponseHeadersRead, + cancellationToken).ConfigureAwait(false); + response.EnsureSuccessStatusCode(); + if (response.Content.Headers.ContentLength is long contentLength && + contentLength != chunk.SizeBytes) + { + throw new InvalidDataException( + $"HTTPS content length {contentLength} did not match advertised chunk size {chunk.SizeBytes}."); + } + using var source = await response.Content.ReadAsStreamAsync().ConfigureAwait(false); + await source.CopyToAsync(destination, 81920, cancellationToken).ConfigureAwait(false); + } + + public void Dispose() + { + if (_disposed) return; + _disposed = true; + _client.Dispose(); + } + + private static Uri BuildChunkUri(string endpoint, CultMeshCdnChunkRef chunk) + { + var builder = new UriBuilder(endpoint); + var separator = string.IsNullOrWhiteSpace(builder.Query) ? string.Empty : builder.Query.TrimStart('?') + "&"; + builder.Query = separator + + "chunkHash=" + Uri.EscapeDataString(CultMeshCdn.NormalizeHash(chunk.ChunkHash, nameof(chunk.ChunkHash))) + + "&recordKey=" + Uri.EscapeDataString(chunk.RecordKey ?? string.Empty); + return builder.Uri; + } + } +} diff --git a/src/GameCult.Mesh/CultMeshPeerExchange.cs b/src/GameCult.Mesh/CultMeshPeerExchange.cs index 2489ebac..c3045c0d 100644 --- a/src/GameCult.Mesh/CultMeshPeerExchange.cs +++ b/src/GameCult.Mesh/CultMeshPeerExchange.cs @@ -446,15 +446,17 @@ namespace GameCult.Mesh /// Fetches peer cards through the shared session owner. public async Task FetchAsync( - CultMeshEndpointId endpointId, + CultMeshSessionTarget target, CultMeshPeerExchangeRequestMessage request, CancellationToken cancellationToken = default) { - if (endpointId == null) throw new ArgumentNullException(nameof(endpointId)); + if (target == null) throw new ArgumentNullException(nameof(target)); if (request == null) throw new ArgumentNullException(nameof(request)); if (string.IsNullOrWhiteSpace(request.VerseId)) throw new ArgumentException("Peer exchange requires a verseId.", nameof(request)); + if (!string.Equals(request.VerseId, target.VerseId, StringComparison.Ordinal)) + throw new ArgumentException("Peer exchange request Verse must match its session target.", nameof(request)); var sessions = _options.Sessions ?? throw new InvalidOperationException("Managed peer exchange requires a CultMeshSessionManager."); - var session = await sessions.ConnectAsync(endpointId, CultMeshProtocols.PeerExchange, cancellationToken).ConfigureAwait(false); + var session = await sessions.ConnectAsync(target, CultMeshProtocols.PeerExchange, cancellationToken).ConfigureAwait(false); var messageId = Guid.NewGuid().ToString("N"); var completion = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); using var responseSubscription = session.OnCultNet(response => @@ -471,7 +473,7 @@ namespace GameCult.Mesh KnownPeerIds = request.KnownPeerIds, Limit = request.Limit }); - return await WaitForManagedResponseAsync(completion.Task, endpointId, cancellationToken).ConfigureAwait(false); + return await WaitForManagedResponseAsync(completion.Task, target, cancellationToken).ConfigureAwait(false); } /// @@ -541,7 +543,7 @@ namespace GameCult.Mesh private async Task WaitForManagedResponseAsync( Task responseTask, - CultMeshEndpointId endpointId, + CultMeshSessionTarget target, CancellationToken cancellationToken) { var timeout = _options.Clock.DelayAsync(_options.ResponseTimeout, cancellationToken); @@ -549,7 +551,7 @@ namespace GameCult.Mesh if (completed != responseTask) { cancellationToken.ThrowIfCancellationRequested(); - throw new TimeoutException($"Timed out waiting for peer exchange response from {endpointId}."); + throw new TimeoutException($"Timed out waiting for peer exchange response from {target}."); } return await responseTask.ConfigureAwait(false); } diff --git a/src/GameCult.Mesh/CultMeshPrimitives.cs b/src/GameCult.Mesh/CultMeshPrimitives.cs index dcc58143..58723950 100644 --- a/src/GameCult.Mesh/CultMeshPrimitives.cs +++ b/src/GameCult.Mesh/CultMeshPrimitives.cs @@ -911,9 +911,6 @@ namespace GameCult.Mesh /// Gets whether this handle can submit a client prediction for the underlying document value. bool CanSubmitPrediction { get; } - /// Gets whether this handle can accept a transparent document mutation. - bool CanSet { get; } - /// Creates a same-schema alias presentation for another CLR document type. CultMeshDocumentHandle AsSchemaAlias() where TAlias : class; } @@ -930,6 +927,9 @@ namespace GameCult.Mesh private readonly CultMeshBoundLiveFeed _feed; private readonly Func? _replace; private readonly Func? _submitPrediction; + // Set only over a CultCache record, whose changes carry the cache's Sequence. + private readonly Func>>? _recordChanges; + private readonly Func<(TDocument Document, long Sequence)>? _sequencedLatest; /// Creates a document handle from a Verse-bound live feed. public CultMeshDocumentHandle( @@ -942,6 +942,17 @@ namespace GameCult.Mesh _submitPrediction = submitPrediction; } + internal CultMeshDocumentHandle( + CultMeshBoundLiveFeed feed, + Func replace, + Func>> recordChanges, + Func<(TDocument Document, long Sequence)> sequencedLatest) + : this(feed, replace) + { + _recordChanges = recordChanges; + _sequencedLatest = sequencedLatest; + } + /// Gets the semantic document id. public string DocumentId => _feed.FeedId; @@ -975,9 +986,6 @@ namespace GameCult.Mesh /// Gets whether this handle can submit a client prediction for the underlying document value. public bool CanSubmitPrediction => _submitPrediction != null; - /// Gets whether this handle can accept a transparent document mutation. - public bool CanSet => CanSubmitPrediction || CanReplace; - /// Reads one coherent document snapshot. public Task LatestAsync() { @@ -1003,6 +1011,31 @@ namespace GameCult.Mesh return Watch().Subscribe(onNext); } + // Latest-value mirrors: a cache-backed handle passes each snapshot's Sequence; other sources pass null. + internal IDisposable WatchSequenced(Action onNext) => _recordChanges == null + ? Watch(document => onNext(document, null)) + : _recordChanges() + .Where(change => change.Document != null) + .Subscribe(change => onNext(change.Document!, change.Sequence)); + + // A mirror subscribes first, then adopts under its own gate. A cache-backed handle reads synchronously there, so + // its Sequence is at or above every change the mirror has applied and a later straggler at or below it drops. + internal async Task AdoptLatestAsync(object gate, Func adopt) + { + if (_sequencedLatest != null) + { + lock (gate) + { + var (document, sequence) = _sequencedLatest(); + return adopt(document, sequence); + } + } + + var latest = await LatestAsync().ConfigureAwait(false); + lock (gate) + return adopt(latest, null); + } + /// /// Projects one typed document field into a stable reactive state pointer. The expression is /// structural metadata: only member access and constant collection indices are accepted. @@ -1048,57 +1081,40 @@ namespace GameCult.Mesh return _submitPrediction(value); } - /// - /// Sets the document value through the configured authority shape. - /// Prediction-backed documents publish a prediction; mutable documents replace authoritatively. - /// - public Task SetAsync(TDocument value) + /// Gets an explicit authoritative replacement writer for this document. + public CultMeshDocumentWriter AuthoritativeWriter() { - if (value == null) throw new ArgumentNullException(nameof(value)); - if (_submitPrediction != null) - return _submitPrediction(value); - if (_replace != null) - return _replace(value); - - throw new NotSupportedException($"Document handle '{DocumentId}' does not accept mutations."); + if (_replace == null) + throw new NotSupportedException($"Document handle '{DocumentId}' is read-only."); + return new CultMeshDocumentWriter( + this, + CultMeshDocumentWriteKind.AuthoritativeReplacement, + _replace); } - /// Reads, updates, and sets the document value through the configured authority shape. - public async Task UpdateAsync(Func update) + /// Gets an explicit client prediction writer for this document. + public CultMeshDocumentWriter PredictionWriter() { - if (update == null) throw new ArgumentNullException(nameof(update)); - var next = update(await LatestAsync().ConfigureAwait(false)); - await SetAsync(next).ConfigureAwait(false); - return next; + if (_submitPrediction == null) + throw new NotSupportedException($"Document handle '{DocumentId}' does not accept client predictions."); + return new CultMeshDocumentWriter( + this, + CultMeshDocumentWriteKind.Prediction, + _submitPrediction); } - /// Reads, updates, and sets the document value through the configured authority shape. - public async Task UpdateAsync(Func> update) + /// Creates a read-only in-memory mirror updated by the document watch. + public async Task> ObserveAsync() { - if (update == null) throw new ArgumentNullException(nameof(update)); - var next = await update(await LatestAsync().ConfigureAwait(false)).ConfigureAwait(false); - await SetAsync(next).ConfigureAwait(false); - return next; + var observed = new CultMeshObservedDocument(this); + await observed.StartAsync().ConfigureAwait(false); + return observed; } - /// - /// Creates a managed reactive document mirror whose local edits can be coalesced into one prediction or replacement. - /// - public async Task> ReactiveAsync( - CultMeshReactiveDocumentOptions? options = null) + /// Creates a read-only in-memory mirror synchronously. + public CultMeshObservedDocument Observe() { - var current = CloneDocument(await LatestAsync().ConfigureAwait(false)); - var reactive = new CultMeshReactiveDocument(this, current, options); - reactive.Start(); - return reactive; - } - - /// - /// Creates a managed reactive document mirror synchronously for host APIs that cannot be async. - /// - public CultMeshReactiveDocument Reactive(CultMeshReactiveDocumentOptions? options = null) - { - return ReactiveAsync(options).ConfigureAwait(false).GetAwaiter().GetResult(); + return ObserveAsync().ConfigureAwait(false).GetAwaiter().GetResult(); } /// Creates a same-schema alias presentation for another CLR document type. @@ -1215,6 +1231,181 @@ namespace GameCult.Mesh } } + /// + /// A read-only local mirror of a watched typed document. Observation never acquires a write + /// capability and never polls or serializes the document while idle. + /// + public sealed class CultMeshObservedDocument : IDisposable + where TDocument : class + { + private readonly object _gate = new(); + private IDisposable? _subscription; + private TDocument _current; + private long _appliedSequence; + private bool _disposed; + + internal CultMeshObservedDocument(CultMeshDocumentHandle document) + { + Document = document ?? throw new ArgumentNullException(nameof(document)); + _current = null!; + } + + /// Gets the underlying read/watch handle. + public CultMeshDocumentHandle Document { get; } + + /// + /// Gets the current borrowed snapshot. Treat the object graph as read-only; local mutation + /// cannot publish and is replaced by the next canonical snapshot. + /// + public TDocument Current + { + get + { + lock (_gate) + { + ThrowIfDisposed(); + return _current; + } + } + } + + internal async Task StartAsync() + { + _subscription = Document.WatchSequenced(ApplyCanonicalSnapshot); + try + { + await RefreshAsync().ConfigureAwait(false); + } + catch + { + Dispose(); + throw; + } + } + + /// Reads and adopts a fresh canonical snapshot. + public Task RefreshAsync() + { + ThrowIfDisposed(); + return Document.AdoptLatestAsync(_gate, (latest, sequence) => + { + ThrowIfDisposed(); + _current = CultMeshDocumentHandle.CloneDocument(latest); + _appliedSequence = sequence ?? _appliedSequence; + return _current; + }); + } + + /// + public void Dispose() + { + lock (_gate) + { + if (_disposed) + return; + _disposed = true; + } + + _subscription?.Dispose(); + } + + private void ApplyCanonicalSnapshot(TDocument canonical, long? sequence) + { + if (canonical == null) + return; + var next = CultMeshDocumentHandle.CloneDocument(canonical); + lock (_gate) + { + if (_disposed || sequence <= _appliedSequence) + return; + _appliedSequence = sequence ?? _appliedSequence; + _current = next; + } + } + + private void ThrowIfDisposed() + { + if (_disposed) + throw new ObjectDisposedException(nameof(CultMeshObservedDocument)); + } + } + + /// Names the one immutable authority path selected by a document writer. + public enum CultMeshDocumentWriteKind + { + /// The writer replaces canonical state through its owning provider. + AuthoritativeReplacement, + + /// The writer submits a client prediction or candidate for validation. + Prediction + } + + /// + /// An explicit, immutable write capability for one typed document. A writer never switches + /// between authoritative replacement and prediction based on runtime availability. + /// + public sealed class CultMeshDocumentWriter + where TDocument : class + { + private readonly Func _write; + + internal CultMeshDocumentWriter( + CultMeshDocumentHandle document, + CultMeshDocumentWriteKind kind, + Func write) + { + Document = document ?? throw new ArgumentNullException(nameof(document)); + Kind = kind; + _write = write ?? throw new ArgumentNullException(nameof(write)); + } + + /// Gets the document read/watch handle paired with this writer. + public CultMeshDocumentHandle Document { get; } + + /// Gets the immutable authority path selected for this writer. + public CultMeshDocumentWriteKind Kind { get; } + + /// Writes one value through this writer's named authority path. + public Task WriteAsync(TDocument value) + { + if (value == null) throw new ArgumentNullException(nameof(value)); + return _write(value); + } + + /// Reads, transforms, and writes through this writer's named authority path. + public async Task UpdateAsync(Func update) + { + if (update == null) throw new ArgumentNullException(nameof(update)); + var next = update(await Document.LatestAsync().ConfigureAwait(false)); + await WriteAsync(next).ConfigureAwait(false); + return next; + } + + /// Reads, transforms, and writes through this writer's named authority path. + public async Task UpdateAsync(Func> update) + { + if (update == null) throw new ArgumentNullException(nameof(update)); + var next = await update(await Document.LatestAsync().ConfigureAwait(false)).ConfigureAwait(false); + await WriteAsync(next).ConfigureAwait(false); + return next; + } + + /// Creates an explicitly writable reactive mirror. + public async Task> ReactiveAsync( + CultMeshReactiveDocumentOptions? options = null) + { + var reactive = new CultMeshReactiveDocument(this, options); + await reactive.StartAsync().ConfigureAwait(false); + return reactive; + } + + /// Creates an explicitly writable reactive mirror synchronously. + public CultMeshReactiveDocument Reactive(CultMeshReactiveDocumentOptions? options = null) + { + return ReactiveAsync(options).ConfigureAwait(false).GetAwaiter().GetResult(); + } + } + /// /// Configures a managed CultMesh reactive document mirror. /// @@ -1225,8 +1416,11 @@ namespace GameCult.Mesh /// public TimeSpan FlushDelay { get; set; } = TimeSpan.FromMilliseconds(16); - /// Gets or sets whether direct edits to Current should be detected and flushed automatically. - public bool DetectLocalChanges { get; set; } = true; + /// + /// Gets or sets the clock that owns delayed flush scheduling. Tests and deterministic hosts may + /// replace the system clock without introducing a second scheduler abstraction. + /// + public ICultMeshClock Clock { get; set; } = CultMeshSystemClock.Instance; /// Gets or sets whether canonical snapshots replace dirty local predictions immediately. public bool ReplaceDirtyCurrentOnCanonicalSnapshot { get; set; } @@ -1272,40 +1466,58 @@ namespace GameCult.Mesh } /// - /// Managed typed document mirror that keeps an editable current value and coalesces local edits into CultMesh mutations. + /// Managed typed document mirror that coalesces explicit local edits through one selected writer. /// public sealed class CultMeshReactiveDocument : IDisposable where TDocument : class { + private static readonly CultMeshComparableMember[] ComparableMembers = + GetComparableMembers(typeof(TDocument)).ToArray(); + private readonly CultMeshDocumentWriter _writer; private readonly CultMeshDocumentHandle _document; private readonly CultMeshReactiveDocumentOptions _options; private readonly object _gate = new(); private IDisposable? _subscription; - private Timer? _flushTimer; - private Timer? _changeDetectionTimer; - private byte[] _lastCleanSnapshot; + private CancellationTokenSource? _flushDelayCancellation; + private TDocument _current; private bool _dirty; private bool _flushQueued; private bool _flushing; private bool _disposed; + private long _appliedSequence; private int _reconciliationVersion; internal CultMeshReactiveDocument( - CultMeshDocumentHandle document, - TDocument current, + CultMeshDocumentWriter writer, CultMeshReactiveDocumentOptions? options) { - _document = document ?? throw new ArgumentNullException(nameof(document)); - Current = current ?? throw new ArgumentNullException(nameof(current)); + _writer = writer ?? throw new ArgumentNullException(nameof(writer)); + _document = writer.Document; + _current = null!; _options = options ?? new CultMeshReactiveDocumentOptions(); - _lastCleanSnapshot = SerializeDocument(Current); } + /// Gets the immutable write capability selected for this mirror. + public CultMeshDocumentWriter Writer => _writer; + /// Gets the underlying document handle. public CultMeshDocumentHandle Document => _document; - /// Gets the editable local document value. - public TDocument Current { get; private set; } + /// + /// Gets a cloned local snapshot. Mutating the returned object cannot mutate or publish the mirror; + /// use or for edits. + /// + public TDocument Snapshot + { + get + { + lock (_gate) + { + ThrowIfDisposed(); + return CultMeshDocumentHandle.CloneDocument(_current); + } + } + } /// Gets whether local edits are waiting to be sent. public bool IsDirty @@ -1320,27 +1532,17 @@ namespace GameCult.Mesh /// Gets the most recent reconciliation snapshot, when a canonical value arrived during local prediction. public CultMeshReactiveDocumentReconciliation? Reconciliation { get; private set; } - internal void Start() + internal async Task StartAsync() { - _subscription = _document.Watch(ApplyCanonicalSnapshot); - if (_options.DetectLocalChanges && _document.CanSet) + _subscription = _document.WatchSequenced(ApplyCanonicalSnapshot); + try { - var interval = _options.FlushDelay <= TimeSpan.Zero - ? TimeSpan.FromMilliseconds(1) - : _options.FlushDelay; - _changeDetectionTimer = new Timer(_ => _ = Task.Run(DetectLocalChangesAsync)); - _changeDetectionTimer.Change(interval, interval); + await RefreshAsync().ConfigureAwait(false); } - } - - /// Marks the current value dirty and schedules a coalesced prediction or replacement. - public void MarkDirty() - { - ThrowIfDisposed(); - lock (_gate) + catch { - _dirty = true; - ScheduleFlushLocked(); + Dispose(); + throw; } } @@ -1351,41 +1553,39 @@ namespace GameCult.Mesh ThrowIfDisposed(); lock (_gate) { - update(Current); + update(_current); _dirty = true; ScheduleFlushLocked(); - return Current; + return CultMeshDocumentHandle.CloneDocument(_current); } } /// Replaces the current local value and schedules a coalesced prediction or replacement. - public TDocument SetCurrent(TDocument value) + public TDocument ReplaceLocal(TDocument value) { if (value == null) throw new ArgumentNullException(nameof(value)); ThrowIfDisposed(); lock (_gate) { - Current = CultMeshDocumentHandle.CloneDocument(value); + _current = CultMeshDocumentHandle.CloneDocument(value); _dirty = true; ScheduleFlushLocked(); - return Current; + return CultMeshDocumentHandle.CloneDocument(_current); } } /// Reads a fresh canonical snapshot and adopts it as the current value. - public async Task RefreshAsync() + public Task RefreshAsync() { ThrowIfDisposed(); - var latest = CultMeshDocumentHandle.CloneDocument( - await _document.LatestAsync().ConfigureAwait(false)); - lock (_gate) + return _document.AdoptLatestAsync(_gate, (latest, sequence) => { - Current = latest; - _lastCleanSnapshot = SerializeDocument(Current); + _current = CultMeshDocumentHandle.CloneDocument(latest); + _appliedSequence = sequence ?? _appliedSequence; _dirty = false; Reconciliation = null; - return Current; - } + return CultMeshDocumentHandle.CloneDocument(_current); + }); } /// Immediately sends the latest local dirty value, if any, through the document authority shape. @@ -1395,8 +1595,6 @@ namespace GameCult.Mesh lock (_gate) { ThrowIfDisposed(); - if (!_dirty && _document.CanSet) - DetectLocalChangesLocked(); if (!_dirty) return; if (_flushing) @@ -1405,16 +1603,15 @@ namespace GameCult.Mesh return; } - _flushTimer?.Change(Timeout.InfiniteTimeSpan, Timeout.InfiniteTimeSpan); + CancelScheduledFlushLocked(); _flushing = true; _dirty = false; - predicted = CultMeshDocumentHandle.CloneDocument(Current); - _lastCleanSnapshot = SerializeDocument(predicted); + predicted = CultMeshDocumentHandle.CloneDocument(_current); } try { - await _document.SetAsync(predicted).ConfigureAwait(false); + await _writer.WriteAsync(predicted).ConfigureAwait(false); } finally { @@ -1422,8 +1619,6 @@ namespace GameCult.Mesh lock (_gate) { _flushing = false; - if (_document.CanSet) - DetectLocalChangesLocked(); shouldFlushAgain = _flushQueued || _dirty; _flushQueued = false; } @@ -1448,74 +1643,54 @@ namespace GameCult.Mesh if (_disposed) return; _disposed = true; + CancelScheduledFlushLocked(); } _subscription?.Dispose(); - _flushTimer?.Dispose(); - _changeDetectionTimer?.Dispose(); } - private void ApplyCanonicalSnapshot(TDocument canonical) + private void ApplyCanonicalSnapshot(TDocument canonical, long? sequence) { if (canonical == null) return; lock (_gate) { - if (_disposed) + if (_disposed || sequence <= _appliedSequence) return; + _appliedSequence = sequence ?? _appliedSequence; - var nextCanonical = CultMeshDocumentHandle.CloneDocument(canonical); + TDocument? replacement = null; if (_dirty || _flushing) { - var predicted = CultMeshDocumentHandle.CloneDocument(Current); - var delta = CreateReconciliationDelta(predicted, nextCanonical); + var delta = CreateReconciliationDelta(_current, canonical); if (delta.Count == 0) { Reconciliation = null; + if (!_options.ReplaceDirtyCurrentOnCanonicalSnapshot) + return; } else { + var nextCanonical = CultMeshDocumentHandle.CloneDocument(canonical); + var predicted = CultMeshDocumentHandle.CloneDocument(_current); + replacement = nextCanonical; Reconciliation = new CultMeshReactiveDocumentReconciliation( nextCanonical, predicted, delta, ++_reconciliationVersion, DateTimeOffset.UtcNow); + if (!_options.ReplaceDirtyCurrentOnCanonicalSnapshot) + return; } - - if (!_options.ReplaceDirtyCurrentOnCanonicalSnapshot) - return; } - Current = nextCanonical; - _lastCleanSnapshot = SerializeDocument(Current); + _current = replacement ?? CultMeshDocumentHandle.CloneDocument(canonical); Reconciliation = null; } } - private Task DetectLocalChangesAsync() - { - lock (_gate) - { - if (_disposed || !_document.CanSet || _dirty || _flushing) - return Task.CompletedTask; - DetectLocalChangesLocked(); - if (_dirty) - ScheduleFlushLocked(); - } - - return Task.CompletedTask; - } - - private void DetectLocalChangesLocked() - { - var currentSnapshot = SerializeDocument(Current); - if (_lastCleanSnapshot.SequenceEqual(currentSnapshot)) - return; - _dirty = true; - } - private void ScheduleFlushLocked() { if (_options.FlushDelay <= TimeSpan.Zero) @@ -1524,13 +1699,48 @@ namespace GameCult.Mesh return; } - _flushTimer ??= new Timer(_ => _ = Task.Run(FlushAsync)); - _flushTimer.Change(_options.FlushDelay, Timeout.InfiniteTimeSpan); + CancelScheduledFlushLocked(); + var cancellation = new CancellationTokenSource(); + _flushDelayCancellation = cancellation; + _ = FlushAfterDelayAsync(cancellation); } - private static byte[] SerializeDocument(TDocument document) + private async Task FlushAfterDelayAsync(CancellationTokenSource cancellation) { - return CultDocumentMessagePackSerialization.SerializeUntyped(document, typeof(TDocument)); + try + { + await _options.Clock.DelayAsync(_options.FlushDelay, cancellation.Token).ConfigureAwait(false); + lock (_gate) + { + if (_disposed || cancellation.IsCancellationRequested) + return; + } + + await FlushAsync().ConfigureAwait(false); + } + catch (OperationCanceledException) when (cancellation.IsCancellationRequested) + { + } + finally + { + lock (_gate) + { + if (ReferenceEquals(_flushDelayCancellation, cancellation)) + _flushDelayCancellation = null; + } + + cancellation.Dispose(); + } + } + + private void CancelScheduledFlushLocked() + { + var cancellation = _flushDelayCancellation; + _flushDelayCancellation = null; + if (cancellation == null) + return; + + cancellation.Cancel(); } private static IReadOnlyDictionary CreateReconciliationDelta( @@ -1538,7 +1748,7 @@ namespace GameCult.Mesh TDocument canonical) { var delta = new Dictionary(StringComparer.Ordinal); - foreach (var member in GetComparableMembers(typeof(TDocument))) + foreach (var member in ComparableMembers) { var predictedValue = member.GetValue(predicted); var canonicalValue = member.GetValue(canonical); @@ -1778,13 +1988,6 @@ namespace GameCult.Mesh return TryGetDocument(out var document) && document.CanSubmitPrediction; } - /// Gets whether one typed document can accept transparent mutations by CLR type or same-schema alias. - public bool CanSet() - where TDocument : class - { - return TryGetDocument(out var document) && document.CanSet; - } - /// Submits a client prediction for one typed document by CLR type or same-schema alias. public Task SubmitPredictionAsync(TDocument value) where TDocument : class @@ -1792,25 +1995,18 @@ namespace GameCult.Mesh return Document().SubmitPredictionAsync(value); } - /// Sets one typed document through its configured authority shape by CLR type or same-schema alias. - public Task SetAsync(TDocument value) + /// Gets an explicit authoritative writer by CLR type or same-schema alias. + public CultMeshDocumentWriter AuthoritativeWriter() where TDocument : class { - return Document().SetAsync(value); + return Document().AuthoritativeWriter(); } - /// Reads, updates, and sets one typed document through its configured authority shape. - public Task UpdateAsync(Func update) + /// Gets an explicit prediction writer by CLR type or same-schema alias. + public CultMeshDocumentWriter PredictionWriter() where TDocument : class { - return Document().UpdateAsync(update); - } - - /// Reads, updates, and sets one typed document through its configured authority shape. - public Task UpdateAsync(Func> update) - where TDocument : class - { - return Document().UpdateAsync(update); + return Document().PredictionWriter(); } /// Watches one typed document by CLR type or same-schema alias. @@ -1827,21 +2023,6 @@ namespace GameCult.Mesh return Document().Watch(onNext); } - /// Creates a managed reactive document mirror by CLR type or same-schema alias. - public Task> ReactiveAsync( - CultMeshReactiveDocumentOptions? options = null) - where TDocument : class - { - return Document().ReactiveAsync(options); - } - - /// Creates a managed reactive document mirror synchronously by CLR type or same-schema alias. - public CultMeshReactiveDocument Reactive( - CultMeshReactiveDocumentOptions? options = null) - where TDocument : class - { - return Document().Reactive(options); - } } /// @@ -3202,13 +3383,27 @@ namespace GameCult.Mesh string pointerId, string? sourceId = null, string? schemaId = null, - CultMeshRouteHint? routeHint = null) + CultMeshRouteHint? routeHint = null, + string? bindingName = null, + string? documentId = null, + string? fieldPath = null, + string? valueKind = null, + string? accessMode = null, + string? authority = null, + string? writeCommand = null) { TargetProp = RequireNonEmpty(targetProp, nameof(targetProp)); PointerId = RequireNonEmpty(pointerId, nameof(pointerId)); SourceId = sourceId; SchemaId = schemaId; RouteHint = routeHint ?? CultMeshRouteHint.Automatic; + BindingName = bindingName ?? ""; + DocumentId = documentId ?? ""; + FieldPath = fieldPath ?? ""; + ValueKind = valueKind ?? ""; + AccessMode = accessMode ?? ""; + Authority = authority ?? ""; + WriteCommand = writeCommand ?? ""; } /// Gets the component property receiving the resolved state value. @@ -3226,6 +3421,27 @@ namespace GameCult.Mesh /// Gets the preferred route for resolving and watching the binding. public CultMeshRouteHint RouteHint { get; } + /// Gets the stable renderer-facing name used to capture this binding in an operation. + public string BindingName { get; } + + /// Gets the optional identity of the provider-owned backing document. + public string DocumentId { get; } + + /// Gets the optional typed field path inside the backing document. + public string FieldPath { get; } + + /// Gets the optional portable value kind. + public string ValueKind { get; } + + /// Gets the optional read, write, read-write, or local-draft access mode. + public string AccessMode { get; } + + /// Gets the authority responsible for accepting writes to this binding. + public string Authority { get; } + + /// Gets the optional typed operation id used for direct writes. + public string WriteCommand { get; } + /// Creates a binding from an existing state pointer. public static CultMeshStateBindingDescriptor FromPointer( string targetProp, @@ -3263,7 +3479,14 @@ namespace GameCult.Mesh string? sourceId = null, string? schemaId = null, string? routeKind = null, - string? routeDescription = null) + string? routeDescription = null, + string? bindingName = null, + string? documentId = null, + string? fieldPath = null, + string? valueKind = null, + string? accessMode = null, + string? authority = null, + string? writeCommand = null) { TargetProp = string.IsNullOrWhiteSpace(targetProp) ? "value" : targetProp!; PointerId = pointerId ?? ""; @@ -3271,6 +3494,13 @@ namespace GameCult.Mesh SchemaId = schemaId ?? ""; RouteKind = routeKind ?? ""; RouteDescription = routeDescription ?? ""; + BindingName = bindingName ?? ""; + DocumentId = documentId ?? ""; + FieldPath = fieldPath ?? ""; + ValueKind = valueKind ?? ""; + AccessMode = accessMode ?? ""; + Authority = authority ?? ""; + WriteCommand = writeCommand ?? ""; } /// Gets the component property receiving the resolved state value. @@ -3291,6 +3521,14 @@ namespace GameCult.Mesh /// Gets the flattened route description. [Key(5)] public string RouteDescription { get; } + [Key(6)] public string BindingName { get; } + [Key(7)] public string DocumentId { get; } + [Key(8)] public string FieldPath { get; } + [Key(9)] public string ValueKind { get; } + [Key(10)] public string AccessMode { get; } + [Key(11)] public string Authority { get; } + [Key(12)] public string WriteCommand { get; } + /// Creates transport-friendly fields from a state binding descriptor. public static CultMeshStateBindingRecord FromBinding(CultMeshStateBindingDescriptor? binding) { @@ -3301,7 +3539,14 @@ namespace GameCult.Mesh binding?.SourceId, binding?.SchemaId, route.Kind, - route.Description); + route.Description, + binding?.BindingName, + binding?.DocumentId, + binding?.FieldPath, + binding?.ValueKind, + binding?.AccessMode, + binding?.Authority, + binding?.WriteCommand); } /// Rehydrates this record as a state binding descriptor. @@ -3323,7 +3568,14 @@ namespace GameCult.Mesh pointerId, SourceId, SchemaId, - route); + route, + BindingName, + DocumentId, + FieldPath, + ValueKind, + AccessMode, + Authority, + WriteCommand); } } diff --git a/src/GameCult.Mesh/CultMeshRealtimeTransports.cs b/src/GameCult.Mesh/CultMeshRealtimeTransports.cs index d087b304..d37b7381 100644 --- a/src/GameCult.Mesh/CultMeshRealtimeTransports.cs +++ b/src/GameCult.Mesh/CultMeshRealtimeTransports.cs @@ -59,7 +59,7 @@ namespace GameCult.Mesh bool CanConnect(CultMeshTransportCandidate candidate); Task ConnectAsync( CultMeshTransportCandidate candidate, - CultMeshEndpointId endpointId, + CultMeshSessionTarget target, CancellationToken cancellationToken = default); } @@ -72,18 +72,18 @@ namespace GameCult.Mesh private bool _disposed; internal CultMeshRealtimeSession( - CultMeshEndpointId endpointId, + CultMeshSessionTarget target, ICultMeshRealtimeTransport transport, CultMeshSessionState state, Action onTransportFailure) { - EndpointId = endpointId ?? throw new ArgumentNullException(nameof(endpointId)); + Target = target ?? throw new ArgumentNullException(nameof(target)); _transport = transport ?? throw new ArgumentNullException(nameof(transport)); State = state ?? throw new ArgumentNullException(nameof(state)); _onTransportFailure = onTransportFailure ?? throw new ArgumentNullException(nameof(onTransportFailure)); } - public CultMeshEndpointId EndpointId { get; } + public CultMeshSessionTarget Target { get; } public CultMeshSessionState State { get; private set; } public string TransportId { get { lock (_gate) return _transport.TransportId; } } diff --git a/src/GameCult.Mesh/CultMeshSessionIdentity.cs b/src/GameCult.Mesh/CultMeshSessionIdentity.cs new file mode 100644 index 00000000..3fad99f9 --- /dev/null +++ b/src/GameCult.Mesh/CultMeshSessionIdentity.cs @@ -0,0 +1,283 @@ +using System; +using System.Collections.Generic; +using System.Collections.Concurrent; +using System.Linq; +using System.Security.Cryptography; +using System.Threading; +using System.Threading.Tasks; +using GameCult.Networking; + +namespace GameCult.Mesh +{ + /// + /// Optional transport proof surfaced by connectors that already authenticate + /// the peer during their native handshake. Other schema clients must complete + /// the portable CultMesh handshake before use. + /// + public interface ICultMeshVerifiedSchemaClient + { + bool IsVerifiedFor( + string verseId, + string authorityRuntimeId, + string protocolId, + string routeGeneration); + } + + /// + /// Proof exposed by content and realtime transports after their native or + /// portable handshake has bound the physical channel to an Odin route. + /// + public interface ICultMeshVerifiedTransport + { + bool IsVerifiedFor( + string verseId, + string authorityRuntimeId, + string protocolId, + string routeGeneration); + } + + internal static class CultMeshTransportIdentity + { + public static void RequireVerified( + object transport, + CultMeshSessionTarget target, + CultMeshProtocolId protocol, + CultMeshTransportCandidate candidate) + { + if (transport is ICultMeshVerifiedTransport verified && + verified.IsVerifiedFor( + target.VerseId, + target.AuthorityRuntimeId, + protocol.Value, + candidate.Generation)) + return; + throw new CultMeshSessionException(new CultMeshSessionFailure( + CultMeshSessionFailureReason.Authority, + $"Transport did not prove CultMesh authority '{target}' for protocol '{protocol}'.", + candidate.Endpoint)); + } + } + + /// + /// Owns the server side of the transport-neutral CultMesh session identity + /// handshake. Application messages are not authority proof; providers attach + /// this gate to every schema transport they advertise. + /// + public sealed class CultMeshSessionIdentityServer : IDisposable + { + private readonly ICultNetSchemaServer _server; + private readonly string _authorityRuntimeId; + private readonly HashSet _verseIds; + private readonly HashSet _protocolIds; + private readonly HashSet _routeGenerations; + private readonly Dictionary _proofSigners; + private readonly ConcurrentDictionary _acceptedPeers = new(); + private readonly Func _handler; + private readonly ICultNetSchemaServerPeerLifecycle? _peerLifecycle; + private bool _disposed; + + public CultMeshSessionIdentityServer( + ICultNetSchemaServer server, + string authorityRuntimeId, + IEnumerable verseIds, + IEnumerable protocolIds, + IEnumerable routeGenerations, + IEnumerable? proofSigners = null) + { + _server = server ?? throw new ArgumentNullException(nameof(server)); + _authorityRuntimeId = Require(authorityRuntimeId, nameof(authorityRuntimeId)); + _verseIds = Clean(verseIds, nameof(verseIds)); + _protocolIds = Clean(protocolIds, nameof(protocolIds)); + _routeGenerations = Clean(routeGenerations, nameof(routeGenerations)); + _proofSigners = (proofSigners ?? Array.Empty()) + .ToDictionary(signer => signer.Route.Generation, StringComparer.Ordinal); + _handler = HandleAsync; + _server.OnCultNet(_handler); + _peerLifecycle = server as ICultNetSchemaServerPeerLifecycle; + if (_peerLifecycle != null) + _peerLifecycle.PeerDisconnected += ForgetPeer; + } + + /// + /// Resolves the runtime identity established by this peer's accepted session-open handshake. + /// Application ingress must use this transport-bound value instead of trusting payload identity fields. + /// + public bool TryGetSourceRuntimeId(ICultNetSchemaServerPeer peer, out string sourceRuntimeId) => + _acceptedPeers.TryGetValue(peer, out sourceRuntimeId!); + + public void Dispose() + { + if (_disposed) return; + _disposed = true; + _server.RemoveCultNetMessageListener(_handler); + if (_peerLifecycle != null) + _peerLifecycle.PeerDisconnected -= ForgetPeer; + _acceptedPeers.Clear(); + } + + private Task HandleAsync(CultMeshSessionOpenMessage request, ICultNetSchemaServerPeer peer) + { + var error = Validate(request); + if (error == null) + error = BindPeer(peer, request.SourceRuntimeId); + _proofSigners.TryGetValue(request.RouteGeneration ?? string.Empty, out var signer); + peer.SendCultNet(new CultMeshSessionAcceptedMessage + { + MessageId = request.MessageId ?? string.Empty, + Accepted = error == null, + VerseId = request.VerseId ?? string.Empty, + AuthorityRuntimeId = _authorityRuntimeId, + ProtocolId = request.ProtocolId ?? string.Empty, + RouteGeneration = request.RouteGeneration ?? string.Empty, + ClientNonce = request.ClientNonce ?? string.Empty, + ProviderKeyId = error == null ? signer?.ProviderKeyId ?? string.Empty : string.Empty, + ProviderSignature = error == null && signer != null ? signer.Sign(request) : string.Empty, + Error = error + }); + return Task.CompletedTask; + } + + private void ForgetPeer(ICultNetSchemaServerPeer peer) => _acceptedPeers.TryRemove(peer, out _); + + private string? BindPeer(ICultNetSchemaServerPeer peer, string sourceRuntimeId) + { + if (_acceptedPeers.TryGetValue(peer, out var established)) + return string.Equals(established, sourceRuntimeId, StringComparison.Ordinal) + ? null + : "session-source-runtime-rebind-forbidden"; + return _acceptedPeers.TryAdd(peer, sourceRuntimeId) + ? null + : BindPeer(peer, sourceRuntimeId); + } + + private string? Validate(CultMeshSessionOpenMessage request) + { + if (string.IsNullOrWhiteSpace(request.MessageId)) return "session-message-id-required"; + if (string.IsNullOrWhiteSpace(request.SourceRuntimeId)) return "source-runtime-id-required"; + if (!TryNonce(request.ClientNonce)) return "client-nonce-invalid"; + if (!string.Equals(request.AuthorityRuntimeId, _authorityRuntimeId, StringComparison.Ordinal)) + return "target-runtime-mismatch"; + if (!_verseIds.Contains(request.VerseId ?? string.Empty)) return "verse-not-served"; + if (!_protocolIds.Contains(request.ProtocolId ?? string.Empty)) return "protocol-not-served"; + if (!_routeGenerations.Contains(request.RouteGeneration ?? string.Empty)) return "route-generation-not-served"; + return null; + } + + private static bool TryNonce(string? value) + { + if (string.IsNullOrWhiteSpace(value)) return false; + try { return Convert.FromBase64String(value).Length == 32; } + catch (FormatException) { return false; } + } + + private static HashSet Clean(IEnumerable values, string name) + { + if (values == null) throw new ArgumentNullException(name); + var result = new HashSet( + values.Where(value => !string.IsNullOrWhiteSpace(value)).Select(value => value.Trim()), + StringComparer.Ordinal); + if (result.Count == 0) throw new ArgumentException("At least one identity is required.", name); + return result; + } + + private static string Require(string value, string name) => + string.IsNullOrWhiteSpace(value) + ? throw new ArgumentException("Value must be non-empty.", name) + : value.Trim(); + } + + internal static class CultMeshSessionIdentityClient + { + public static async Task VerifyAsync( + ICultNetSchemaClient client, + string sourceRuntimeId, + CultMeshSessionTarget target, + CultMeshProtocolId protocol, + CultMeshTransportCandidate candidate, + CultMeshAuthorityTrustPolicy? trust, + DateTimeOffset now, + TimeSpan timeout, + CancellationToken cancellationToken = default) + { + if (client == null) throw new ArgumentNullException(nameof(client)); + if (candidate == null) throw new ArgumentNullException(nameof(candidate)); + var route = candidate.AuthorityRoute ?? new CultMeshAuthorityRoute( + target.AuthorityRuntimeId, + candidate.Endpoint, + new[] { protocol.Value }, + candidate.Priority, + candidate.Generation); + var verifiedRoute = candidate.VerifiedAuthority ?? + (trust ?? throw new CultMeshSessionException(new CultMeshSessionFailure( + CultMeshSessionFailureReason.Authentication, + "No consumer trust policy verified this CultMesh route.", + candidate.Endpoint))).Verify(target.VerseId, route, now); + if (client is ICultMeshVerifiedSchemaClient verified) + { + if (!verified.IsVerifiedFor( + target.VerseId, + target.AuthorityRuntimeId, + protocol.Value, + candidate.Generation)) + { + throw new CultMeshSessionException(new CultMeshSessionFailure( + CultMeshSessionFailureReason.Authority, + $"Native transport identity did not match CultMesh target '{target}' for '{protocol}'.")); + } + // Native identity is sufficient only for an explicitly selected + // loopback development route. Remote schema sessions still prove + // possession of the provider key certified by Odin. + if (verifiedRoute.IsLocalDevelopment) return; + } + var messageId = Guid.NewGuid().ToString("N"); + var nonce = new byte[32]; + using (var random = RandomNumberGenerator.Create()) random.GetBytes(nonce); + var completion = new TaskCompletionSource( + TaskCreationOptions.RunContinuationsAsynchronously); + client.OnCultNet(response => + { + if (string.Equals(response.MessageId, messageId, StringComparison.Ordinal)) + completion.TrySetResult(response); + }); + var request = new CultMeshSessionOpenMessage + { + MessageId = messageId, + SourceRuntimeId = sourceRuntimeId, + VerseId = target.VerseId, + AuthorityRuntimeId = target.AuthorityRuntimeId, + ProtocolId = protocol.Value, + RouteGeneration = candidate.Generation, + ClientNonce = Convert.ToBase64String(nonce) + }; + client.SendCultNet(request); + + using var deadline = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + deadline.CancelAfter(timeout); + var cancelled = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + using (deadline.Token.Register(() => cancelled.TrySetResult(true))) + { + if (await Task.WhenAny(completion.Task, cancelled.Task).ConfigureAwait(false) != completion.Task) + { + cancellationToken.ThrowIfCancellationRequested(); + throw new CultMeshSessionException(new CultMeshSessionFailure( + CultMeshSessionFailureReason.Timeout, + $"Timed out verifying authority runtime '{target.AuthorityRuntimeId}'.")); + } + } + + var accepted = await completion.Task.ConfigureAwait(false); + if (!accepted.Accepted || + !string.Equals(accepted.VerseId, target.VerseId, StringComparison.Ordinal) || + !string.Equals(accepted.AuthorityRuntimeId, target.AuthorityRuntimeId, StringComparison.Ordinal) || + !string.Equals(accepted.ProtocolId, protocol.Value, StringComparison.Ordinal) || + !string.Equals(accepted.RouteGeneration, candidate.Generation, StringComparison.Ordinal) || + !CultMeshAuthorityProof.VerifySession(request, accepted, verifiedRoute)) + { + throw new CultMeshSessionException(new CultMeshSessionFailure( + CultMeshSessionFailureReason.Authority, + $"Route did not prove CultMesh authority '{target}' for protocol '{protocol}': " + + (accepted.Error ?? "identity-mismatch"))); + } + } + } +} diff --git a/src/GameCult.Mesh/CultMeshSessions.cs b/src/GameCult.Mesh/CultMeshSessions.cs index 87f18158..9768dfd5 100644 --- a/src/GameCult.Mesh/CultMeshSessions.cs +++ b/src/GameCult.Mesh/CultMeshSessions.cs @@ -9,18 +9,53 @@ using R3; namespace GameCult.Mesh { - public sealed class CultMeshEndpointId : IEquatable + public sealed class CultMeshRuntimeId : IEquatable { - private CultMeshEndpointId(string value) { Value = value; } + private CultMeshRuntimeId(string value) { Value = value; } public string Value { get; } - public static CultMeshEndpointId Parse(string value) => new CultMeshEndpointId( - string.IsNullOrWhiteSpace(value) ? throw new FormatException("CultMesh endpoint identity must be non-empty.") : value.Trim()); - public bool Equals(CultMeshEndpointId? other) => other != null && string.Equals(Value, other.Value, StringComparison.Ordinal); - public override bool Equals(object? obj) => Equals(obj as CultMeshEndpointId); + public static CultMeshRuntimeId Parse(string value) => new CultMeshRuntimeId( + string.IsNullOrWhiteSpace(value) ? throw new FormatException("CultMesh runtime identity must be non-empty.") : value.Trim()); + public bool Equals(CultMeshRuntimeId? other) => other != null && string.Equals(Value, other.Value, StringComparison.Ordinal); + public override bool Equals(object? obj) => Equals(obj as CultMeshRuntimeId); public override int GetHashCode() => StringComparer.Ordinal.GetHashCode(Value); public override string ToString() => Value; } + /// + /// Identifies one provider inside one Verse. Odin resolves this stable pair to + /// disposable physical routes; neither value is a socket address. + /// + public sealed class CultMeshSessionTarget : IEquatable + { + /// Creates one stable provider target inside one Verse. + public CultMeshSessionTarget(string verseId, string authorityRuntimeId) + { + VerseId = string.IsNullOrWhiteSpace(verseId) + ? throw new ArgumentException("Verse identity is required.", nameof(verseId)) + : verseId.Trim(); + AuthorityRuntimeId = CultMeshRuntimeId.Parse(authorityRuntimeId).Value; + } + + /// Gets the stable Verse identity. + public string VerseId { get; } + /// Gets the stable authority runtime identity advertised by that Verse. + public string AuthorityRuntimeId { get; } + /// Compatibility projection. Use . + [Obsolete("Use AuthorityRuntimeId. Provider identity is not a physical endpoint.")] + public string ProviderRuntimeId => AuthorityRuntimeId; + internal string SessionKey => VerseId + "\u001f" + AuthorityRuntimeId; + + public bool Equals(CultMeshSessionTarget? other) => + other != null && + string.Equals(VerseId, other.VerseId, StringComparison.Ordinal) && + string.Equals(AuthorityRuntimeId, other.AuthorityRuntimeId, StringComparison.Ordinal); + public override bool Equals(object? obj) => Equals(obj as CultMeshSessionTarget); + public override int GetHashCode() => HashCode.Combine( + StringComparer.Ordinal.GetHashCode(VerseId), + StringComparer.Ordinal.GetHashCode(AuthorityRuntimeId)); + public override string ToString() => VerseId + "/" + AuthorityRuntimeId; + } + public sealed class CultMeshProtocolId : IEquatable { public CultMeshProtocolId(string value) { Value = string.IsNullOrWhiteSpace(value) ? throw new ArgumentException("Protocol identity is required.", nameof(value)) : value; } @@ -46,15 +81,33 @@ namespace GameCult.Mesh public sealed class CultMeshTransportCandidate { - public CultMeshTransportCandidate(string endpoint, CultMeshTransportPathKind pathKind = CultMeshTransportPathKind.Direct, int priority = 0) + public CultMeshTransportCandidate( + string endpoint, + CultMeshTransportPathKind pathKind = CultMeshTransportPathKind.Direct, + int priority = 0, + string authorityRuntimeId = "", + string generation = "", + CultMeshAuthorityRoute? authorityRoute = null) { Endpoint = string.IsNullOrWhiteSpace(endpoint) ? throw new ArgumentException("Physical endpoint is required.", nameof(endpoint)) : endpoint; PathKind = pathKind; Priority = priority; + AuthorityRuntimeId = authorityRuntimeId?.Trim() ?? string.Empty; + Generation = generation?.Trim() ?? string.Empty; + AuthorityRoute = authorityRoute; } public string Endpoint { get; } public CultMeshTransportPathKind PathKind { get; } public int Priority { get; } + public string AuthorityRuntimeId { get; } + public string Generation { get; } + public CultMeshAuthorityRoute? AuthorityRoute { get; } + internal CultMeshVerifiedAuthorityRoute? VerifiedAuthority { get; private set; } + internal CultMeshTransportCandidate WithVerifiedAuthority(CultMeshVerifiedAuthorityRoute verified) + { + VerifiedAuthority = verified ?? throw new ArgumentNullException(nameof(verified)); + return this; + } } public enum CultMeshSessionStatus { Connecting, Online, Degraded, Reconnecting, Offline } @@ -110,6 +163,66 @@ namespace GameCult.Mesh CancellationToken cancellationToken = default); } + /// + /// Connects exact URI routes through a caller-supplied URI-capable CultNet client. + /// This keeps transport packages outside CultMesh while preserving advertised paths. + /// + public sealed class CultMeshUriSchemaTransportConnector : ICultMeshTransportConnector + { + private readonly HashSet _schemes; + private readonly Func _createClient; + + public CultMeshUriSchemaTransportConnector( + string connectorId, + IEnumerable schemes, + Func createClient, + int priority = 0) + { + if (string.IsNullOrWhiteSpace(connectorId)) + throw new ArgumentException("Connector identity is required.", nameof(connectorId)); + ConnectorId = connectorId.Trim(); + _schemes = new HashSet( + (schemes ?? throw new ArgumentNullException(nameof(schemes))) + .Where(value => !string.IsNullOrWhiteSpace(value)) + .Select(value => value.Trim()), + StringComparer.OrdinalIgnoreCase); + if (_schemes.Count == 0) + throw new ArgumentException("At least one URI scheme is required.", nameof(schemes)); + _createClient = createClient ?? throw new ArgumentNullException(nameof(createClient)); + Priority = priority; + } + + public string ConnectorId { get; } + public int Priority { get; } + + public bool CanConnect(CultMeshTransportCandidate candidate) => + candidate != null && + Uri.TryCreate(candidate.Endpoint, UriKind.Absolute, out var uri) && + _schemes.Contains(uri.Scheme); + + public async Task ConnectAsync( + CultMeshTransportCandidate candidate, + CultMeshProtocolId protocol, + CancellationToken cancellationToken = default) + { + if (!CanConnect(candidate)) + throw new NotSupportedException($"Connector '{ConnectorId}' does not support '{candidate?.Endpoint}'."); + var endpoint = new Uri(candidate.Endpoint, UriKind.Absolute); + var client = _createClient(candidate.Endpoint) + ?? throw new InvalidOperationException($"Connector '{ConnectorId}' returned no schema client."); + try + { + await client.ConnectAsync(endpoint, cancellationToken).ConfigureAwait(false); + return client; + } + catch + { + client.Dispose(); + throw; + } + } + } + /// /// Explicit compatibility connector for the previous RUDP/LiteNetLib schema lanes. /// Prefer for registered schemas. @@ -222,18 +335,17 @@ namespace GameCult.Mesh public sealed class CultMeshSession : IDisposable { private readonly Subject _states = new(); - private readonly ConcurrentBag _subscriptions = new(); private readonly ManagedSchemaChannel _channel; private long _physicalGeneration; private bool _disposed; - internal CultMeshSession(CultMeshEndpointId endpointId, CultMeshProtocolId protocol, ICultNetSchemaClient channel, CultMeshSessionState state) + internal CultMeshSession(CultMeshSessionTarget target, CultMeshProtocolId protocol, ICultNetSchemaClient channel, CultMeshSessionState state) { - EndpointId = endpointId; + Target = target; Protocol = protocol; _channel = new ManagedSchemaChannel(channel); State = state; } - public CultMeshEndpointId EndpointId { get; } + public CultMeshSessionTarget Target { get; } public CultMeshProtocolId Protocol { get; } internal ICultNetSchemaClient Channel => _channel; public CultMeshSessionState State { get; private set; } @@ -244,10 +356,7 @@ namespace GameCult.Mesh public IDisposable OnCultNet(Action callback) where T : ICultNetSchemaMessage { if (callback == null) throw new ArgumentNullException(nameof(callback)); - var subscription = new SessionSubscription(callback); - Channel.OnCultNet(subscription.Invoke); - _subscriptions.Add(subscription); - return subscription; + return _channel.Subscribe(callback); } internal void Transition(CultMeshSessionState state) { State = state; _states.OnNext(state); } internal void ReplacePhysicalChannel(ICultNetSchemaClient channel) @@ -259,7 +368,6 @@ namespace GameCult.Mesh { if (_disposed) return; _disposed = true; - foreach (var subscription in _subscriptions) subscription.Dispose(); _channel.Dispose(); _states.Dispose(); } @@ -267,7 +375,7 @@ namespace GameCult.Mesh private sealed class ManagedSchemaChannel : ICultNetSchemaClient { private readonly object _gate = new(); - private readonly List _registrations = new(); + private readonly Dictionary _registrations = new(); private ICultNetSchemaClient _physical; private bool _disposed; @@ -292,14 +400,23 @@ namespace GameCult.Mesh } } public void OnCultNet(Action callback) where T : ICultNetSchemaMessage + { + _ = Subscribe(callback); + } + public IDisposable Subscribe(Action callback) where T : ICultNetSchemaMessage { if (callback == null) throw new ArgumentNullException(nameof(callback)); lock (_gate) { ThrowIfDisposed(); - var registration = new ChannelRegistration(callback); - registration.Attach(_physical); - _registrations.Add(registration); + if (!_registrations.TryGetValue(typeof(T), out var existing)) + { + var created = new ChannelRegistration(); + created.Attach(_physical); + _registrations.Add(typeof(T), created); + existing = created; + } + return ((ChannelRegistration)existing).Subscribe(callback); } } public void Replace(ICultNetSchemaClient physical) @@ -310,7 +427,7 @@ namespace GameCult.Mesh { ThrowIfDisposed(); previous = _physical; - foreach (var registration in _registrations) registration.Attach(physical); + foreach (var registration in _registrations.Values) registration.Attach(physical); _physical = physical; } previous.Dispose(); @@ -323,27 +440,69 @@ namespace GameCult.Mesh if (_disposed) return; _disposed = true; physical = _physical; + foreach (var registration in _registrations.Values) registration.Dispose(); _registrations.Clear(); } physical.Dispose(); } private void ThrowIfDisposed() { if (_disposed) throw new ObjectDisposedException(nameof(ManagedSchemaChannel)); } - private interface IChannelRegistration { void Attach(ICultNetSchemaClient physical); } + private interface IChannelRegistration : IDisposable { void Attach(ICultNetSchemaClient physical); } private sealed class ChannelRegistration : IChannelRegistration where T : ICultNetSchemaMessage { - private readonly Action _callback; - public ChannelRegistration(Action callback) => _callback = callback; - public void Attach(ICultNetSchemaClient physical) => physical.OnCultNet(_callback); - } - } + private readonly object _gate = new(); + private readonly Dictionary> _callbacks = new(); + private long _nextId; + private bool _disposed; - private sealed class SessionSubscription : IDisposable where T : ICultNetSchemaMessage - { - private Action? _callback; - public SessionSubscription(Action callback) { _callback = callback; } - public void Invoke(T message) => Volatile.Read(ref _callback)?.Invoke(message); - public void Dispose() => Interlocked.Exchange(ref _callback, null); + public IDisposable Subscribe(Action callback) + { + lock (_gate) + { + if (_disposed) throw new ObjectDisposedException(nameof(ChannelRegistration)); + var id = ++_nextId; + _callbacks.Add(id, callback); + return new Subscription(this, id); + } + } + + public void Attach(ICultNetSchemaClient physical) => physical.OnCultNet(Dispatch); + + public void Dispose() + { + lock (_gate) + { + _disposed = true; + _callbacks.Clear(); + } + } + + private void Dispatch(T message) + { + Action[] callbacks; + lock (_gate) callbacks = _callbacks.Values.ToArray(); + foreach (var callback in callbacks) callback(message); + } + + private void Unsubscribe(long id) + { + lock (_gate) _callbacks.Remove(id); + } + + private sealed class Subscription : IDisposable + { + private ChannelRegistration? _owner; + private readonly long _id; + + public Subscription(ChannelRegistration owner, long id) + { + _owner = owner; + _id = id; + } + + public void Dispose() => Interlocked.Exchange(ref _owner, null)?.Unsubscribe(_id); + } + } } private sealed class SessionSchemaClient : ICultNetSchemaClient @@ -382,6 +541,11 @@ namespace GameCult.Mesh public ICultMeshClock Clock { get; set; } = CultMeshSystemClock.Instance; public ICultMeshDiagnosticSink Diagnostics { get; set; } = CultMeshNullDiagnosticSink.Instance; public int MaxRacedCandidates { get; set; } = 2; + public string RuntimeId { get; set; } = "cultmesh-client"; + public TimeSpan IdentityHandshakeTimeout { get; set; } = TimeSpan.FromSeconds(5); + /// Gets or sets the consumer-owned trust roots and explicit local-development policy. + public CultMeshAuthorityTrustPolicy Trust { get; set; } = new( + CultMeshAuthorityTrustMode.AuthenticatedRemote); } public sealed class CultMeshSessionManager : IDisposable @@ -441,21 +605,23 @@ namespace GameCult.Mesh throw new ArgumentException("Realtime transport connectors cannot contain null entries.", nameof(realtimeConnectors)); _options = options ?? new CultMeshSessionManagerOptions(); if (_options.MaxRacedCandidates <= 0) throw new ArgumentOutOfRangeException(nameof(options)); + if (string.IsNullOrWhiteSpace(_options.RuntimeId)) throw new ArgumentException("Client runtime identity is required.", nameof(options)); + if (_options.IdentityHandshakeTimeout <= TimeSpan.Zero) throw new ArgumentOutOfRangeException(nameof(options)); } public async Task ConnectAsync( - CultMeshEndpointId endpointId, + CultMeshSessionTarget target, CultMeshProtocolId protocol, CancellationToken cancellationToken = default) { - if (endpointId == null) throw new ArgumentNullException(nameof(endpointId)); + if (target == null) throw new ArgumentNullException(nameof(target)); if (protocol == null) throw new ArgumentNullException(nameof(protocol)); ThrowIfDisposed(); - var key = endpointId.Value + "\u001f" + protocol.Value; + var key = target.SessionKey + "\u001f" + protocol.Value; while (true) { var lazy = _sessions.GetOrAdd(key, _ => new Lazy>( - () => ConnectOwnedAsync(key, endpointId, protocol), LazyThreadSafetyMode.ExecutionAndPublication)); + () => ConnectOwnedAsync(key, target, protocol), LazyThreadSafetyMode.ExecutionAndPublication)); try { var session = await AwaitForCallerAsync(lazy.Value, cancellationToken).ConfigureAwait(false); @@ -471,20 +637,20 @@ namespace GameCult.Mesh } /// - /// Connects one reusable streaming content plane by stable endpoint identity. + /// Connects one reusable streaming content plane by stable Verse/provider identity. /// Preferred connector tiers are exhausted before legacy fallback tiers are attempted. /// public async Task ConnectContentAsync( - CultMeshEndpointId endpointId, + CultMeshSessionTarget target, CancellationToken cancellationToken = default) { - if (endpointId == null) throw new ArgumentNullException(nameof(endpointId)); + if (target == null) throw new ArgumentNullException(nameof(target)); ThrowIfDisposed(); - var key = endpointId.Value; + var key = target.SessionKey; while (true) { var lazy = _contentSessions.GetOrAdd(key, _ => new Lazy>( - () => ConnectContentOwnedAsync(key, endpointId), LazyThreadSafetyMode.ExecutionAndPublication)); + () => ConnectContentOwnedAsync(key, target), LazyThreadSafetyMode.ExecutionAndPublication)); try { var session = await AwaitForCallerAsync(lazy.Value, cancellationToken).ConfigureAwait(false); @@ -501,31 +667,30 @@ namespace GameCult.Mesh private async Task ConnectContentOwnedAsync( string key, - CultMeshEndpointId endpointId) + CultMeshSessionTarget target) { - var result = await ResolveContentPathAsync(endpointId).ConfigureAwait(false); + var result = await ResolveContentPathAsync(target).ConfigureAwait(false); var session = new CultMeshContentSession( - endpointId, + target, result.Transport, new CultMeshSessionState(CultMeshSessionStatus.Online, _options.Clock.UtcNow, result.Candidate), failed => InvalidateContentSession(key, failed)); - Emit(endpointId, CultMeshProtocols.Content, "online", result.Candidate.Endpoint); + Emit(target, CultMeshProtocols.Content, "online", result.Candidate.Endpoint); return session; } - private async Task ResolveContentPathAsync(CultMeshEndpointId endpointId) + private async Task ResolveContentPathAsync(CultMeshSessionTarget target) { if (_contentConnectors.Length == 0) throw Failure( CultMeshSessionFailureReason.UnsupportedPath, "No streaming content connectors are configured. Register the TCP content connector or the explicit legacy RUDP connector."); - var discovery = await _discovery.ResolveAsync(new CultMeshDiscoveryQuery(endpointId.Value)).ConfigureAwait(false); - var candidates = discovery.Candidates - .SelectMany(candidate => candidate.Descriptor.DiscoveryEndpoints) - .Distinct(StringComparer.Ordinal) - .Select(endpoint => new CultMeshTransportCandidate(endpoint)) - .ToArray(); + var discovery = await _discovery.ResolveAsync(new CultMeshDiscoveryQuery( + target.AuthorityRuntimeId, + new[] { target.VerseId }, + target.AuthorityRuntimeId)).ConfigureAwait(false); + var candidates = BoundCandidates(discovery, target, CultMeshProtocols.Content); var routes = candidates .SelectMany(candidate => _contentConnectors .Where(connector => connector.CanConnect(candidate)) @@ -536,13 +701,13 @@ namespace GameCult.Mesh if (routes.Length == 0) throw Failure( CultMeshSessionFailureReason.UnsupportedPath, - $"No streaming content connector supports an advertised route for '{endpointId}'."); + $"No streaming content connector supports an advertised route for '{target}'."); var failures = new List(); foreach (var tier in routes.GroupBy(route => route.Connector.Priority).OrderBy(group => group.Key)) { var attempts = tier.Take(_options.MaxRacedCandidates) - .Select(route => ConnectContentCandidateAsync(endpointId, route)) + .Select(route => ConnectContentCandidateAsync(target, route)) .ToList(); while (attempts.Count > 0) { @@ -566,18 +731,33 @@ namespace GameCult.Mesh } var last = failures.LastOrDefault(); + var reason = last is CultMeshSessionException typed + ? typed.Failure.Reason + : last is TimeoutException + ? CultMeshSessionFailureReason.Timeout + : CultMeshSessionFailureReason.Transport; throw Failure( - last is TimeoutException ? CultMeshSessionFailureReason.Timeout : CultMeshSessionFailureReason.Transport, - $"No streaming content path connected for '{endpointId}'.", + reason, + $"No streaming content path connected for '{target}'.", last); } private static async Task ConnectContentCandidateAsync( - CultMeshEndpointId endpointId, + CultMeshSessionTarget target, ContentRoute route) { - var transport = await route.Connector.ConnectAsync(route.Candidate, endpointId).ConfigureAwait(false); - return new ConnectedContentPath(route.Candidate, transport); + var transport = await route.Connector.ConnectAsync(route.Candidate, target).ConfigureAwait(false); + try + { + CultMeshTransportIdentity.RequireVerified( + transport, target, CultMeshProtocols.Content, route.Candidate); + return new ConnectedContentPath(route.Candidate, transport); + } + catch + { + transport.Dispose(); + throw; + } } private void InvalidateContentSession(string key, CultMeshContentSession session) @@ -593,7 +773,7 @@ namespace GameCult.Mesh if (_contentSessions.TryRemove(key, out var removed) && removed.IsValueCreated && removed.Value.Status == TaskStatus.RanToCompletion) removed.Value.Result.Dispose(); - Emit(session.EndpointId, CultMeshProtocols.Content, "offline", session.State.Path?.Endpoint ?? string.Empty); + Emit(session.Target, CultMeshProtocols.Content, "offline", session.State.Path?.Endpoint ?? string.Empty); } /// @@ -601,16 +781,16 @@ namespace GameCult.Mesh /// register QUIC explicitly, while RUDP compatibility remains opt-in. /// public async Task ConnectRealtimeAsync( - CultMeshEndpointId endpointId, + CultMeshSessionTarget target, CancellationToken cancellationToken = default) { - if (endpointId == null) throw new ArgumentNullException(nameof(endpointId)); + if (target == null) throw new ArgumentNullException(nameof(target)); ThrowIfDisposed(); - var key = endpointId.Value; + var key = target.SessionKey; while (true) { var lazy = _realtimeSessions.GetOrAdd(key, _ => new Lazy>( - () => ConnectRealtimeOwnedAsync(key, endpointId), LazyThreadSafetyMode.ExecutionAndPublication)); + () => ConnectRealtimeOwnedAsync(key, target), LazyThreadSafetyMode.ExecutionAndPublication)); try { var session = await AwaitForCallerAsync(lazy.Value, cancellationToken).ConfigureAwait(false); @@ -627,30 +807,30 @@ namespace GameCult.Mesh private async Task ConnectRealtimeOwnedAsync( string key, - CultMeshEndpointId endpointId) + CultMeshSessionTarget target) { - var result = await ResolveRealtimePathAsync(endpointId).ConfigureAwait(false); + var result = await ResolveRealtimePathAsync(target).ConfigureAwait(false); var session = new CultMeshRealtimeSession( - endpointId, + target, result.Transport, new CultMeshSessionState(CultMeshSessionStatus.Online, _options.Clock.UtcNow, result.Candidate), failed => InvalidateRealtimeSession(key, failed)); - Emit(endpointId, CultMeshProtocols.RealtimeState, "online", result.Candidate.Endpoint); + Emit(target, CultMeshProtocols.RealtimeState, "online", result.Candidate.Endpoint); return session; } - private async Task ResolveRealtimePathAsync(CultMeshEndpointId endpointId) + private async Task ResolveRealtimePathAsync(CultMeshSessionTarget target) { if (_realtimeConnectors.Length == 0) throw Failure( CultMeshSessionFailureReason.UnsupportedPath, "No realtime state connectors are configured. Register a QUIC connector explicitly."); - var discovery = await _discovery.ResolveAsync(new CultMeshDiscoveryQuery(endpointId.Value)).ConfigureAwait(false); - var routes = discovery.Candidates - .SelectMany(candidate => candidate.Descriptor.DiscoveryEndpoints) - .Distinct(StringComparer.Ordinal) - .Select(endpoint => new CultMeshTransportCandidate(endpoint)) + var discovery = await _discovery.ResolveAsync(new CultMeshDiscoveryQuery( + target.AuthorityRuntimeId, + new[] { target.VerseId }, + target.AuthorityRuntimeId)).ConfigureAwait(false); + var routes = BoundCandidates(discovery, target, CultMeshProtocols.RealtimeState) .SelectMany(candidate => _realtimeConnectors .Where(connector => connector.CanConnect(candidate)) .Select(connector => new RealtimeRoute(candidate, connector))) @@ -660,13 +840,13 @@ namespace GameCult.Mesh if (routes.Length == 0) throw Failure( CultMeshSessionFailureReason.UnsupportedPath, - $"No realtime connector supports an advertised route for '{endpointId}'."); + $"No realtime connector supports an advertised route for '{target}'."); var failures = new List(); foreach (var tier in routes.GroupBy(route => route.Connector.Priority).OrderBy(group => group.Key)) { var attempts = tier.Take(_options.MaxRacedCandidates) - .Select(route => ConnectRealtimeCandidateAsync(endpointId, route)) + .Select(route => ConnectRealtimeCandidateAsync(target, route)) .ToList(); while (attempts.Count > 0) { @@ -690,18 +870,33 @@ namespace GameCult.Mesh } var last = failures.LastOrDefault(); + var reason = last is CultMeshSessionException typed + ? typed.Failure.Reason + : last is TimeoutException + ? CultMeshSessionFailureReason.Timeout + : CultMeshSessionFailureReason.Transport; throw Failure( - last is TimeoutException ? CultMeshSessionFailureReason.Timeout : CultMeshSessionFailureReason.Transport, - $"No realtime state path connected for '{endpointId}'.", + reason, + $"No realtime state path connected for '{target}'.", last); } private static async Task ConnectRealtimeCandidateAsync( - CultMeshEndpointId endpointId, + CultMeshSessionTarget target, RealtimeRoute route) { - var transport = await route.Connector.ConnectAsync(route.Candidate, endpointId).ConfigureAwait(false); - return new ConnectedRealtimePath(route.Candidate, transport); + var transport = await route.Connector.ConnectAsync(route.Candidate, target).ConfigureAwait(false); + try + { + CultMeshTransportIdentity.RequireVerified( + transport, target, CultMeshProtocols.RealtimeState, route.Candidate); + return new ConnectedRealtimePath(route.Candidate, transport); + } + catch + { + transport.Dispose(); + throw; + } } private void InvalidateRealtimeSession(string key, CultMeshRealtimeSession session) @@ -718,26 +913,26 @@ namespace GameCult.Mesh if (_realtimeSessions.TryRemove(key, out var removed) && removed.IsValueCreated && removed.Value.Status == TaskStatus.RanToCompletion) removed.Value.Result.Dispose(); - Emit(session.EndpointId, CultMeshProtocols.RealtimeState, "offline", endpoint); + Emit(session.Target, CultMeshProtocols.RealtimeState, "offline", endpoint); } - private async Task ConnectOwnedAsync(string key, CultMeshEndpointId endpointId, CultMeshProtocolId protocol) + private async Task ConnectOwnedAsync(string key, CultMeshSessionTarget target, CultMeshProtocolId protocol) { - var result = await ResolveConnectedPathAsync(endpointId, protocol).ConfigureAwait(false); - var session = new CultMeshSession(endpointId, protocol, result.Client, + var result = await ResolveConnectedPathAsync(target, protocol).ConfigureAwait(false); + var session = new CultMeshSession(target, protocol, result.Client, new CultMeshSessionState(CultMeshSessionStatus.Online, _options.Clock.UtcNow, result.Candidate)); ObservePhysicalFailure(key, session, result); - Emit(endpointId, protocol, "online", result.Candidate.Endpoint); + Emit(target, protocol, "online", result.Candidate.Endpoint); return session; } - private async Task ResolveConnectedPathAsync(CultMeshEndpointId endpointId, CultMeshProtocolId protocol) + private async Task ResolveConnectedPathAsync(CultMeshSessionTarget target, CultMeshProtocolId protocol) { - var discovery = await _discovery.ResolveAsync(new CultMeshDiscoveryQuery(endpointId.Value)).ConfigureAwait(false); - var candidates = discovery.Candidates.SelectMany(candidate => candidate.Descriptor.DiscoveryEndpoints) - .Distinct(StringComparer.Ordinal) - .Select(endpoint => new CultMeshTransportCandidate(endpoint)) - .ToArray(); + var discovery = await _discovery.ResolveAsync(new CultMeshDiscoveryQuery( + target.AuthorityRuntimeId, + new[] { target.VerseId }, + target.AuthorityRuntimeId)).ConfigureAwait(false); + var candidates = BoundCandidates(discovery, target, protocol); var routes = candidates .SelectMany(candidate => _connectors .Where(connector => connector.CanConnect(candidate)) @@ -746,20 +941,20 @@ namespace GameCult.Mesh .ThenBy(route => route.Candidate.Priority) .ToArray(); if (candidates.Length == 0) - throw Failure(CultMeshSessionFailureReason.Resolution, $"No route candidates for '{endpointId}'."); + throw Failure(CultMeshSessionFailureReason.Resolution, $"No route candidates for '{target}'."); if (routes.Length == 0) throw Failure( CultMeshSessionFailureReason.Transport, - $"No transport path connected for '{endpointId}' and protocol '{protocol}'.", + $"No transport path connected for '{target}' and protocol '{protocol}'.", Failure( CultMeshSessionFailureReason.UnsupportedPath, - $"No registered connector supports an advertised route for '{endpointId}'.")); + $"No registered connector supports an advertised route for '{target}'.")); var failures = new List(); foreach (var tier in routes.GroupBy(route => route.Connector.Priority).OrderBy(group => group.Key)) { var attempts = tier.Take(_options.MaxRacedCandidates) - .Select(route => ConnectCandidateAsync(route, protocol)) + .Select(route => ConnectCandidateAsync(target, route, protocol)) .ToList(); while (attempts.Count > 0) { @@ -776,8 +971,13 @@ namespace GameCult.Mesh } } var last = failures.LastOrDefault(); - throw Failure(last is TimeoutException ? CultMeshSessionFailureReason.Timeout : CultMeshSessionFailureReason.Transport, - $"No transport path connected for '{endpointId}' and protocol '{protocol}'.", last); + var reason = last is CultMeshSessionException typed + ? typed.Failure.Reason + : last is TimeoutException + ? CultMeshSessionFailureReason.Timeout + : CultMeshSessionFailureReason.Transport; + throw Failure(reason, + $"No transport path connected for '{target}' and protocol '{protocol}'.", last); } private void ObservePhysicalFailure(string key, CultMeshSession session, ConnectedPath path) @@ -807,17 +1007,21 @@ namespace GameCult.Mesh _options.Clock.UtcNow, failedPath.Candidate, failure)); - Emit(session.EndpointId, session.Protocol, "reconnecting", failedPath.Candidate.Endpoint); + Emit(session.Target, session.Protocol, "reconnecting", failedPath.Candidate.Endpoint); try { - var replacement = await ResolveConnectedPathAsync(session.EndpointId, session.Protocol).ConfigureAwait(false); + _discovery.Invalidate(new CultMeshDiscoveryQuery( + session.Target.AuthorityRuntimeId, + new[] { session.Target.VerseId }, + session.Target.AuthorityRuntimeId)); + var replacement = await ResolveConnectedPathAsync(session.Target, session.Protocol).ConfigureAwait(false); session.ReplacePhysicalChannel(replacement.Client); session.Transition(new CultMeshSessionState( CultMeshSessionStatus.Online, _options.Clock.UtcNow, replacement.Candidate)); ObservePhysicalFailure(key, session, replacement); - Emit(session.EndpointId, session.Protocol, "online", replacement.Candidate.Endpoint); + Emit(session.Target, session.Protocol, "online", replacement.Candidate.Endpoint); } catch (Exception reconnectError) { @@ -830,14 +1034,34 @@ namespace GameCult.Mesh reconnectError.Message, failedPath.Candidate.Endpoint))); _sessions.TryRemove(key, out _); - Emit(session.EndpointId, session.Protocol, "offline", failedPath.Candidate.Endpoint); + Emit(session.Target, session.Protocol, "offline", failedPath.Candidate.Endpoint); } } - private static async Task ConnectCandidateAsync(SchemaRoute route, CultMeshProtocolId protocol) + private async Task ConnectCandidateAsync( + CultMeshSessionTarget target, + SchemaRoute route, + CultMeshProtocolId protocol) { var client = await route.Connector.ConnectAsync(route.Candidate, protocol).ConfigureAwait(false); - return new ConnectedPath(route.Candidate, client); + try + { + await CultMeshSessionIdentityClient.VerifyAsync( + client, + _options.RuntimeId, + target, + protocol, + route.Candidate, + _options.Trust, + _options.Clock.UtcNow, + _options.IdentityHandshakeTimeout).ConfigureAwait(false); + return new ConnectedPath(route.Candidate, client); + } + catch + { + client.Dispose(); + throw; + } } private static async Task AwaitForCallerAsync(Task shared, CancellationToken cancellationToken) @@ -855,12 +1079,62 @@ namespace GameCult.Mesh private CultMeshSessionException Failure(CultMeshSessionFailureReason reason, string message, Exception? inner = null) => new CultMeshSessionException(new CultMeshSessionFailure(reason, message), inner); - private void Emit(CultMeshEndpointId endpoint, CultMeshProtocolId protocol, string state, string path) + private CultMeshTransportCandidate[] BoundCandidates( + CultMeshDiscoveryState discovery, + CultMeshSessionTarget target, + CultMeshProtocolId protocol) + { + var routes = discovery.Candidates + .SelectMany(candidate => candidate.Descriptor.AuthorityRoutes) + .Where(route => string.Equals( + route.AuthorityRuntimeId, + target.AuthorityRuntimeId, + StringComparison.Ordinal)) + .Where(route => route.Supports(protocol)) + .ToArray(); + var verified = new List(); + var rejected = new List(); + foreach (var route in routes) + { + try + { + verified.Add(new CultMeshTransportCandidate( + route.Endpoint, + priority: route.Priority, + authorityRuntimeId: route.AuthorityRuntimeId, + generation: route.Generation, + authorityRoute: route) + .WithVerifiedAuthority(_options.Trust.Verify( + target.VerseId, + route, + _options.Clock.UtcNow))); + } + catch (CultMeshSessionException error) + { + rejected.Add(error); + } + } + if (verified.Count == 0 && rejected.Count > 0) + { + throw Failure( + CultMeshSessionFailureReason.Authentication, + $"No trusted route remained for CultMesh target '{target}' and protocol '{protocol}'.", + new AggregateException(rejected)); + } + return verified + .GroupBy(candidate => candidate.Endpoint, StringComparer.Ordinal) + .Select(group => group.OrderBy(candidate => candidate.Priority).First()) + .OrderBy(candidate => candidate.Priority) + .ThenBy(candidate => candidate.Endpoint, StringComparer.Ordinal) + .ToArray(); + } + + private void Emit(CultMeshSessionTarget target, CultMeshProtocolId protocol, string state, string path) { _options.Diagnostics.Emit(new CultMeshDiagnosticEvent( Interlocked.Increment(ref _diagnosticSequence), _options.Clock.UtcNow, CultMeshReliabilityOrgan.Session, CultMeshDiagnosticKind.PathChanged, - endpoint.Value + ":" + protocol.Value, endpoint.Value, state, + target.SessionKey + ":" + protocol.Value, target.AuthorityRuntimeId, state, sourceId: "session-manager", endpoint: path)); } diff --git a/src/GameCult.Mesh/CultMeshSimulationFacts.cs b/src/GameCult.Mesh/CultMeshSimulationFacts.cs index 6efa8711..11c7d296 100644 --- a/src/GameCult.Mesh/CultMeshSimulationFacts.cs +++ b/src/GameCult.Mesh/CultMeshSimulationFacts.cs @@ -13,6 +13,7 @@ namespace GameCult.Mesh /// Committed simulation fact derived from witness consensus. /// [CultDocument("gamecult.mesh.simulation_fact", "gamecult.mesh.simulation_fact.v1")] + [MessagePackObject] public sealed class CultMeshSimulationFact { /// diff --git a/src/GameCult.Mesh/CultMeshSnapshots.cs b/src/GameCult.Mesh/CultMeshSnapshots.cs index 03fc4aad..d40eef03 100644 --- a/src/GameCult.Mesh/CultMeshSnapshots.cs +++ b/src/GameCult.Mesh/CultMeshSnapshots.cs @@ -102,7 +102,7 @@ namespace GameCult.Mesh CultNetDocumentRegistry? registry) { _sharedSession = session ?? throw new ArgumentNullException(nameof(session)); - _endpoint = session.State.Path?.Endpoint ?? session.EndpointId.Value; + _endpoint = session.State.Path?.Endpoint ?? session.Target.AuthorityRuntimeId; _defaults = CultMesh.CloneSnapshotRequestOptions(options); _registry = registry ?? new CultNetDocumentRegistry(); _client = session.Channel; @@ -112,26 +112,26 @@ namespace GameCult.Mesh public static async Task ConnectAsync( CultMeshSessionManager sessions, - CultMeshEndpointId endpointId, + CultMeshSessionTarget target, CultMeshSnapshotRequestOptions options, CultNetDocumentRegistry? registry = null, CancellationToken cancellationToken = default) { if (sessions == null) throw new ArgumentNullException(nameof(sessions)); - var session = await sessions.ConnectAsync(endpointId, CultMeshProtocols.Documents, cancellationToken).ConfigureAwait(false); + var session = await sessions.ConnectAsync(target, CultMeshProtocols.Documents, cancellationToken).ConfigureAwait(false); return new CultMeshSnapshotSession(session, options ?? new CultMeshSnapshotRequestOptions(), registry); } public static async Task ConnectAsync( CultMeshSessionManager sessions, - CultMeshEndpointId endpointId, + CultMeshSessionTarget target, CultMeshProtocolId protocol, CultMeshSnapshotRequestOptions options, CultNetDocumentRegistry? registry = null, CancellationToken cancellationToken = default) { if (sessions == null) throw new ArgumentNullException(nameof(sessions)); - var session = await sessions.ConnectAsync(endpointId, protocol, cancellationToken).ConfigureAwait(false); + var session = await sessions.ConnectAsync(target, protocol, cancellationToken).ConfigureAwait(false); return new CultMeshSnapshotSession(session, options ?? new CultMeshSnapshotRequestOptions(), registry); } @@ -1023,7 +1023,7 @@ namespace GameCult.Mesh } else { - documents.Add(CultDocumentMessagePackSerialization.Deserialize(record.Payload)); + documents.Add((TDocument)CultDocumentMessagePackSerialization.DeserializeUntyped(typeof(TDocument), record.Payload)); } } diff --git a/src/GameCult.Mesh/CultMeshTcpContentTransport.cs b/src/GameCult.Mesh/CultMeshTcpContentTransport.cs index 0c3eb19e..dcb07feb 100644 --- a/src/GameCult.Mesh/CultMeshTcpContentTransport.cs +++ b/src/GameCult.Mesh/CultMeshTcpContentTransport.cs @@ -11,6 +11,22 @@ using GameCult.Networking; namespace GameCult.Mesh { + /// Identity served by one dedicated content listener. + public sealed class CultMeshTcpContentServerOptions + { + public string VerseId { get; set; } = string.Empty; + public string AuthorityRuntimeId { get; set; } = string.Empty; + public string RouteGeneration { get; set; } = string.Empty; + public CultMeshSessionProofSigner? ProofSigner { get; set; } + + internal void Validate() + { + if (string.IsNullOrWhiteSpace(VerseId)) throw new InvalidOperationException("Content Verse identity is required."); + if (string.IsNullOrWhiteSpace(AuthorityRuntimeId)) throw new InvalidOperationException("Content authority runtime identity is required."); + if (string.IsNullOrWhiteSpace(RouteGeneration)) throw new InvalidOperationException("Content route generation is required."); + } + } + /// Creates dedicated TCP streams for immutable CultMesh content. public sealed class CultMeshTcpContentTransportConnector : ICultMeshContentTransportConnector { @@ -33,11 +49,11 @@ namespace GameCult.Mesh public async Task ConnectAsync( CultMeshTransportCandidate candidate, - CultMeshEndpointId endpointId, + CultMeshSessionTarget target, CancellationToken cancellationToken = default) { if (candidate == null) throw new ArgumentNullException(nameof(candidate)); - if (endpointId == null) throw new ArgumentNullException(nameof(endpointId)); + if (target == null) throw new ArgumentNullException(nameof(target)); if (!TryParseEndpoint(candidate.Endpoint, out var host, out var port)) throw new NotSupportedException($"TCP content connector does not support '{candidate.Endpoint}'."); @@ -48,8 +64,10 @@ namespace GameCult.Mesh { await CultMeshTcpContentWire.AwaitAsync(client.ConnectAsync(host, port), timeout.Token) .ConfigureAwait(false); + await CultMeshTcpContentWire.VerifyAuthorityAsync( + client.GetStream(), target, candidate, timeout.Token).ConfigureAwait(false); Interlocked.Increment(ref _connectCount); - return new CultMeshTcpContentTransport(candidate.Endpoint, client); + return new CultMeshTcpContentTransport(candidate.Endpoint, client, target, candidate.Generation); } catch { @@ -72,7 +90,7 @@ namespace GameCult.Mesh } } - internal sealed class CultMeshTcpContentTransport : ICultMeshContentTransport + internal sealed class CultMeshTcpContentTransport : ICultMeshContentTransport, ICultMeshVerifiedTransport { private readonly TcpClient _client; private readonly NetworkStream _stream; @@ -80,12 +98,22 @@ namespace GameCult.Mesh private readonly ConcurrentDictionary _pending = new(StringComparer.Ordinal); private readonly CancellationTokenSource _shutdown = new(); private readonly Task _receiveLoop; + private readonly string _verseId; + private readonly string _authorityRuntimeId; + private readonly string _routeGeneration; private bool _disposed; - public CultMeshTcpContentTransport(string endpoint, TcpClient client) + public CultMeshTcpContentTransport( + string endpoint, + TcpClient client, + CultMeshSessionTarget target, + string routeGeneration) { Endpoint = endpoint; _client = client ?? throw new ArgumentNullException(nameof(client)); + _verseId = target?.VerseId ?? throw new ArgumentNullException(nameof(target)); + _authorityRuntimeId = target.AuthorityRuntimeId; + _routeGeneration = routeGeneration ?? string.Empty; _stream = client.GetStream(); _receiveLoop = ReceiveLoopAsync(_shutdown.Token); } @@ -93,6 +121,16 @@ namespace GameCult.Mesh public string TransportId => "tcp-content"; public string Endpoint { get; } + public bool IsVerifiedFor( + string verseId, + string authorityRuntimeId, + string protocolId, + string routeGeneration) => + string.Equals(_verseId, verseId, StringComparison.Ordinal) && + string.Equals(_authorityRuntimeId, authorityRuntimeId, StringComparison.Ordinal) && + string.Equals(CultMeshProtocols.Content.Value, protocolId, StringComparison.Ordinal) && + string.Equals(_routeGeneration, routeGeneration, StringComparison.Ordinal); + public async Task CopyChunkToAsync( CultMeshCdnChunkRef chunk, Stream destination, @@ -212,6 +250,7 @@ namespace GameCult.Mesh private readonly TcpListener _listener; private readonly Func _resolveChunk; private readonly Func _canServeHash; + private readonly CultMeshTcpContentServerOptions _options; private readonly CancellationTokenSource _shutdown = new(); private readonly ConcurrentDictionary _clients = new(); private readonly Task _acceptLoop; @@ -221,8 +260,9 @@ namespace GameCult.Mesh public CultMeshTcpContentServer( TcpListener listener, CultCache content, + CultMeshTcpContentServerOptions options, Func? canServeHash = null) - : this(listener, CreateCacheResolver(content), canServeHash) + : this(listener, CreateCacheResolver(content), options, canServeHash) { } @@ -234,10 +274,13 @@ namespace GameCult.Mesh public CultMeshTcpContentServer( TcpListener listener, Func resolveChunk, + CultMeshTcpContentServerOptions options, Func? canServeHash = null) { _listener = listener ?? throw new ArgumentNullException(nameof(listener)); _resolveChunk = resolveChunk ?? throw new ArgumentNullException(nameof(resolveChunk)); + _options = options ?? throw new ArgumentNullException(nameof(options)); + _options.Validate(); _canServeHash = canServeHash ?? (_ => true); _listener.Start(); _acceptLoop = AcceptLoopAsync(_shutdown.Token); @@ -278,6 +321,7 @@ namespace GameCult.Mesh try { using var stream = client.GetStream(); + if (!await AcceptAuthorityAsync(stream, cancellationToken).ConfigureAwait(false)) return; while (!cancellationToken.IsCancellationRequested) { var requestBytes = await CultMeshTcpContentWire.ReadFrameAsync(stream, cancellationToken).ConfigureAwait(false); @@ -296,6 +340,47 @@ namespace GameCult.Mesh } } + private async Task AcceptAuthorityAsync(Stream stream, CancellationToken cancellationToken) + { + var requestBytes = await CultMeshTcpContentWire.ReadFrameAsync(stream, cancellationToken).ConfigureAwait(false); + var request = CultNetSchemaMessageSerialization.Deserialize(requestBytes) as CultMeshSessionOpenMessage + ?? throw new InvalidDataException("TCP content session did not begin with an authority handshake."); + var accepted = !string.IsNullOrWhiteSpace(request.MessageId) && + !string.IsNullOrWhiteSpace(request.SourceRuntimeId) && + IsNonce(request.ClientNonce) && + string.Equals(request.VerseId, _options.VerseId, StringComparison.Ordinal) && + string.Equals(request.AuthorityRuntimeId, _options.AuthorityRuntimeId, StringComparison.Ordinal) && + string.Equals(request.ProtocolId, CultMeshProtocols.Content.Value, StringComparison.Ordinal) && + string.Equals(request.RouteGeneration, _options.RouteGeneration, StringComparison.Ordinal); + var response = new CultMeshSessionAcceptedMessage + { + MessageId = request.MessageId, + Accepted = accepted, + VerseId = _options.VerseId, + AuthorityRuntimeId = _options.AuthorityRuntimeId, + ProtocolId = CultMeshProtocols.Content.Value, + RouteGeneration = _options.RouteGeneration, + ClientNonce = request.ClientNonce ?? string.Empty, + ProviderKeyId = accepted ? _options.ProofSigner?.ProviderKeyId ?? string.Empty : string.Empty, + ProviderSignature = accepted && _options.ProofSigner != null + ? _options.ProofSigner.Sign(request) + : string.Empty, + Error = accepted ? null : "Content session target does not match this authority route." + }; + await CultMeshTcpContentWire.WriteFrameAsync( + stream, + CultNetSchemaMessageSerialization.Serialize(response), + cancellationToken).ConfigureAwait(false); + return accepted; + } + + private static bool IsNonce(string? value) + { + if (string.IsNullOrWhiteSpace(value)) return false; + try { return Convert.FromBase64String(value).Length == 32; } + catch (FormatException) { return false; } + } + private async Task SendChunkAsync( Stream stream, CultMeshContentChunkRequestMessage request, @@ -391,6 +476,48 @@ namespace GameCult.Mesh await stream.FlushAsync(cancellationToken).ConfigureAwait(false); } + public static async Task VerifyAuthorityAsync( + Stream stream, + CultMeshSessionTarget target, + CultMeshTransportCandidate candidate, + CancellationToken cancellationToken) + { + var request = new CultMeshSessionOpenMessage + { + MessageId = Guid.NewGuid().ToString("N"), + SourceRuntimeId = "cultmesh-content-client", + VerseId = target.VerseId, + AuthorityRuntimeId = target.AuthorityRuntimeId, + ProtocolId = CultMeshProtocols.Content.Value, + RouteGeneration = candidate.Generation, + ClientNonce = CreateNonce() + }; + await WriteFrameAsync(stream, CultNetSchemaMessageSerialization.Serialize(request), cancellationToken) + .ConfigureAwait(false); + var responseBytes = await ReadFrameAsync(stream, cancellationToken).ConfigureAwait(false); + var response = CultNetSchemaMessageSerialization.Deserialize(responseBytes) as CultMeshSessionAcceptedMessage + ?? throw new InvalidDataException("TCP content authority handshake returned an unexpected schema."); + if (!string.Equals(response.MessageId, request.MessageId, StringComparison.Ordinal) || + !response.Accepted || + !string.Equals(response.VerseId, target.VerseId, StringComparison.Ordinal) || + !string.Equals(response.AuthorityRuntimeId, target.AuthorityRuntimeId, StringComparison.Ordinal) || + !string.Equals(response.ProtocolId, CultMeshProtocols.Content.Value, StringComparison.Ordinal) || + !string.Equals(response.RouteGeneration, candidate.Generation, StringComparison.Ordinal) || + candidate.VerifiedAuthority == null || + !CultMeshAuthorityProof.VerifySession(request, response, candidate.VerifiedAuthority)) + throw new CultMeshSessionException(new CultMeshSessionFailure( + CultMeshSessionFailureReason.Authority, + response.Error ?? "TCP content endpoint did not prove the selected authority route.", + candidate.Endpoint)); + } + + private static string CreateNonce() + { + var nonce = new byte[32]; + using (var random = System.Security.Cryptography.RandomNumberGenerator.Create()) random.GetBytes(nonce); + return Convert.ToBase64String(nonce); + } + public static async Task ReadFrameAsync(Stream stream, CancellationToken cancellationToken) { var header = new byte[HeaderBytes]; diff --git a/src/GameCult.Mesh/CultMeshVerse.cs b/src/GameCult.Mesh/CultMeshVerse.cs index 547a46ec..003dc161 100644 --- a/src/GameCult.Mesh/CultMeshVerse.cs +++ b/src/GameCult.Mesh/CultMeshVerse.cs @@ -91,6 +91,57 @@ namespace GameCult.Mesh } } + /// + /// Binds one authoritative runtime to one physical route. This relationship is + /// the routing authority; the legacy endpoint/runtime lists are projections only. + /// + public sealed class CultMeshAuthorityRoute + { + public CultMeshAuthorityRoute( + string authorityRuntimeId, + string endpoint, + IEnumerable? protocolIds = null, + int priority = 0, + string? generation = null, + CultMeshRouteCertificate? certificate = null) + { + AuthorityRuntimeId = RequireNonEmpty(authorityRuntimeId, nameof(authorityRuntimeId)); + Endpoint = RequireNonEmpty(endpoint, nameof(endpoint)); + ProtocolIds = Clean(protocolIds); + if (priority < 0) throw new ArgumentOutOfRangeException(nameof(priority)); + Priority = priority; + Generation = string.IsNullOrWhiteSpace(generation) + ? AuthorityRuntimeId + "@" + Endpoint + : generation!.Trim(); + Certificate = certificate; + } + + public string AuthorityRuntimeId { get; } + public string Endpoint { get; } + public IReadOnlyList ProtocolIds { get; } + public int Priority { get; } + public string Generation { get; } + /// Gets the optional Odin-signed binding from this route to its provider key. + public CultMeshRouteCertificate? Certificate { get; } + + public bool Supports(CultMeshProtocolId protocol) => + protocol != null && + (ProtocolIds.Count == 0 || ProtocolIds.Contains(protocol.Value, StringComparer.Ordinal)); + + private static string[] Clean(IEnumerable? values) => + values?.Where(value => !string.IsNullOrWhiteSpace(value)) + .Select(value => value.Trim()) + .Distinct(StringComparer.Ordinal) + .OrderBy(value => value, StringComparer.Ordinal) + .ToArray() + ?? Array.Empty(); + + private static string RequireNonEmpty(string value, string paramName) => + string.IsNullOrWhiteSpace(value) + ? throw new ArgumentException("Value must be non-empty.", paramName) + : value.Trim(); + } + /// /// Describes one CultMesh Verse: a rule-bearing consensus graph. /// @@ -107,14 +158,36 @@ namespace GameCult.Mesh IEnumerable? discoveryEndpoints = null, IEnumerable? authorityRuntimeIds = null, string? parentVerseId = null, - string? description = null) + string? description = null, + IEnumerable? authorityRoutes = null) { VerseId = RequireNonEmpty(verseId, nameof(verseId)); DisplayName = RequireNonEmpty(displayName, nameof(displayName)); AuthorityModel = authorityModel; Compatibility = compatibility ?? throw new ArgumentNullException(nameof(compatibility)); - DiscoveryEndpoints = Clean(discoveryEndpoints); - AuthorityRuntimeIds = Clean(authorityRuntimeIds); + var legacyEndpoints = Clean(discoveryEndpoints); + var legacyRuntimeIds = Clean(authorityRuntimeIds); + var suppliedRoutes = authorityRoutes?.ToArray(); + if (suppliedRoutes?.Any(route => route == null) == true) + throw new ArgumentException("Authority routes cannot contain null.", nameof(authorityRoutes)); + if (suppliedRoutes != null && (legacyEndpoints.Length > 0 || legacyRuntimeIds.Length > 0)) + throw new ArgumentException( + "Supply bound authority routes or legacy endpoint/runtime lists, never both.", + nameof(authorityRoutes)); + if (suppliedRoutes == null && legacyEndpoints.Length > 0 && legacyRuntimeIds.Length > 1) + throw new InvalidOperationException( + "Legacy Verse routing with multiple authority runtimes is ambiguous. " + + "Publish explicit CultMeshAuthorityRoute bindings."); + + AuthorityRoutes = NormalizeRoutes(suppliedRoutes ?? + (legacyRuntimeIds.Length == 1 + ? legacyEndpoints.Select(endpoint => new CultMeshAuthorityRoute(legacyRuntimeIds[0], endpoint)) + : Array.Empty())); + DiscoveryEndpoints = AuthorityRoutes.Count > 0 + ? AuthorityRoutes.Select(route => route.Endpoint).Distinct(StringComparer.Ordinal).ToArray() + : legacyEndpoints; + AuthorityRuntimeIds = AuthorityRoutes.Select(route => route.AuthorityRuntimeId) + .Distinct(StringComparer.Ordinal).OrderBy(value => value, StringComparer.Ordinal).ToArray(); ParentVerseId = parentVerseId; Description = description; } @@ -143,6 +216,8 @@ namespace GameCult.Mesh /// Gets known authoritative runtime ids, when authority is cluster-shaped. /// public IReadOnlyList AuthorityRuntimeIds { get; } + /// Gets exact authority-to-route bindings used for identity-sensitive resolution. + public IReadOnlyList AuthorityRoutes { get; } /// /// Gets the parent Verse id for subscribed overlays or branches. /// @@ -177,10 +252,29 @@ namespace GameCult.Mesh private static string[] Clean(IEnumerable? values) { - return values?.Where(value => !string.IsNullOrWhiteSpace(value)).Distinct(StringComparer.Ordinal).ToArray() + return values?.Where(value => !string.IsNullOrWhiteSpace(value)) + .Select(value => value.Trim()) + .Distinct(StringComparer.Ordinal) + .ToArray() ?? Array.Empty(); } + private static CultMeshAuthorityRoute[] NormalizeRoutes(IEnumerable routes) + { + return routes + .GroupBy(route => new + { + route.AuthorityRuntimeId, + route.Endpoint, + Protocols = string.Join("\u001f", route.ProtocolIds) + }) + .Select(group => group.OrderBy(route => route.Priority).First()) + .OrderBy(route => route.Priority) + .ThenBy(route => route.AuthorityRuntimeId, StringComparer.Ordinal) + .ThenBy(route => route.Endpoint, StringComparer.Ordinal) + .ToArray(); + } + private static string RequireNonEmpty(string value, string paramName) { if (string.IsNullOrWhiteSpace(value)) diff --git a/src/GameCult.Mesh/CultMeshVerseDiscovery.cs b/src/GameCult.Mesh/CultMeshVerseDiscovery.cs index 69cc9e12..1e98fc16 100644 --- a/src/GameCult.Mesh/CultMeshVerseDiscovery.cs +++ b/src/GameCult.Mesh/CultMeshVerseDiscovery.cs @@ -33,6 +33,24 @@ namespace GameCult.Mesh }, DiscoveryEndpoints = verse.DiscoveryEndpoints.ToArray(), AuthorityRuntimeIds = verse.AuthorityRuntimeIds.ToArray(), + AuthorityRoutes = verse.AuthorityRoutes.Select(route => new CultMeshAuthorityRouteMessage + { + AuthorityRuntimeId = route.AuthorityRuntimeId, + Endpoint = route.Endpoint, + ProtocolIds = route.ProtocolIds.ToArray(), + Priority = route.Priority, + Generation = route.Generation, + Certificate = route.Certificate == null ? null : new CultMeshRouteCertificateMessage + { + ProviderKeyId = route.Certificate.ProviderKey.KeyId, + ProviderPublicKeyX = route.Certificate.ProviderKey.X, + ProviderPublicKeyY = route.Certificate.ProviderKey.Y, + OdinKeyId = route.Certificate.OdinKeyId, + IssuedAtUnixMilliseconds = route.Certificate.IssuedAtUnixMilliseconds, + ExpiresAtUnixMilliseconds = route.Certificate.ExpiresAtUnixMilliseconds, + Signature = route.Certificate.Signature + } + }).ToArray(), ParentVerseId = verse.ParentVerseId, Description = verse.Description }; @@ -50,6 +68,21 @@ namespace GameCult.Mesh out var parsed) ? parsed : CultMeshVerseAuthorityModel.SubscribedOverlay; + var routes = message.AuthorityRoutes?.Select(route => new CultMeshAuthorityRoute( + route.AuthorityRuntimeId, + route.Endpoint, + route.ProtocolIds, + route.Priority, + route.Generation, + route.Certificate == null ? null : new CultMeshRouteCertificate( + new CultMeshEcdsaP256PublicKey( + route.Certificate.ProviderKeyId, + route.Certificate.ProviderPublicKeyX, + route.Certificate.ProviderPublicKeyY), + route.Certificate.OdinKeyId, + route.Certificate.IssuedAtUnixMilliseconds, + route.Certificate.ExpiresAtUnixMilliseconds, + route.Certificate.Signature))).ToArray(); return new CultMeshVerseDescriptor( message.VerseId, message.DisplayName, @@ -60,10 +93,38 @@ namespace GameCult.Mesh message.Compatibility.CompatibleVerseIds, message.Compatibility.RequiredPluginIds, message.Compatibility.OptionalPluginIds), - message.DiscoveryEndpoints, - message.AuthorityRuntimeIds, + routes is { Length: > 0 } ? null : message.DiscoveryEndpoints, + routes is { Length: > 0 } ? null : message.AuthorityRuntimeIds, message.ParentVerseId, - message.Description); + message.Description, + routes is { Length: > 0 } ? routes : null); + } + + /// Creates the canonical filtered Verse catalog response for any CultNet transport adapter. + public static CultMeshVerseCatalogResponseMessage CreateCatalogResponse( + CultMeshVerseCatalog catalog, + CultMeshVerseCatalogRequestMessage request) + { + if (catalog == null) throw new ArgumentNullException(nameof(catalog)); + if (request == null) throw new ArgumentNullException(nameof(request)); + var verseIds = request.VerseIds == null || request.VerseIds.Length == 0 + ? null + : new HashSet(request.VerseIds, StringComparer.Ordinal); + return new CultMeshVerseCatalogResponseMessage + { + MessageId = string.IsNullOrWhiteSpace(request.MessageId) + ? Guid.NewGuid().ToString("N") + : request.MessageId, + Verses = catalog.Verses + .Where(verse => verseIds == null || verseIds.Contains(verse.VerseId)) + .Where(verse => string.IsNullOrWhiteSpace(request.TransportVersion) || + string.Equals( + verse.Compatibility.TransportVersion, + request.TransportVersion, + StringComparison.Ordinal)) + .Select(verse => verse.ToMessage()) + .ToArray() + }; } } @@ -93,24 +154,7 @@ namespace GameCult.Mesh /// public CultMeshVerseCatalogResponseMessage CreateResponse(CultMeshVerseCatalogRequestMessage request) { - if (request == null) throw new ArgumentNullException(nameof(request)); - var verseIds = request.VerseIds == null || request.VerseIds.Length == 0 - ? null - : new HashSet(request.VerseIds, StringComparer.Ordinal); - var verses = _catalog.Verses - .Where(verse => verseIds == null || verseIds.Contains(verse.VerseId)) - .Where(verse => string.IsNullOrWhiteSpace(request.TransportVersion) || - string.Equals(verse.Compatibility.TransportVersion, request.TransportVersion, StringComparison.Ordinal)) - .Select(verse => verse.ToMessage()) - .ToArray(); - - return new CultMeshVerseCatalogResponseMessage - { - MessageId = string.IsNullOrWhiteSpace(request.MessageId) - ? Guid.NewGuid().ToString("N") - : request.MessageId, - Verses = verses - }; + return CultMeshVerseMessages.CreateCatalogResponse(_catalog, request); } /// @@ -163,6 +207,12 @@ namespace GameCult.Mesh /// public Func? CreateClient { get; set; } + /// + /// Gets or sets an endpoint-aware client factory for external transport packages. + /// When present, this owns discovery transport selection for each exact Odin route. + /// + public Func? CreateClientForEndpoint { get; set; } + /// Gets or sets the clock that owns discovery deadlines. public ICultMeshClock Clock { get; set; } = CultMeshSystemClock.Instance; @@ -172,6 +222,9 @@ namespace GameCult.Mesh /// Gets or sets the identity of this bootstrap lookup source. public string SourceId { get; set; } = "configured-bootstrap"; + /// Gets or sets the CultMesh transport contract requested from rendezvous. + public string TransportVersion { get; set; } = "cultmesh.v0"; + /// Gets or sets how long a successful compatibility observation remains fresh. public TimeSpan ObservationTtl { get; set; } = TimeSpan.FromMinutes(5); @@ -205,7 +258,6 @@ namespace GameCult.Mesh CultMeshVerseCatalogRequestMessage? request = null) { if (string.IsNullOrWhiteSpace(endpoint)) throw new ArgumentException("Value must be non-empty.", nameof(endpoint)); - var (host, port) = CultNetSchemaWriteForwarder.ParseEndpoint(endpoint); var messageId = Guid.NewGuid().ToString("N"); var completion = new TaskCompletionSource( TaskCreationOptions.RunContinuationsAsynchronously); @@ -224,7 +276,13 @@ namespace GameCult.Mesh Emit(CultMeshDiagnosticKind.ConnectionAttempt, messageId, endpoint, "connecting"); try { - client.Connect(host, port); + if (client is ICultNetUriSchemaClient uriClient) + await uriClient.ConnectAsync(new Uri(endpoint)).ConfigureAwait(false); + else + { + var (host, port) = CultNetSchemaWriteForwarder.ParseEndpoint(endpoint); + client.Connect(host, port); + } var backgroundFailure = (client as ICultNetSchemaClientHealth)?.BackgroundFailure; await WaitForConnectionAsync(client, endpoint, backgroundFailure).ConfigureAwait(false); client.SendCultNet(new CultMeshVerseCatalogRequestMessage @@ -289,7 +347,8 @@ namespace GameCult.Mesh private ICultNetSchemaClient CreateClient(string endpoint) { - return _options.CreateClient?.Invoke() + return _options.CreateClientForEndpoint?.Invoke(endpoint) + ?? _options.CreateClient?.Invoke() ?? CultNetSchemaClients.CreateForEndpoint(endpoint, _options.Security, _options.ConfigureClient); } diff --git a/src/GameCult.Mesh/README.md b/src/GameCult.Mesh/README.md index f6e0ea30..8acb6c9d 100644 --- a/src/GameCult.Mesh/README.md +++ b/src/GameCult.Mesh/README.md @@ -18,12 +18,25 @@ simulation observations. ## First Node ```csharp +using GameCult.Caching; using GameCult.Mesh; +using GameCult.Networking; -using var node = await CultMesh.StartNodeAsync("world.ccmp"); +// CounterState is the typed document defined in the getting-started tutorial. +var documents = new CultNetDocumentRegistry() + .Register(CultNetDocumentBinding.ForDocument()); +using var node = await CultMesh.StartNodeAsync("world.cc", new CultMeshNodeOptions +{ + DatabaseOptions = new CultNetDatabaseOptions + { + RuntimeId = "counter-provider", + DocumentRegistry = documents + } +}); -var player = await node.Database.GetAsync(playerKey); -await node.Database.PutAsync(playerKey, player); +var key = new CultRecordKey("counter:main"); +await node.Database.PutAsync(key, new CounterState { CounterId = key.Value }); +var counter = await node.Database.GetAsync(key); ``` If you want one inspectable path that starts with a durable local cache file and @@ -34,13 +47,17 @@ wire, see [docs/typed-document-path.md](docs/typed-document-path.md). ## Eve / CultUI Surfaces -CultMesh includes the portable Eve/CultUI surface document contract and a fluent -builder for daemon-published UI. The builder is deliberately shaped like the -legacy CultUI/Unity properties panel API: compose menus and forms in readable -imperative code, while the emitted document carries typed CultMesh operation -bindings, state bindings, style tokens, and embedded surface slots. +CultMesh supplies the typed state and operation binding descriptors used by Eve. +The renderer-neutral `GameCult.Eve.Surface` package in the Eve repository owns +the portable surface document and fluent builder. Add that package when a daemon +publishes UI; do not add EveUnity to the provider. The emitted document carries +typed CultMesh operation bindings, state bindings, style tokens, and embedded +surface slots while remaining independent of any lowering runtime. ```csharp +using GameCult.Eve.Surface; +using GameCult.Mesh; + var menu = EveSurface.Create("aetheria.main_menu.root") .Provider("aetheria", "game.menu") .TitleSubtitle("AETHERIA", "TERMINUS") @@ -84,7 +101,7 @@ Enable durable authoritative shard logs when a node should serve replica catch-up after restart: ```csharp -using var node = await CultMesh.StartNodeAsync("world.ccmp", new CultMeshNodeOptions +using var node = await CultMesh.StartNodeAsync("world.cc", new CultMeshNodeOptions { EnableDurableShardLogs = true }); @@ -236,7 +253,7 @@ shape without local store plumbing: ```csharp var health = CultMesh.DocumentFromSingleFile( - "daemon-publication.ccmp", + "daemon-publication.cc", new CultRecordKey("daemon:aetheria.health.v1"), verse); @@ -388,6 +405,11 @@ receives one shared-memory bootstrap descriptor, then observes later commits through the mapped ring header. Per-frame CultNet documents are neither needed nor desirable. +The mapped ring uses a named memory map on Windows and a capability-named, +publisher-owned temporary backing file on Unix. This distinction belongs to +the CultMesh adapter; providers and consumers use the same descriptor/cursor +contract on both platforms. + ```csharp using var entities = await CultMesh.SubscribeLiveBodyAsync( subscriptions, diff --git a/src/GameCult.Mesh/docs/browser-verse-transport.md b/src/GameCult.Mesh/docs/browser-verse-transport.md new file mode 100644 index 00000000..fcfde998 --- /dev/null +++ b/src/GameCult.Mesh/docs/browser-verse-transport.md @@ -0,0 +1,221 @@ +# Browser Verse Transport Boundary + +Date: 2026-08-17 + +## Status + +This is the implementation map for the browser Verse boundary. +`samples/eve-two-runtime` remains the clean package-consumer/jsdom checkpoint. +`samples/eve-browser-network` runs a durable C# provider, a real Chromium Eve +lowerer, an independent C# headless observer, and a separate local Odin fixture +over binary loopback CultNet WebSocket routes under an explicit +`local-development` trust policy. It proves one canonical +command effect, idempotent retry, shared observation, provider restart on a new +physical route, browser rediscovery, retained lease resubscription, and a second +canonical command after reconnection. + +The current substrate has a browser client boundary and local live proof, but +not a complete public Verse proof: + +- `cultmesh-ts` is a Node package and imports `node:dgram` at its root; +- `cultnet-ts/contracts` is the browser-safe contract entrypoint; the package + root still includes Node transport, filesystem, crypto, stream, and UDP + modules and must not enter browser bundles; +- C# and TypeScript now share `cultnet.database_subscribe.v0`, + `cultnet.database_unsubscribe.v0`, and `cultnet.database_change_raw.v0`; + `cultmesh-browser` consumes those messages as explicit document leases; +- `cultmesh-browser` owns stable identity, route refresh, resubscription, and + typed operation correlation without exposing direct document writes; +- `CultMeshBrowserOdinRendezvous` resolves stable Verse/provider identity with + `cultmesh.verse_catalog_request.v0` and + `cultmesh.verse_catalog_response.v0`; application callbacks no longer need to + smuggle provider URLs into the browser client; +- `CultMeshBrowserClient` owns the final authority gate: remote use requires a + consumer-pinned Odin P-256 root, an Odin-signed route certificate, `wss://`, + and provider possession proof over a fresh nonce. A catalog source cannot + bypass that gate by returning a route-shaped object; +- Kotlin has a channel-aware WebSocket transport adapter, TypeScript has the + browser client, and `GameCult.Networking.WebSockets` provides the bounded C# + schema server host adapter; +- Hermodr can lower Odin state into a browser, but it is an infrastructure + runtime, not the clean-consumer game API; +- the sample's separate local Odin fixture serves the production Verse-catalog + contract and is restarted with a changed provider route; the same Chromium + client automatically rediscovers that route. +- the sample validates its surface against Eve's canonical TypeScript contract, + but its provider still carries a minimal local C# DTO mirror. The + released-artifact gate requires the renderer-neutral C# Eve surface contract + as an independently consumable Eve artifact; Unity must not be that contract's + package owner. + +## Authority Map + +Owner: + +- CultNet owns binary WebSocket framing and schema-v0 message validation. +- CultMesh owns stable Verse/provider identity, document leases, + subscriptions, route replacement, and command/receipt correlation. +- Odin owns discovery of the current `wss://` candidate for a stable Verse and + provider identity. +- The provider daemon owns application state, operations, receipts, and the Eve + surface. +- Eve owns surface and command contracts; the browser lowerer owns DOM only. + +Inputs: + +- an Odin-discovered route certificate and consumer-configured Odin root; +- stable Verse and provider identities; +- provider key possession over a fresh client nonce; +- requested schema ids and record keys; +- typed operation invocations and idempotency keys; +- canonical CultNet schema-v0 MessagePack messages. + +Outputs: + +- leased typed document snapshots and live changes; +- typed operation receipts; +- explicit connection/session diagnostics; +- deterministic unsubscribe and disposal; +- route replacement without changing provider or document identity. + +Derived state: + +- browser caches, DOM trees, loading/error presentation, and optimistic visual + affordances are projections only; +- a socket URL is a disposable discovery candidate, never provider identity; +- reconnect status is observable session state, not application truth. + +Forbidden writers: + +- browser lowerers cannot mutate provider documents directly; +- Hermodr, service workers, renderer caches, and app-specific HTTP endpoints + cannot become alternate gameplay authorities; +- no JSON polling API may mirror the database as a second state system; +- `node:dgram`, Node streams, filesystem stores, and Node crypto cannot enter a + browser bundle; +- a late socket, stale subscription generation, or reconnect replay cannot + overwrite a newer session; +- application code cannot own reconnect sleeps, endpoint ranking, or + resubscription loops. + +Shared paths: + +- initial load, live update, reconnect, tab background/resume, command retry, + and provider route rotation all use one CultMesh document-lease/session path; +- browser, Unity, headless C#, and future native clients consume the same + provider-owned documents and receipts; +- every command crosses the same typed operation boundary regardless of which + Eve lowerer emitted it. + +Deletion line: + +- the jsdom checkpoint remains labeled as a Node-hosted DOM proof; +- browser applications import `cultmesh-browser` and + `cultnet-ts/contracts`, never the Node-only package roots; +- subscription messages remain one shared schema contract across the C# and + TypeScript catalogs; +- the real provider adapter must consume the canonical WebSocket message lane; + no sample-only WebSocket protocol is permitted. + +## Required Runtime Shape + +The browser-facing package must be importable by a standards-mode browser or a +normal bundler without Node polyfills. It exposes a narrow API shaped like: + +```ts +const mesh = await CultMeshBrowserClient.connect({ + rendezvous: odin, + verseId: "sample.counter", + providerId: "sample.counter-provider", + runtimeId: "sample.browser", +}); + +using surface = await mesh.leaseRawDocument({ + schemaId: "gamecult.eve.surface.v1", + recordKey: "eve:surface:counter", +}); +surface.watch(record => render(decodeCultNetPayload(record))); + +const receipt = await mesh.invoke({ + serviceId: "sample.counter", + operation: "increment", + payloadSchema: "sample.increment.v1", + payload: {}, + idempotencyKey: crypto.randomUUID(), +}); +``` + +The names are illustrative. The ownership is not: application code supplies +identity and intent; CultMesh owns route/session continuity; the provider owns +the result. + +## Wire Contract + +WebSocket carries binary MessagePack CultNet schema-v0 messages. The first +implementation must support: + +- `cultnet.hello.v0`; +- authentication/session verification; +- `cultnet.snapshot_request.v0` and + `cultnet.snapshot_response_raw.v0`; +- `cultnet.database_subscribe.v0`, + `cultnet.database_unsubscribe.v0`, and + `cultnet.database_change_raw.v0`; +- `cultnet.operation_request.v0` and + `cultnet.operation_response.v0`; +- `cultmesh.verse_catalog_request.v0` and + `cultmesh.verse_catalog_response.v0` on the Odin route; +- `cultnet.error.v0`. + +The WebSocket frame is a transport body, not a second message schema. Payload +size, queue depth, subscription count, and command rate are bounded. Schema, +identity, authority, and idempotency validation happen before provider code. + +## Acceptance Gate + +Current executable coverage from `scripts/verify-eve-browser-network.mjs`: + +- authenticated binary C# WebSocket provider; +- separate local Odin fixture using the canonical Verse-catalog messages; +- real Chromium and retained C# headless consumers, both discovered through + the same Odin identity; +- shared surface and counter document leases; +- Eve button to typed operation to provider receipt to shared state; +- duplicate idempotency key produces one effect; +- provider restart on a different physical port rehydrates the `.cc` state; +- the retained browser lease rediscovers through Odin and resubscribes; +- the retained C# client independently rediscovers, resubscribes, invokes the + second command through `CultMeshClient.InvokeAsync`, and receives the + provider-authored receipt without application transport or reconnect code; +- Chromium observes the C# command's canonical state and receipt chronology; +- 10,000 typed no-op operations traverse the retained C# WebSocket session; + the gate records throughput, p99, post-GC managed growth, and private-memory + movement without growing provider state; +- generated browser bundle contains no `node:` builtin import. + +The full public Verse gate still requires the deployed Odin daemon rather than +the local contract fixture and released-artifact-only .NET consumption. The +source-contract gate runs on Windows and Linux CI. + +From an empty temporary consumer using packed/released artifacts only: + +1. start a provider daemon with one durable counter document and one validated + Eve surface; +2. advertise it by stable identity through a local Odin fixture or real Odin; +3. connect one real Chromium page and one C# headless client; +4. lease the same surface and counter document in both runtimes; +5. click the browser button and require one provider-authored canonical receipt; +6. observe the same state version and receipt id in both clients; +7. disconnect or rotate the physical WebSocket endpoint, then require both + leases to resume without application reconnect code; +8. invoke from C# after route replacement and require Chromium to observe the + same provider-authored state and receipt; +9. restart the provider and prove state plus receipt persistence; +10. run a non-persistent operation workload long enough to falsify retained + callback or managed-heap growth; +11. reject malformed schemas, direct document writes, duplicate command + effects, stale subscription generations, and unsupported authority; +12. inspect the browser bundle and fail if it contains Node builtin imports. + +The gate runs on Windows and Linux. A screenshot is useful presentation +evidence, but the receipt/state chronology is the authority proof. diff --git a/src/GameCult.Mesh/docs/content-sessions.md b/src/GameCult.Mesh/docs/content-sessions.md index 1eae6b6c..313a21cf 100644 --- a/src/GameCult.Mesh/docs/content-sessions.md +++ b/src/GameCult.Mesh/docs/content-sessions.md @@ -15,9 +15,12 @@ The consumer creates a provider over its existing identity-first session manager, then gives that provider to the transfer owner: ```csharp +var target = new CultMeshSessionTarget( + "aetheria.public", + "service:aetheria.daemon"); var provider = mesh.ContentProvider( "aetheria.daemon", - "service:aetheria.daemon"); + target); var transfer = new CultMeshContentTransferService( transferStateCache, diff --git a/src/GameCult.Mesh/docs/cross-runtime-primitives-roadmap.md b/src/GameCult.Mesh/docs/cross-runtime-primitives-roadmap.md index 53db8c7b..93fb3f3c 100644 --- a/src/GameCult.Mesh/docs/cross-runtime-primitives-roadmap.md +++ b/src/GameCult.Mesh/docs/cross-runtime-primitives-roadmap.md @@ -121,7 +121,7 @@ var viewport = await aetheria.Zone(zoneId) await aetheria.Entity(ravenId) .Pilot() - .MoveAsync(new CultMath.float2(1f, 0f), claim: "pilot-control"); + .MoveAsync(CultVec2.Right, claim: "pilot-control"); ``` Current seed: @@ -134,7 +134,7 @@ var verse = await CultMesh.ConnectVerseAsync( new[] { new CultMeshAuthorityClaim("pilot-control", shardId: "zone:raven") }); var aetheria = verse.Use(context => new AetheriaGeneratedFacade(context)); -await aetheria.Entity(ravenId).Pilot.MoveAsync(new CultMath.float2(1f, 0f)); +await aetheria.Entity(ravenId).Pilot.MoveAsync(CultVec2.Right); ``` ```ts diff --git a/src/GameCult.Mesh/docs/document-write-authority.md b/src/GameCult.Mesh/docs/document-write-authority.md new file mode 100644 index 00000000..0d8d8a68 --- /dev/null +++ b/src/GameCult.Mesh/docs/document-write-authority.md @@ -0,0 +1,86 @@ +# CultMesh Document Write Authority + +CultMesh document reads, authoritative replacements, and client predictions are +different capabilities. A generic setter cannot choose between them without +hiding an authority decision from the caller. + +## Authority map + +Owner: + +- `CultMeshDocumentHandle` owns typed snapshot and watch access; +- an authoritative writer owns canonical replacement capability; +- a prediction writer owns candidate/prediction submission capability; +- `CultMeshReactiveDocument` owns only a local editable mirror and its + coalescing schedule. + +Inputs: + +- a bound typed live feed; +- an optional authoritative replacement delegate; +- an optional prediction delegate; +- explicit local mutations made through the reactive document API. + +Outputs: + +- coherent typed snapshots and watches; +- authoritative replacements only through an explicitly requested + authoritative writer; +- predictions only through an explicitly requested prediction writer; +- one coalesced write for each explicit burst of local mutations. + +Derived state: + +- the reactive snapshot is a local mirror, not canonical truth; +- reconciliation metadata compares a pending local write with an incoming + canonical snapshot; +- debounce timers schedule already-declared work and never inspect documents + for undeclared mutations. + +Forbidden writers: + +- `SetAsync` may not select prediction or replacement based on whichever + delegate happens to exist; +- reading or watching a document may not start serialization polling; +- mutating an exposed object graph may not silently become a network write; +- alias conversion may not erase which writer was selected. + +Shared paths: + +- direct edits, nested edits, automation, and UI bindings use `Update` or + `ReplaceLocal` on the same reactive mirror; +- explicit flush and debounced flush use the same selected writer; +- canonical watch updates and refresh use the same snapshot adoption path. + +Cut line: + +- remove `CanSet`, `SetAsync`, and ambiguous `UpdateAsync`/`ReactiveAsync` + entry points from document handles and catalogs; +- obtain either an authoritative writer or a prediction writer before editing; +- remove change-detection timers and full-document idle serialization; +- expose cloned snapshots so callers cannot mutate the live mirror behind the + mutation boundary. + +## Required proof + +- a handle carrying both capabilities cannot write until the caller selects a + writer; +- authoritative and prediction writers reach only their named delegates; +- nested edits made through `Update` are coalesced and published; +- direct mutation of a returned snapshot cannot publish; +- an idle reactive document creates no periodic detection work; +- scheduling 1, 100, or 1,000 reactive documents creates zero idle work, and + editing one percent schedules only that one percent; +- canonical reconciliation and in-flight edits remain deterministic. + +Run the cross-runtime scheduling gate from a CultLib checkout: + +```powershell +powershell -ExecutionPolicy Bypass -File scripts/verify-reactive-document-scaling.ps1 +``` + +The gate runs the C#, TypeScript, and Python implementations against the same +1/100/1,000-document cases. It proves ownership and scheduling proportionality; +it does not claim allocation, payload-size, or end-to-end latency budgets. Those +need a separate measured benchmark against representative document bodies and +real transports. diff --git a/src/GameCult.Mesh/docs/durable-node-quickstart.md b/src/GameCult.Mesh/docs/durable-node-quickstart.md index 3d08ae0f..f85fac97 100644 --- a/src/GameCult.Mesh/docs/durable-node-quickstart.md +++ b/src/GameCult.Mesh/docs/durable-node-quickstart.md @@ -55,7 +55,7 @@ var key = new CultRecordKey("note:intro"); var documents = new CultNetDocumentRegistry() .Register(CultNetDocumentBinding.ForDocument()); -using (var node = await CultMesh.CreateNodeAsync("world.ccmp", new CultMeshNodeOptions +using (var node = await CultMesh.CreateNodeAsync("world.cc", new CultMeshNodeOptions { DatabaseOptions = new CultNetDatabaseOptions { @@ -76,7 +76,7 @@ using (var node = await CultMesh.CreateNodeAsync("world.ccmp", new CultMeshNodeO await node.FlushAsync(); } -using (var reopened = await CultMesh.CreateNodeAsync("world.ccmp", new CultMeshNodeOptions +using (var reopened = await CultMesh.CreateNodeAsync("world.cc", new CultMeshNodeOptions { DatabaseOptions = new CultNetDatabaseOptions { @@ -92,7 +92,7 @@ using (var reopened = await CultMesh.CreateNodeAsync("world.ccmp", new CultMeshN What happened: -1. `CultMesh.CreateNodeAsync("world.ccmp", ...)` opened a durable CultCache +1. `CultMesh.CreateNodeAsync("world.cc", ...)` opened a durable CultCache snapshot file and wrapped it in a CultMesh node. 2. `node.Database.PutAsync(...)` wrote a typed document through the public CultMesh database surface. @@ -105,7 +105,7 @@ What happened: If you peel this path apart, the authority split stays simple: -- CultCache: `world.ccmp`, schema ids, record keys, typed document payloads, +- CultCache: `world.cc`, schema ids, record keys, typed document payloads, local indexes, flush/load - CultNet: `CultNetDocumentRegistry`, raw document puts/snapshots, subscriptions, shard routing, transport-facing replication diff --git a/src/GameCult.Mesh/docs/getting-started/01-persist-typed-state.md b/src/GameCult.Mesh/docs/getting-started/01-persist-typed-state.md index b2fce2a0..f7924ec8 100644 --- a/src/GameCult.Mesh/docs/getting-started/01-persist-typed-state.md +++ b/src/GameCult.Mesh/docs/getting-started/01-persist-typed-state.md @@ -1,22 +1,68 @@ -# 1. Persist Typed Application State +# 1. Persist The Counter -Start with a typed CultCache document and a durable CultMesh node. The complete -sample is [the durable node quickstart](../durable-node-quickstart.md). +The tutorial builds one application all the way through: a provider-owned +counter, an Increment operation, one browser lowerer, and one C# headless +client. Start with the state the provider actually owns. -At the end of that guide you should have: +```csharp +using GameCult.Caching; +using GameCult.Caching.MessagePack; +using GameCult.Mesh; +using GameCult.Networking; +using MessagePack; -- a document type with a stable schema id; -- a `CultMeshNode` whose database owns the live local document; -- a `.cc`/CultCache-backed durable state path; -- a restart proof that reads the same typed document. +[CultDocument("sample.counter_state", "sample.counter_state.v1")] +[MessagePackObject] +public sealed class CounterState +{ + [Key(0)] public string CounterId { get; set; } = ""; + [Key(1)] public int Count { get; set; } +} +``` -Do not create an Eve-shaped copy of this state. The Eve surface published in -chapter 3 points at typed records and commands owned by the provider. +Open a `.cc` node, register the document once, and write through its database +boundary: -Verification: +```csharp +var key = new CultRecordKey("counter:main"); +var documentTypes = new[] { typeof(CounterState) }; +var registry = CultMesh.CreateCultCacheDocumentRegistry(documentTypes); +var documents = CultMesh.CreateCultNetDocumentRegistry(documentTypes, registry); +using var node = await CultMesh.CreateNodeAsync("counter.cc", new CultMeshNodeOptions +{ + StartServer = false, + CacheOptions = new CultCacheOpenOptions { Registry = registry }, + DatabaseOptions = new CultNetDatabaseOptions + { + RuntimeId = "sample.counter-daemon", + DocumentRegistry = documents + } +}); + +await node.Database.PutAsync(key, new CounterState +{ + CounterId = key.Value, + Count = 0 +}); +await node.FlushAsync(); +``` + +CultCache owns durable local truth. CultNet owns the portable schema mapping. +CultMesh exposes the database and later carries subscriptions and operations. +The renderer owns none of them. + +The executable provider adds its receipt index to this document so the counter +mutation and idempotency decision are committed together. See +[`samples/eve-browser-network/Program.cs`](../../../../samples/eve-browser-network/Program.cs) +for the exact running type and transaction. + +Fast persistence check: ```powershell dotnet test tests/GameCult.Networking.Tests/GameCult.Networking.Tests.csproj --filter FullyQualifiedName~CultMeshNode_CanRoundTrip_DurableTypedDocument_Through_PublicDatabaseSurface ``` +This check proves the local typed state boundary. Chapter 4 boots the real +provider and proves the same state across Chromium and C# after restart. + Next: [connect a client by stable identity](02-connect-by-identity.md). diff --git a/src/GameCult.Mesh/docs/getting-started/02-connect-by-identity.md b/src/GameCult.Mesh/docs/getting-started/02-connect-by-identity.md index 9301b49f..47905fec 100644 --- a/src/GameCult.Mesh/docs/getting-started/02-connect-by-identity.md +++ b/src/GameCult.Mesh/docs/getting-started/02-connect-by-identity.md @@ -1,26 +1,75 @@ # 2. Connect A Client By Stable Identity -Clients address a provider or Verse by stable identity. Discovery owns the -current physical candidates and the session manager owns connection reuse, -path rotation, and transport failure state. +Clients address a live session by stable Verse/provider identity. In this tutorial, +`sample.counter` is the Verse id and `sample.counter-daemon` is the authority +runtime Odin currently advertises for it. Application clients pass that typed +pair, never a socket address and never one ambiguous string that might mean +either identity. A product may expose only a Verse choice to its user; its +selection owner then resolves the provider runtime from Odin's descriptor before +opening the session. Discovery owns the current physical routes, and the session +manager owns connection reuse, path rotation, and transport failure state. +Each Odin route binds that authority runtime to one endpoint, protocol set, and +generation. For a remote Verse, Odin signs that exact tuple and the provider's +P-256 public key. The client pins an Odin root from its own configuration, then +requires the connected provider to sign a fresh client nonce. A route cannot +become usable by merely repeating the expected Verse and runtime strings. ```csharp using GameCult.Mesh; +using GameCult.Networking.WebSockets; +var odinEndpoint = Environment.GetEnvironmentVariable("ODIN_ENDPOINT") + ?? throw new InvalidOperationException("Set ODIN_ENDPOINT to the configured rendezvous route."); +const string verseId = "sample.counter"; +var target = new CultMeshSessionTarget(verseId, "sample.counter-daemon"); +using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(30)); +var cancellationToken = timeout.Token; +var odinRoot = new CultMeshEcdsaP256PublicKey( + Environment.GetEnvironmentVariable("ODIN_ROOT_KEY_ID") + ?? throw new InvalidOperationException("Set ODIN_ROOT_KEY_ID."), + Environment.GetEnvironmentVariable("ODIN_ROOT_P256_X") + ?? throw new InvalidOperationException("Set ODIN_ROOT_P256_X."), + Environment.GetEnvironmentVariable("ODIN_ROOT_P256_Y") + ?? throw new InvalidOperationException("Set ODIN_ROOT_P256_Y.")); using var mesh = new CultMeshClient(new CultMeshClientOptions { - RendezvousEndpoints = new[] { "rudp://odin.gamecult.net:3076" } + RendezvousEndpoints = new[] { odinEndpoint }, + Discovery = new CultMeshVerseDiscoveryClientOptions + { + CreateClientForEndpoint = _ => new CultNetWebSocketSchemaClient() + }, + Sessions = new CultMeshSessionManagerOptions + { + Trust = new CultMeshAuthorityTrustPolicy( + CultMeshAuthorityTrustMode.AuthenticatedRemote, + new[] { odinRoot }) + }, + Connectors = new ICultMeshTransportConnector[] + { + new CultMeshUriSchemaTransportConnector( + "cultnet-websocket", + new[] { "wss" }, + _ => new CultNetWebSocketSchemaClient()) + } }); -var pilotSurface = await mesh.DocumentAsync( - "aetheria.daemon", - "eve:surface:aetheria.pilot", +using var counterLease = await mesh.LeaseDocumentAsync( + target, + "counter:main", cancellationToken); +var counter = counterLease.Handle; -using var surfaceWatch = pilotSurface.Watch().Subscribe(Render); +using var counterWatch = counter.Watch(value => + Console.WriteLine($"Count: {value.Count}")); ``` -Keep `mesh` for the application lifetime. The document handle is stable: when +`CounterState` is the application document from chapter 1. Eve documents enter +in chapter 3, where the sample installs the renderer-neutral Eve contract +package explicitly. This identity/session API does not depend on a UI runtime. + +Keep `mesh` for the application lifetime and dispose each document lease when +the consuming screen or system closes. Leases for the same identity share one +subscription. The document handle is stable while leased: when Odin advertises a new physical route after a partition, CultMesh migrates the session, restores the server-side subscription, refreshes the typed snapshot, and continues the same watch. @@ -30,7 +79,7 @@ protocol-level state or diagnostics: ```csharp var session = await mesh.ConnectAsync( - "aetheria.daemon", + target, CultMeshProtocols.Documents, cancellationToken); @@ -41,14 +90,62 @@ using var stateWatch = session.WatchState().Subscribe(state => Application code does not reconnect, sleep, rank endpoints, or replace the session. Those decisions belong to CultMesh. -Verification: +For a deliberately local, moddable Verse, select the exception explicitly and +use only in-process or loopback routes: + +```csharp +Sessions = new CultMeshSessionManagerOptions +{ + Trust = new CultMeshAuthorityTrustPolicy(CultMeshAuthorityTrustMode.LocalDevelopment) +} +``` + +`LocalDevelopment` never accepts an unsigned non-loopback route. Verse names, +provider names, and Odin wire fields cannot turn this exception on. Remote +routes require a trusted Odin signature, provider nonce proof, certificate +validity, and a protected channel. + +The remote example requires both `GameCult.Mesh` and +`GameCult.Networking.WebSockets` at the same CultLib package version. Core +CultMesh does not silently install a network stack: the consumer chooses the +secure connector and its TLS policy. The default TCP connector exists for the +explicit loopback-development case and cannot satisfy authenticated-remote +trust. + +An authenticated provider receives an Odin-certified route and its matching +provider private key from deployment configuration. It attaches the proof gate +to the same WSS schema server before publishing the route: + +```csharp +using var identity = new CultMeshSessionIdentityServer( + schemaServer, + authorityRuntimeId: "sample.counter-daemon", + verseIds: new[] { "sample.counter" }, + protocolIds: new[] { CultMeshProtocols.Documents.Value }, + routeGenerations: new[] { certifiedRoute.Generation }, + proofSigners: new[] { new CultMeshSessionProofSigner(certifiedRoute, providerPrivateKey) }); +``` + +`certifiedRoute` is the exact signed route Odin publishes. The provider never +holds Odin's signing key, and an unsigned WSS route is still untrusted. + +Fast session-management check (this uses controlled transports; it is not the +network proof): ```powershell dotnet test tests/GameCult.Mesh.Tests/GameCult.Mesh.Tests.csproj --filter FullyQualifiedName~CultMeshSessionManagerTests ``` +The local checkpoint in chapter 4 additionally advertises a better-priority +endpoint for the wrong authority runtime. Both the C# and browser clients must +ignore it, verify the intended peer, survive route replacement, and reject +wrong-source operation responses. + Advanced hosts may inject lookup sources, connectors, clocks, persistence, and diagnostics through the lower-level discovery and session APIs. Application features should not need them. +Chapter 4 runs the real network chronology and proves that the retained lease +survives Odin moving `sample.counter` to a new physical provider route. + Next: [publish one Eve surface to multiple runtimes](03-publish-an-eve-surface.md). diff --git a/src/GameCult.Mesh/docs/getting-started/03-publish-an-eve-surface.md b/src/GameCult.Mesh/docs/getting-started/03-publish-an-eve-surface.md index 666acf1d..5cb5b742 100644 --- a/src/GameCult.Mesh/docs/getting-started/03-publish-an-eve-surface.md +++ b/src/GameCult.Mesh/docs/getting-started/03-publish-an-eve-surface.md @@ -29,21 +29,114 @@ CultMesh document session. Other runtime lowerers must preserve the same boundary: they may reject or report unsupported capabilities, but they may not import provider internals. +## The Application-Owned Part + +Install the renderer-neutral `GameCult.Eve.Surface` package in the provider. +Do not install EveUnity or the browser lowerer there. A provider defines typed +state, one typed operation transaction, and one Eve surface. CultNet envelope +encoding is framework work. This complete method builds the surface document; +it has no hidden transport or state variables: + +```csharp +using GameCult.Eve.Surface; +using GameCult.Mesh; + +static EveSurfaceDocument CreateCounterSurface() +{ + var route = new CultMeshRouteHint(CultMeshLocalityKind.Network, "cultmesh"); + var count = new CultMeshStateBindingDescriptor( + "value", + "sample.counter.count", + "sample.counter_state:counter:main", + "sample.counter_state.v1", + route); + var increment = CultMesh.OperationBinding( + "sample.counter.increment", + "Increment", + "sample.increment.v1", + route); + + return EveSurface.Create("sample.counter") + .Provider("sample.counter-ui", "sample.daemon") + .Title("CultMesh browser counter") + .RootColumn("counter.root", root => root + .Metric("counter.value", "Canonical count", "0", count) + .Button("counter.increment", "Increment", increment)) + .Build(); +} +``` + +The operation handler owns validation, idempotency, state mutation, and its +durable receipt. `CultNetOperationServer` owns only route/schema checks and +typed envelope serialization. The complete, compiling provider transaction—including +the schema server, typed request/receipt types, and exactly-once mutation—is in +[`samples/eve-browser-network/Program.cs`](../../../../samples/eve-browser-network/Program.cs); +it contains no hand-written operation base64 or MessagePack dispatch switch. + ## Multi-Runtime Checkpoint Run the same provider and connect at least two lowerers. Both must report the same provider id, surface id, surface version, and command receipt ids. Native pixels may differ; provider truth may not. -Current verification for the Unity consumer: +The complete checkpoint is executable from CultLib: ```powershell -cd E:\Projects\EveUnity -powershell -ExecutionPolicy Bypass -File .\scripts\run-release-consumer-tests.ps1 +cd CultLib +pwsh -File ./scripts/verify-eve-getting-started.ps1 ``` -The browser/Electron counterpart and a single command-driven two-runtime sample -remain required before this chapter can be considered complete. Their absence -is tracked as developer-experience debt rather than papered over with bespoke -setup instructions. +This one command runs clean .NET and TypeScript package-artifact checkpoints, +then the real browser/C# network checkpoint. They remain separate programs +because they prove different boundaries; the developer should not have to +orchestrate them. + +The .NET layer packs CultLib's managed dependency graph and +`GameCult.Eve.Surface`, then restores and runs an empty project whose only +GameCult dependency is `PackageReference Include="GameCult.Eve.Surface"`. +This proves the renderer-neutral provider contract does not require a sibling +Eve or CultLib source tree after packaging. + +The real-process C# sample is then rebuilt from that same temporary NuGet feed. +The source project references in its `.csproj` are contributor convenience only +and are disabled by the checkpoint; they are not the consumer contract. + +The first layer builds and packs `cultcache-ts`, `cultnet-ts`, `cultmesh-ts`, +`@gamecult/eve-contracts`, and `@gamecult/eve-browser-lowering`, installs the +tarballs into an empty temporary consumer, and runs +`samples/eve-two-runtime/sample.mjs`. Eve's generated runtime validators reject +an invalid surface or receipt before either enters the store. One provider-owned +counter surface is lowered into a jsdom DOM host and observed by an independent +headless CultMesh observer. A DOM command crosses the typed operation boundary, +both consumers converge on the same state and canonical receipt identity, a +duplicate idempotency key is not applied twice, and the state plus receipt +survive reopening the `.cc` store. + +The second layer runs a durable C# provider, a wrong-authority decoy, a separate +local Odin fixture, a real Chromium Eve lowerer, and an independent C# headless observer over binary +CultNet WebSocket lanes. The open browser resolves the `sample.counter` Verse +through the canonical catalog, rejects the decoy route, survives its provider +restarting on a different route, and resubscribes. The retained C# client then invokes a second typed operation +through `CultMeshClient.InvokeAsync`; Chromium and C# observe the same +provider-authored state and receipt chronology. The same retained C# session +then completes 10,000 typed no-op operations under explicit p99 and post-GC +managed-memory gates, so the tutorial pressure-tests the API it teaches. + +Current verification for the Unity consumer remains: + +```powershell +cd ../EveUnity +pwsh -File ./scripts/run-release-consumer-tests.ps1 +``` + +The artifact layer proves clean package consumption, canonical Eve contract +validation, and two independent lowering/observation consumers inside Node. It +does **not** prove the network layer; the second program owns that evidence. + +The network program is +[`samples/eve-browser-network`](../../../../samples/eve-browser-network/README.md). +Run it separately only while diagnosing that layer. A smoke against the +deployed Odin daemon remains a separate infrastructure gate. Its owner map +and full acceptance contract are recorded in +[`browser-verse-transport.md`](../browser-verse-transport.md). diff --git a/src/GameCult.Mesh/docs/getting-started/04-keep-a-typescript-provider-alive.md b/src/GameCult.Mesh/docs/getting-started/04-keep-a-typescript-provider-alive.md deleted file mode 100644 index 15f10a3a..00000000 --- a/src/GameCult.Mesh/docs/getting-started/04-keep-a-typescript-provider-alive.md +++ /dev/null @@ -1,122 +0,0 @@ -# 4. Keep A TypeScript Provider Alive - -A provider process should declare desired typed publications and domain command -handlers once. It should not rebuild discovery registration, lease renewal, -reconnect, republish, receipt delivery, and withdrawal loops in a renderer or -scheduled export script. - -`CultMeshProviderSession` owns that lifecycle. The application still owns its -state and command transaction. A transport connector owns the physical CultNet -route. Eve advertisements and surfaces are ordinary typed publications; the -session does not import Eve or learn provider gameplay. - -```ts -import { - CultMeshProviderSession, - type CultMeshProviderReceiptStore, - type CultMeshProviderTransport, -} from "cultmesh-ts"; - -export async function runProvider( - transport: CultMeshProviderTransport, - receipts: CultMeshProviderReceiptStore, -) { - const provider = new CultMeshProviderSession({ - identity: { - providerId: "voidbot.swarm", - serviceInstanceId: "voidbot-worker-7", - endpointId: "odin:voidbot-worker-7", - verseId: "voidbot.local", - }, - transport, - receiptStore: receipts, - publications: [ - { - publicationId: "voidbot.swarm.advertisement", - documentType: "gamecult.eve.provider_advertisement", - schemaId: "gamecult.eve.provider_advertisement.v1", - recordKey: "voidbot.swarm", - value: providerAdvertisement, - }, - { - publicationId: "voidbot.swarm.surface", - documentType: "gamecult.eve.surface_state", - schemaId: "gamecult.eve.surface_state.v1", - recordKey: "voidbot.swarm", - value: currentSurface, - }, - ], - commandHandlers: { - "swarm.set_heat": async command => - applyHeatCommandTransaction(command.commandId, command.payload), - }, - }); - - await provider.start(); - - await provider.upsertPublication({ - publicationId: "voidbot.swarm.surface", - documentType: "gamecult.eve.surface_state", - schemaId: "gamecult.eve.surface_state.v1", - recordKey: "voidbot.swarm", - value: nextSurface, - }); - - process.once("SIGTERM", () => { - provider.stop().catch(error => { - console.error("CultMesh provider shutdown completed with cleanup failures.", error); - process.exitCode = 1; - }); - }); -} -``` - -The four identities are deliberately distinct: - -- `providerId` names the product capability clients select; -- `serviceInstanceId` names the running owner incarnation; -- `endpointId` names the stable discovery/session route. -- `verseId` names the discovery and state-sharing domain. - -Do not replace any of them with a process id, socket address, body producer id, -or renderer id. - -## Receipt Rule - -The receipt store is required and must be durable in a daemon. The in-memory -store exported by the package is for tests and disposable tools only. The -domain command handler must apply its mutation idempotently using `commandId`; -the provider session deduplicates delivery and republishes the stored typed -receipt, but it cannot make an unrelated database mutation atomic by wishing -at it harder. `commandId` is the idempotency key: later delivery of the same -provider, service-instance, and command kind returns the original receipt even -if a sender incorrectly changes the payload. Reusing the ID for another kind -or identity is rejected as a conflict. - -The store is also the receipt outbox. `put` commits the receipt before any wire -publication. `listPending` supplies receipts after startup or reconnect, and -`markPublished` removes one only after the active connection accepts it. Scope -the store to the provider and service-instance identity. If the store is -unavailable, the session enters `degraded` and stops accepting commands until a -renewal can drain the outbox again; a store failure is not a transport failure. - -## Verification Checkpoint - -Use an injected scheduler and transport fault double to prove the actual -lifecycle: - -1. initial registration publishes every desired document in stable order; -2. lease renewal occurs before expiry; -3. a failed route produces `reconnecting`, then republishes current desired - state on the replacement connection; -4. duplicate command delivery runs one handler transaction and republishes the - same receipt; -5. failed receipt transport reconnects and replays the durable outbox exactly - until `markPublished` succeeds; -6. shutdown drains command persistence, withdraws the lease and remaining - publications, and closes even when cleanup reports failures; -7. stopping during registration cannot let the stale connection publish; -8. observers and conflicting command IDs cannot become lifecycle authorities. - -The package suite contains this timeline in -`packages/cultmesh-ts/test/provider-session.test.ts`. diff --git a/src/GameCult.Mesh/docs/getting-started/04-run-the-multiruntime-game.md b/src/GameCult.Mesh/docs/getting-started/04-run-the-multiruntime-game.md new file mode 100644 index 00000000..ce18fdb3 --- /dev/null +++ b/src/GameCult.Mesh/docs/getting-started/04-run-the-multiruntime-game.md @@ -0,0 +1,67 @@ +# 4. Run The Multi-Runtime Game + +The counter is now one small multiplayer game: + +- a C# provider owns `CounterState`, Increment, idempotency, receipts, the + `.cc` store, and the Eve surface; +- Chromium discovers that provider through Odin, lowers the surface, and + invokes Increment; +- an independent C# client leases the same counter and invokes through the same + typed operation boundary; +- both clients remain attached when Odin moves the Verse to a replacement + physical provider route. +- a better-priority route bound to another runtime remains unreachable to both + clients throughout the chronology. + +Run the whole chronology from the CultLib checkout: + +```powershell +pwsh -File ./scripts/verify-eve-getting-started.ps1 +``` + +The command owns setup and teardown. Do not start Odin, the provider, or the +browser by hand. A successful run prints two JSON records followed by: + +```text +Eve getting-started verification passed: clean .NET and TypeScript artifacts, DOM and headless convergence, real Chromium and C# clients, Odin discovery, receipts, persistence, and route replacement. +``` + +## What The Command Proves + +The first checkpoint packs the CultLib managed dependency closure and Eve's C# +surface contract, then restores and runs an empty .NET project using only the +resulting `GameCult.Eve.Surface` package. The second checkpoint packs the +TypeScript CultCache, CultNet, CultMesh, and Eve +packages, installs those artifacts into an empty temporary consumer, and proves +DOM/headless convergence plus `.cc` reopen. This rejects a sample that only +works through repository-relative source imports. + +The third checkpoint boots separate Odin, intended provider, wrong-authority +decoy provider, Chromium, and C# headless processes. Its C# application is built +from the temporary NuGet feed produced by the first checkpoint, not sibling +project references. Chromium performs the first increment. The provider then restarts on +a different route, Odin advertises the replacement, and the retained clients +resubscribe without application reconnect code. C# performs the second +increment. Both clients must observe count 2 and both canonical receipt ids. + +Finally, the retained C# session sends 10,000 typed no-op operations. The gate +records throughput, p99 latency, managed-heap growth, and private-memory growth. +It fails at 250 ms p99 or more than 8 MiB post-GC managed growth. + +## What It Does Not Prove + +The included Odin is a real CultNet Verse-catalog provider but still a local +fixture. It does not prove deployed GameCult infrastructure. The package layer +uses packed checkout artifacts until the package owners publish the named npm +packages. Those are explicit release gates, not reasons to put endpoint or +source-tree fallbacks into application code. + +The process harness lives in +[`samples/eve-browser-network`](../../../../samples/eve-browser-network/README.md). +Application code should copy the typed state, operation, and Eve composition +shape—not its benchmark flags or fixture orchestration. + +For a TypeScript provider daemon, continue with the +[`CultMeshProviderSession` guide](../../../../packages/cultmesh-ts/README.md#long-lived-providers). +That lifecycle is an alternative provider runtime for the same ownership +model, not a fifth chapter that changes this tutorial into a different product. diff --git a/src/GameCult.Mesh/docs/getting-started/README.md b/src/GameCult.Mesh/docs/getting-started/README.md index 7914247c..66ded7a4 100644 --- a/src/GameCult.Mesh/docs/getting-started/README.md +++ b/src/GameCult.Mesh/docs/getting-started/README.md @@ -4,12 +4,43 @@ This series builds one typed application state and presents it through multiple Eve runtimes. It is maintained as an API acceptance test: examples use public APIs, name the owner of each decision, and include a verification checkpoint. +Application consumers install released packages and do not need the source +repositories. This checkout also contains a maintainer checkpoint that creates +the same clean consumers from local package artifacts: + +```powershell +pwsh -File ./scripts/verify-eve-getting-started.ps1 +``` + +The maintainer command takes an explicit `-EveRoot` when Eve is not a sibling: + +```powershell +pwsh -File ./scripts/verify-eve-getting-started.ps1 -EveRoot D:\src\Eve +``` + +It installs missing build dependencies and runs three explicit evidence layers. +First it packs the +CultLib managed dependency closure plus `GameCult.Eve.Surface`, then restores +and runs an empty .NET consumer using only `PackageReference`. Next it packs the five +TypeScript packages into another empty temporary consumer, validates the Eve surface +and receipt, executes one idempotent counter command, proves a DOM lowerer and +headless observer converge, and reopens the durable `.cc` store. Then it boots a +durable C# provider, local Odin fixture, real Chromium lowerer, and independent +C# client. Chromium invokes before provider replacement; C# invokes afterward. +Both retained clients prove discovery by identity, canonical receipts, provider +restart on another route, rejection of a better-priority wrong-authority route, +resubscription, and durable convergence. The retained +C# session also completes 10,000 non-persistent typed operations under p99 and +post-GC managed-memory gates. +Set `CHROME_PATH` only when no installed Chrome, Chromium, Edge, or Playwright +Chromium can be found automatically. + Follow the chapters in order: 1. [Persist typed application state](01-persist-typed-state.md) 2. [Connect a client by stable identity](02-connect-by-identity.md) 3. [Publish one Eve surface to multiple runtimes](03-publish-an-eve-surface.md) -4. [Keep a TypeScript provider alive](04-keep-a-typescript-provider-alive.md) +4. [Run the multi-runtime game](04-run-the-multiruntime-game.md) The intended application shape is small: @@ -19,6 +50,8 @@ provider daemon -> typed CultCache documents -> CultMesh advertisement +------------------------+--------------------+ | | | Eve browser EveUnity another lowerer + | | | + player input player input headless / AI ``` The provider owns live truth. Eve owns the surface contract. Each runtime owns @@ -34,3 +67,6 @@ native projection. CultMesh owns discovery and connection continuity. pull request. - A setup step that depends on repository internals is an API defect, not acceptable tutorial folklore. +- The maintainer command may read owner checkouts to create packages. Every + sample process it launches must consume those package artifacts rather than + compiling through sibling project references. diff --git a/src/GameCult.Mesh/docs/public-api.md b/src/GameCult.Mesh/docs/public-api.md index 235cf2ba..df1735b6 100644 --- a/src/GameCult.Mesh/docs/public-api.md +++ b/src/GameCult.Mesh/docs/public-api.md @@ -50,8 +50,12 @@ not the brand surface. ### Verses `CultMeshVerseDescriptor` describes a rule-bearing consensus graph. It includes -transport compatibility, rules hash, authority model, discovery endpoints, -known authority runtimes, optional parent Verse id, and plugin requirements. +transport compatibility, rules hash, authority model, exact authority-route +bindings, optional parent Verse id, and plugin requirements. Each route binds +an authority runtime to one endpoint, protocol set, priority, and generation. +The old endpoint/runtime arrays are derived projections; a legacy descriptor +with several authority runtimes is rejected because it cannot identify which +runtime owns which route. `CultMeshVerseCatalog` is the local reactive catalog for discovered Verses. It can publish discovery updates and find compatible transfer targets. @@ -208,6 +212,16 @@ portable machinery that makes it feel native. slices, or browser typed arrays. `CultMesh.DescribeNativeSliceView(...)` exposes the full view id, schema id, row count, columns, route, native handle, and dense stride for tools and adapters. +- `CultMeshFrameBodyPublisher` and `CultMeshMappedFrameBodyCursor` carry + fixed-layout hot bodies through a triple-buffered memory mapping. Producers + reserve a writable slot and commit in place; consumers retain one bootstrap + capability and acquire only newer read leases. Windows uses a named memory + map; Unix uses a private file-backed map because .NET does not support named + maps there. The capability and cursor API is identical, and publisher + disposal revokes new opens and removes the Unix backing file. `Stats()` reports committed + frames, blocked writes, and unavoidable copies. Direct slot writes report + zero copies; the convenience `TryPublish(ReadOnlySpan)` path reports + one explicit fallback copy instead of laundering it as zero-copy transport. - `CultMeshRouteHint`, `CultMeshRouteRecord`, and `CultMeshLocalityKind` name locality choices without forcing application code to choose transport-specific APIs. `CultMesh.RouteRecord(...)` is the shared way to flatten and rehydrate @@ -393,8 +407,8 @@ var view = new CultMeshNativeSliceViewDescriptor( rowCount, new[] { - CultMeshNativeSliceColumn.For("position"), - CultMeshNativeSliceColumn.For("velocity") + CultMeshNativeSliceColumn.For("position"), + CultMeshNativeSliceColumn.For("velocity") }, new CultMeshRouteHint(CultMeshLocalityKind.SharedMemory)); ``` diff --git a/src/GameCult.Mesh/docs/reliability-migration-plan.md b/src/GameCult.Mesh/docs/reliability-migration-plan.md index 658d5e0a..47e1b190 100644 --- a/src/GameCult.Mesh/docs/reliability-migration-plan.md +++ b/src/GameCult.Mesh/docs/reliability-migration-plan.md @@ -101,8 +101,9 @@ state for inspection and compatibility. **Owner:** `CultMeshSessionManager`. -**Inputs:** stable endpoint identity, application protocol identity, discovery -candidate stream, transport connectors, path policy, clock, and retry policy. +**Inputs:** stable Verse/provider identity, application protocol identity, +discovery candidate stream, transport connectors, path policy, clock, and retry +policy. **Outputs:** a reusable `CultMeshSession` with typed connection state, chosen path, protocol channel, diagnostics, and watches for state/path changes. @@ -249,7 +250,7 @@ The target API is identity-first and watchable: ```csharp await using var session = await mesh.ConnectAsync( - CultMeshEndpointId.Parse("odin:yggdrasil"), + new CultMeshSessionTarget("yggdrasil.public", "odin:yggdrasil"), CultMeshProtocols.Documents, cancellationToken); diff --git a/src/GameCult.Mesh/docs/transport-planes.md b/src/GameCult.Mesh/docs/transport-planes.md index 314c55b4..fea2b69b 100644 --- a/src/GameCult.Mesh/docs/transport-planes.md +++ b/src/GameCult.Mesh/docs/transport-planes.md @@ -9,9 +9,10 @@ be equally good at files, realtime state, and same-machine memory access. registered connector for each traffic class. A connector owns only its physical connection and byte movement. -**Inputs:** stable endpoint identity, advertised physical candidates, connector -support and priority, caller cancellation, and locality evidence for body -negotiation. +**Inputs:** stable Verse/authority-runtime identity, exact Odin-bound physical +routes with protocol and generation, +connector support and priority, caller cancellation, and locality evidence for +body negotiation. **Outputs:** @@ -63,13 +64,29 @@ operation. transport diagnostics. None of these grant provider authority or become content identity. +Before a schema, content, or realtime session becomes online, consumer-owned +trust verifies an Odin-signed route certificate binding Verse, authority +runtime, protocol set, endpoint, generation, validity interval, and provider +P-256 key. Schema and content lanes then require a fresh client nonce signed by +that provider key. QUIC uses the signed endpoint plus its TLS +endpoint/certificate proof. A connector that returns bytes without this chain +is rejected; racing every endpoint in a Verse descriptor is not route +selection. Unsigned traffic exists only under an explicitly selected +`LocalDevelopment` policy and only for in-process or loopback routes. + +Route and session signatures use ECDSA P-256 with SHA-256 and fixed-width IEEE +P1363 `r || s` encoding, matching browser WebCrypto. Their canonical +transcripts are ordered UTF-8 fields prefixed by four-byte big-endian lengths; +serializer bytes are never signature authority. + **Forbidden writers:** applications and renderers do not choose unadvertised endpoints, implement reconnect loops, or bypass verification. Schema messages must not carry bulk bodies. RUDP/LiteNetLib cannot become an implicit default for any traffic class. QUIC does not own schemas or immutable files merely because it can transport bytes. -**Shared paths:** discovery and identity resolution remain common. Each traffic +**Shared paths:** exact route resolution and peer verification remain common. +Each traffic class has its own connector set, connection cache, priority tiers, and failure state. All content connectors feed the same transfer and promotion owner. @@ -84,7 +101,8 @@ new seam does not pretend that migration has already occurred. ## Security and release gate The current TCP control and content connectors are plaintext and therefore -restricted to local or otherwise trusted deployment. Adding TLS must not move +restricted to explicit loopback development. They cannot carry an authenticated +remote session. Adding TLS must not move discovery, authorization, content verification, or application semantics into the TCP adapter. diff --git a/src/GameCult.Mesh/docs/typed-document-path.md b/src/GameCult.Mesh/docs/typed-document-path.md index 3ee3183c..aab47930 100644 --- a/src/GameCult.Mesh/docs/typed-document-path.md +++ b/src/GameCult.Mesh/docs/typed-document-path.md @@ -109,7 +109,7 @@ messages directly. ```csharp using GameCult.Mesh; -using var node = await CultMesh.CreateNodeAsync("world.ccmp", new CultMeshNodeOptions +using var node = await CultMesh.CreateNodeAsync("world.cc", new CultMeshNodeOptions { DatabaseOptions = new CultNetDatabaseOptions { diff --git a/src/GameCult.Mesh/docs/verses.md b/src/GameCult.Mesh/docs/verses.md index 656dcfaf..fbd9a522 100644 --- a/src/GameCult.Mesh/docs/verses.md +++ b/src/GameCult.Mesh/docs/verses.md @@ -21,8 +21,10 @@ The same transport can carry many Verses: subscribed overlay - `Compatibility`: transport version, rules hash, compatible Verse ids, and plugin requirements -- `DiscoveryEndpoints`: endpoints where nodes for the Verse can be found -- `AuthorityRuntimeIds`: known authoritative runtimes when cluster-shaped +- `AuthorityRoutes`: exact authority-runtime, endpoint, protocol, priority, and + generation bindings. This is the routing authority. +- `DiscoveryEndpoints`: read-only endpoint projection for legacy inspection +- `AuthorityRuntimeIds`: read-only authority projection for legacy inspection - `ParentVerseId`: source Verse for overlays or branches ## Compatibility @@ -57,6 +59,13 @@ static config, cloud discovery, peer gossip, or mod portals. It does not care where the descriptor came from. That keeps discovery pluggable while the Verse model stays stable. +An endpoint and an authority id are not two independent facts. A descriptor +with explicit routes binds them. The legacy parallel arrays are accepted only +when exactly one authority runtime exists; a multi-authority legacy descriptor +is ambiguous and fails closed. Session establishment then requires the selected +physical route to prove the exact Verse, authority runtime, protocol, and route +generation before the session becomes online. + Peer exchange is the torrent-shaped layer under Verse discovery. A Verse catalog says which graph exists; a peer catalog says which nodes can currently serve or observe parts of that graph. Peer cards can advertise discovery, diff --git a/src/GameCult.Networking.WebSockets/CultNetWebSocketMessageIO.cs b/src/GameCult.Networking.WebSockets/CultNetWebSocketMessageIO.cs new file mode 100644 index 00000000..de10a66e --- /dev/null +++ b/src/GameCult.Networking.WebSockets/CultNetWebSocketMessageIO.cs @@ -0,0 +1,46 @@ +using System; +using System.IO; +using System.Net.WebSockets; +using System.Threading; +using System.Threading.Tasks; + +namespace GameCult.Networking.WebSockets; + +internal static class CultNetWebSocketMessageIO +{ + public static Task SendAsync(WebSocket socket, byte[] payload, CancellationToken cancellationToken) => + socket.SendAsync(new ArraySegment(payload), WebSocketMessageType.Binary, true, cancellationToken); + + public static async Task ReceiveAsync( + WebSocket socket, + int maxMessageBytes, + CancellationToken cancellationToken) + { + var segment = new byte[Math.Min(16 * 1024, maxMessageBytes)]; + using var message = new MemoryStream(); + while (true) + { + var result = await socket.ReceiveAsync(new ArraySegment(segment), cancellationToken) + .ConfigureAwait(false); + if (result.MessageType == WebSocketMessageType.Close) return null; + if (result.MessageType != WebSocketMessageType.Binary) + { + await socket.CloseAsync( + WebSocketCloseStatus.InvalidMessageType, + "CultNet requires binary MessagePack WebSocket messages.", + cancellationToken).ConfigureAwait(false); + return null; + } + if (message.Length + result.Count > maxMessageBytes) + { + await socket.CloseAsync( + WebSocketCloseStatus.MessageTooBig, + "CultNet message exceeds the configured maximum.", + cancellationToken).ConfigureAwait(false); + return null; + } + message.Write(segment, 0, result.Count); + if (result.EndOfMessage) return message.ToArray(); + } + } +} diff --git a/src/GameCult.Networking.WebSockets/CultNetWebSocketSchemaClient.cs b/src/GameCult.Networking.WebSockets/CultNetWebSocketSchemaClient.cs new file mode 100644 index 00000000..394fe207 --- /dev/null +++ b/src/GameCult.Networking.WebSockets/CultNetWebSocketSchemaClient.cs @@ -0,0 +1,166 @@ +using System.Collections.Concurrent; +using System.Net.WebSockets; +using GameCult.Networking; + +namespace GameCult.Networking.WebSockets; + +/// Schema-v0 client over a binary WebSocket route. +public sealed class CultNetWebSocketSchemaClient : ICultNetUriSchemaClient, ICultNetSchemaClientHealth +{ + private readonly ConcurrentDictionary> _handlers = new(); + private readonly object _handlerGate = new(); + private readonly SemaphoreSlim _sendGate = new(1, 1); + private readonly CancellationTokenSource _shutdown = new(); + private readonly TaskCompletionSource _backgroundFailure = new( + TaskCreationOptions.RunContinuationsAsynchronously); + private readonly Action? _configure; + private readonly int _maxMessageBytes; + private ClientWebSocket? _socket; + private bool _disposed; + + /// Creates a binary WebSocket schema client. + public CultNetWebSocketSchemaClient( + Action? configure = null, + int maxMessageBytes = 4 * 1024 * 1024) + { + if (maxMessageBytes <= 0) throw new ArgumentOutOfRangeException(nameof(maxMessageBytes)); + _configure = configure; + _maxMessageBytes = maxMessageBytes; + } + + /// + public bool Connected => !_disposed && _socket?.State == WebSocketState.Open; + + /// + public Task BackgroundFailure => _backgroundFailure.Task; + + /// + public void Connect(string host, int port) + { + if (string.IsNullOrWhiteSpace(host)) throw new ArgumentException("Host is required.", nameof(host)); + if (port <= 0 || port > 65535) throw new ArgumentOutOfRangeException(nameof(port)); + ConnectAsync(new Uri($"ws://{host}:{port}/cultmesh"), CancellationToken.None) + .GetAwaiter().GetResult(); + } + + /// Connects to an exact WebSocket route. + public async Task ConnectAsync(Uri endpoint, CancellationToken cancellationToken = default) + { + if (endpoint == null) throw new ArgumentNullException(nameof(endpoint)); + if (endpoint.Scheme is not ("ws" or "wss")) + throw new ArgumentException("CultNet WebSocket endpoint must use ws:// or wss://.", nameof(endpoint)); + ThrowIfDisposed(); + if (_socket != null) throw new InvalidOperationException("CultNet WebSocket client is already connected."); + var socket = new ClientWebSocket(); + _configure?.Invoke(socket.Options); + try + { + await socket.ConnectAsync(endpoint, cancellationToken).ConfigureAwait(false); + _socket = socket; + _ = ReceiveLoopAsync(socket, _shutdown.Token); + } + catch + { + socket.Dispose(); + throw; + } + } + + /// + public void SendCultNet(T message) where T : ICultNetSchemaMessage + { + if (message == null) throw new ArgumentNullException(nameof(message)); + ThrowIfDisposed(); + var socket = _socket; + if (socket?.State != WebSocketState.Open) + throw new InvalidOperationException("CultNet WebSocket client is not connected."); + var payload = CultNetSchemaMessageSerialization.Serialize(message); + if (payload.Length > _maxMessageBytes) + throw new InvalidOperationException( + $"CultNet message exceeds the {_maxMessageBytes}-byte WebSocket limit."); + _sendGate.Wait(); + try + { + CultNetWebSocketMessageIO.SendAsync(socket, payload, _shutdown.Token).GetAwaiter().GetResult(); + } + finally + { + _sendGate.Release(); + } + } + + /// + public void OnCultNet(Action callback) where T : ICultNetSchemaMessage + { + if (callback == null) throw new ArgumentNullException(nameof(callback)); + ThrowIfDisposed(); + lock (_handlerGate) + { + if (!_handlers.TryGetValue(typeof(T), out var handlers)) + _handlers[typeof(T)] = handlers = new List(); + handlers.Add(callback); + } + } + + /// + public void Dispose() + { + if (_disposed) return; + _disposed = true; + _shutdown.Cancel(); + var socket = _socket; + _socket = null; + try + { + if (socket?.State is WebSocketState.Open or WebSocketState.CloseReceived) + { + using var closeTimeout = new CancellationTokenSource(TimeSpan.FromSeconds(1)); + socket.CloseAsync(WebSocketCloseStatus.NormalClosure, "CultNet client disposed", closeTimeout.Token) + .GetAwaiter().GetResult(); + } + } + catch (Exception error) when (error is WebSocketException or OperationCanceledException) + { + socket?.Abort(); + } + socket?.Dispose(); + _sendGate.Dispose(); + _shutdown.Dispose(); + } + + private async Task ReceiveLoopAsync(WebSocket socket, CancellationToken cancellationToken) + { + try + { + while (!_disposed && socket.State == WebSocketState.Open && !cancellationToken.IsCancellationRequested) + { + var payload = await CultNetWebSocketMessageIO.ReceiveAsync(socket, _maxMessageBytes, cancellationToken) + .ConfigureAwait(false); + if (payload == null) return; + Dispatch(CultNetSchemaMessageSerialization.Deserialize(payload)); + } + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + } + catch (Exception error) when (!_disposed) + { + _backgroundFailure.TrySetResult(error); + } + } + + private void Dispatch(ICultNetSchemaMessage message) + { + Delegate[] handlers; + lock (_handlerGate) + handlers = _handlers.TryGetValue(message.GetType(), out var registered) + ? registered.ToArray() + : Array.Empty(); + foreach (var handler in handlers) handler.DynamicInvoke(message); + } + + private void ThrowIfDisposed() + { + if (_disposed) throw new ObjectDisposedException(nameof(CultNetWebSocketSchemaClient)); + } +} diff --git a/src/GameCult.Networking.WebSockets/CultNetWebSocketSchemaServer.cs b/src/GameCult.Networking.WebSockets/CultNetWebSocketSchemaServer.cs new file mode 100644 index 00000000..fa2728d8 --- /dev/null +++ b/src/GameCult.Networking.WebSockets/CultNetWebSocketSchemaServer.cs @@ -0,0 +1,222 @@ +using System.Collections.Concurrent; +using System.Net; +using System.Net.WebSockets; +using GameCult.Networking; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Routing; + +namespace GameCult.Networking.WebSockets; + +/// Controls the bounded CultNet schema-v0 WebSocket endpoint. +public sealed class CultNetWebSocketEndpointOptions +{ + /// Maximum accepted binary CultNet message size. + public int MaxMessageBytes { get; init; } = 4 * 1024 * 1024; + + /// + /// Authorizes the HTTP upgrade before it can become a CultNet peer. Production hosts + /// should bind this to their authenticated session middleware. + /// + public Func>? AuthorizeAsync { get; init; } + + /// Allows an unauthenticated endpoint for explicit local development only. + public bool AllowAnonymousDevelopment { get; init; } +} + +/// +/// Adapts one binary WebSocket endpoint to the transport-neutral CultNet schema server port. +/// The host owns HTTP, TLS, and authentication; this adapter owns bounded binary message +/// assembly, schema validation, handler dispatch, peer lifetime, and serialized sends. +/// +public sealed class CultNetWebSocketSchemaServer : + ICultNetSchemaServer, + ICultNetSchemaServerPeerLifecycle, + IAsyncDisposable +{ + private readonly ConcurrentDictionary _handlers = new(); + private readonly ConcurrentDictionary _peers = new(); + private bool _disposed; + + /// + public event Action? PeerDisconnected; + + /// Gets the active upgraded peer count. + public int PeerCount => _peers.Count; + + /// + public void OnCultNet(Func callback) + where TMessage : ICultNetSchemaMessage + { + ArgumentNullException.ThrowIfNull(callback); + ThrowIfDisposed(); + _handlers[typeof(TMessage)] = callback; + } + + /// + public void RemoveCultNetMessageListener(Delegate callback) + where TMessage : ICultNetSchemaMessage + { + if (_handlers.TryGetValue(typeof(TMessage), out var existing) && existing == callback) + _handlers.TryRemove(typeof(TMessage), out _); + } + + /// Serves one already-authorized WebSocket until it closes. + public async Task AcceptAsync( + WebSocket socket, + EndPoint? remoteEndPoint = null, + int maxMessageBytes = 4 * 1024 * 1024, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(socket); + ThrowIfDisposed(); + if (maxMessageBytes <= 0) throw new ArgumentOutOfRangeException(nameof(maxMessageBytes)); + var peer = new WebSocketPeer(socket, remoteEndPoint, maxMessageBytes); + if (!_peers.TryAdd(peer, 0)) throw new InvalidOperationException("CultNet WebSocket peer already exists."); + try + { + while (!_disposed && socket.State == WebSocketState.Open && !cancellationToken.IsCancellationRequested) + { + var payload = await CultNetWebSocketMessageIO.ReceiveAsync(socket, maxMessageBytes, cancellationToken) + .ConfigureAwait(false); + if (payload == null) break; + var message = CultNetSchemaMessageSerialization.Deserialize(payload); + if (!_handlers.TryGetValue(message.GetType(), out var handler)) continue; + if (handler.DynamicInvoke(message, peer) is Task task) await task.ConfigureAwait(false); + } + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + } + finally + { + if (_peers.TryRemove(peer, out _)) PeerDisconnected?.Invoke(peer); + await peer.DisposeAsync().ConfigureAwait(false); + } + } + + /// + public async ValueTask DisposeAsync() + { + if (_disposed) return; + _disposed = true; + var peers = _peers.Keys.ToArray(); + _peers.Clear(); + foreach (var peer in peers) + { + PeerDisconnected?.Invoke(peer); + await peer.DisposeAsync().ConfigureAwait(false); + } + _handlers.Clear(); + } + + private void ThrowIfDisposed() + { + ObjectDisposedException.ThrowIf(_disposed, this); + } + + private sealed class WebSocketPeer : + ICultNetSchemaServerPeer, + ICultNetSchemaServerPeerLocation, + IAsyncDisposable + { + private readonly WebSocket _socket; + private readonly SemaphoreSlim _sendGate = new(1, 1); + private readonly int _maxMessageBytes; + private bool _disposed; + + public WebSocketPeer(WebSocket socket, EndPoint? remoteEndPoint, int maxMessageBytes) + { + _socket = socket; + RemoteEndPoint = remoteEndPoint; + _maxMessageBytes = maxMessageBytes; + } + + public EndPoint? RemoteEndPoint { get; } + + public void SendCultNet(TMessage message) where TMessage : ICultNetSchemaMessage + { + ArgumentNullException.ThrowIfNull(message); + if (_disposed || _socket.State != WebSocketState.Open) + throw new InvalidOperationException("CultNet WebSocket peer is not open."); + var payload = CultNetSchemaMessageSerialization.Serialize(message); + if (payload.Length > _maxMessageBytes) + throw new InvalidOperationException( + $"CultNet message exceeds the {_maxMessageBytes}-byte WebSocket limit."); + _sendGate.Wait(); + try + { + CultNetWebSocketMessageIO.SendAsync(_socket, payload, CancellationToken.None) + .GetAwaiter().GetResult(); + } + finally + { + _sendGate.Release(); + } + } + + public async ValueTask DisposeAsync() + { + if (_disposed) return; + _disposed = true; + try + { + if (_socket.State is WebSocketState.Open or WebSocketState.CloseReceived) + { + using var closeTimeout = new CancellationTokenSource(TimeSpan.FromSeconds(1)); + await _socket.CloseAsync(WebSocketCloseStatus.NormalClosure, "CultNet peer closed", closeTimeout.Token) + .ConfigureAwait(false); + } + } + catch (Exception error) when (error is WebSocketException or OperationCanceledException) + { + _socket.Abort(); + } + _socket.Dispose(); + _sendGate.Dispose(); + } + } +} + +/// ASP.NET Core endpoint mapping for the CultNet WebSocket schema server. +public static class CultNetWebSocketEndpointRouteBuilderExtensions +{ + /// Maps one authenticated binary CultNet WebSocket endpoint. + public static IEndpointConventionBuilder MapCultNetWebSocket( + this IEndpointRouteBuilder endpoints, + string pattern, + CultNetWebSocketSchemaServer server, + CultNetWebSocketEndpointOptions options) + { + ArgumentNullException.ThrowIfNull(endpoints); + ArgumentException.ThrowIfNullOrWhiteSpace(pattern); + ArgumentNullException.ThrowIfNull(server); + ArgumentNullException.ThrowIfNull(options); + if (options.MaxMessageBytes <= 0) throw new ArgumentOutOfRangeException(nameof(options.MaxMessageBytes)); + if (options.AuthorizeAsync == null && !options.AllowAnonymousDevelopment) + throw new InvalidOperationException( + "CultNet WebSocket endpoints require AuthorizeAsync unless AllowAnonymousDevelopment is explicitly enabled."); + + return endpoints.Map(pattern, async context => + { + if (!context.WebSockets.IsWebSocketRequest) + { + context.Response.StatusCode = StatusCodes.Status426UpgradeRequired; + return; + } + if (options.AuthorizeAsync != null && !await options.AuthorizeAsync(context).ConfigureAwait(false)) + { + context.Response.StatusCode = StatusCodes.Status401Unauthorized; + return; + } + using var socket = await context.WebSockets.AcceptWebSocketAsync().ConfigureAwait(false); + await server.AcceptAsync( + socket, + context.Connection.RemoteIpAddress == null + ? null + : new IPEndPoint(context.Connection.RemoteIpAddress, context.Connection.RemotePort), + options.MaxMessageBytes, + context.RequestAborted).ConfigureAwait(false); + }); + } +} diff --git a/src/GameCult.Networking.WebSockets/GameCult.Networking.WebSockets.csproj b/src/GameCult.Networking.WebSockets/GameCult.Networking.WebSockets.csproj new file mode 100644 index 00000000..fb8e0bd5 --- /dev/null +++ b/src/GameCult.Networking.WebSockets/GameCult.Networking.WebSockets.csproj @@ -0,0 +1,19 @@ + + + + net10.0;netstandard2.1 + enable + enable + CultNet schema-v0 WebSocket host adapter for browser and web runtimes + GameCult.Networking.WebSockets + game;networking;websocket;cultnet;cultmesh + + + + + + + + + + diff --git a/src/GameCult.Networking/Contracts/cultnet.database-change-raw.schema.json b/src/GameCult.Networking/Contracts/cultnet.database-change-raw.schema.json new file mode 100644 index 00000000..dd650fa0 --- /dev/null +++ b/src/GameCult.Networking/Contracts/cultnet.database-change-raw.schema.json @@ -0,0 +1,24 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/GameCult/cultnet-ts/contracts/cultnet.database-change-raw.schema.json", + "title": "CultNet Raw Database Change Message", + "type": "object", + "required": ["schemaVersion", "messageId", "subscriptionId", "changeKind"], + "additionalProperties": false, + "properties": { + "schemaVersion": { "const": "cultnet.database_change_raw.v0" }, + "messageId": { "type": "string", "minLength": 1 }, + "subscriptionId": { "type": "string", "minLength": 1 }, + "changeKind": { "enum": ["added", "updated", "removed"] }, + "document": { "$ref": "./cultnet.raw-document-record.schema.json" }, + "recordKey": { "type": "string", "minLength": 1 }, + "schemaId": { "type": "string", "minLength": 1 } + }, + "allOf": [ + { + "if": { "properties": { "changeKind": { "const": "removed" } } }, + "then": { "required": ["recordKey", "schemaId"] }, + "else": { "required": ["document"] } + } + ] +} diff --git a/src/GameCult.Networking/Contracts/cultnet.database-subscribe.schema.json b/src/GameCult.Networking/Contracts/cultnet.database-subscribe.schema.json new file mode 100644 index 00000000..08304199 --- /dev/null +++ b/src/GameCult.Networking/Contracts/cultnet.database-subscribe.schema.json @@ -0,0 +1,19 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/GameCult/cultnet-ts/contracts/cultnet.database-subscribe.schema.json", + "title": "CultNet Database Subscribe Message", + "type": "object", + "required": ["schemaVersion", "messageId", "subscriptionId", "includeSnapshot"], + "additionalProperties": false, + "properties": { + "schemaVersion": { "const": "cultnet.database_subscribe.v0" }, + "messageId": { "type": "string", "minLength": 1 }, + "subscriptionId": { "type": "string", "minLength": 1 }, + "schemaIds": { "type": "array", "items": { "type": "string", "minLength": 1 } }, + "recordKeys": { "type": "array", "items": { "type": "string", "minLength": 1 } }, + "includeSnapshot": { "type": "boolean" }, + "consumerRuntimeId": { "type": "string", "minLength": 1 }, + "bodyIds": { "type": "array", "items": { "type": "string", "minLength": 1 } }, + "supportedBodyTransports": { "type": "array", "items": { "type": "string", "minLength": 1 } } + } +} diff --git a/src/GameCult.Networking/Contracts/cultnet.database-unsubscribe.schema.json b/src/GameCult.Networking/Contracts/cultnet.database-unsubscribe.schema.json new file mode 100644 index 00000000..7d4b0855 --- /dev/null +++ b/src/GameCult.Networking/Contracts/cultnet.database-unsubscribe.schema.json @@ -0,0 +1,13 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/GameCult/cultnet-ts/contracts/cultnet.database-unsubscribe.schema.json", + "title": "CultNet Database Unsubscribe Message", + "type": "object", + "required": ["schemaVersion", "messageId", "subscriptionId"], + "additionalProperties": false, + "properties": { + "schemaVersion": { "const": "cultnet.database_unsubscribe.v0" }, + "messageId": { "type": "string", "minLength": 1 }, + "subscriptionId": { "type": "string", "minLength": 1 } + } +} diff --git a/src/GameCult.Networking/CultNetDatabase.cs b/src/GameCult.Networking/CultNetDatabase.cs index 6e1ed41f..996e85f5 100644 --- a/src/GameCult.Networking/CultNetDatabase.cs +++ b/src/GameCult.Networking/CultNetDatabase.cs @@ -2,6 +2,7 @@ using System; using System.Collections.Generic; using System.Linq; using System.Reflection; +using System.Threading; using System.Threading.Tasks; using GameCult.Caching; using R3; @@ -503,6 +504,22 @@ namespace GameCult.Networking /// public IReadOnlyList ClientAuthorityScopes => _clientAuthorityScopes; + /// Gets whether this database instance can authoritatively replace the document at a key. + public bool CanWriteAuthoritatively(CultRecordKey key) where T : class + { + ThrowIfDisposed(); + var descriptor = _cache.Registry.GetRequired(); + return ResolveShardInternal(descriptor, key).IsPrimary; + } + + /// Gets whether this database instance can submit a prediction for the document at a key. + public bool CanSubmitPrediction(CultRecordKey key) where T : class + { + ThrowIfDisposed(); + var descriptor = _cache.Registry.GetRequired(); + return _clientAuthorityScopes.Any(scope => scope.Matches(_runtimeId, descriptor, key)); + } + /// /// Gets mutation log entries for a shard after the supplied sequence. /// @@ -780,21 +797,6 @@ namespace GameCult.Networking return _cache.GetByIndex(alias, value); } - /// - /// Opens a reactive POCO presentation for one distributed CultCache document. - /// - public CultManagedDocument Document(CultRecordKey key) where T : class - { - ThrowIfDisposed(); - return new CultManagedDocument( - key, - () => _cache.Get(key), - async value => { await PutAsync(key, value).ConfigureAwait(false); }, - WatchRecord(key) - .Where(change => change.Document != null) - .Select(change => change.Document!)); - } - /// /// Adds or replaces a document at a specific key. /// @@ -803,26 +805,30 @@ namespace GameCult.Networking ThrowIfDisposed(); if (document == null) throw new ArgumentNullException(nameof(document)); - var descriptor = _cache.Registry.GetRequired(); + var descriptor = _cache.Registry.GetRequired(document.GetType()); var shard = ResolveShardInternal(descriptor, key); EnsurePrimary(shard, descriptor.SchemaId, key); var previous = _cache.Get(key); var handle = await _cache.UpsertAsync(document, new CultRecordHandle(key)).ConfigureAwait(false); - AppendMutationLogEntry( - shard, - previous == null ? CultNetDatabaseChangeKind.Added : CultNetDatabaseChangeKind.Updated, - key, - descriptor.SchemaId, - document, - previous); - Publish(new CultNetDatabaseChange( - previous == null ? CultNetDatabaseChangeKind.Added : CultNetDatabaseChangeKind.Updated, - key, - descriptor.SchemaId, - shard, - document, - previous)); + void PublishCommittedPut() + { + AppendMutationLogEntry( + shard, + previous == null ? CultNetDatabaseChangeKind.Added : CultNetDatabaseChangeKind.Updated, + key, + descriptor.SchemaId, + document, + previous); + Publish(new CultNetDatabaseChange( + previous == null ? CultNetDatabaseChangeKind.Added : CultNetDatabaseChangeKind.Updated, + key, + descriptor.SchemaId, + shard, + document, + previous)); + } + PublishCommittedPut(); return handle; } @@ -834,7 +840,7 @@ namespace GameCult.Networking ThrowIfDisposed(); if (document == null) throw new ArgumentNullException(nameof(document)); - var descriptor = _cache.Registry.GetRequired(); + var descriptor = _cache.Registry.GetRequired(document.GetType()); var shard = ResolveShardInternal(descriptor, key); EnsureClientAuthority(descriptor, key); @@ -865,20 +871,24 @@ namespace GameCult.Networking if (previous != null) { _cache.Remove(new CultRecordHandle(key)); - AppendMutationLogEntry( - shard, - CultNetDatabaseChangeKind.Removed, - key, - descriptor.SchemaId, - document: null, - previousDocument: previous); - Publish(new CultNetDatabaseChange( - CultNetDatabaseChangeKind.Removed, - key, - descriptor.SchemaId, - shard, - document: null, - previousDocument: previous)); + void PublishCommittedDelete() + { + AppendMutationLogEntry( + shard, + CultNetDatabaseChangeKind.Removed, + key, + descriptor.SchemaId, + document: null, + previousDocument: previous); + Publish(new CultNetDatabaseChange( + CultNetDatabaseChangeKind.Removed, + key, + descriptor.SchemaId, + shard, + document: null, + previousDocument: previous)); + } + PublishCommittedDelete(); } return Task.CompletedTask; @@ -904,31 +914,38 @@ namespace GameCult.Networking var document = await _documents.ApplyRawDocumentPutMessageAsync(_cache, message).ConfigureAwait(false); var kind = previous == null ? CultNetDatabaseChangeKind.Added : CultNetDatabaseChangeKind.Updated; var predictionKey = PredictionKey(descriptor.SchemaId, key); - if (_predictedDocuments.Remove(predictionKey)) + var reconcilesPrediction = _predictedDocuments.ContainsKey(predictionKey); + if (reconcilesPrediction) { kind = CultNetDatabaseChangeKind.Reconciled; } - AppendMutationLogEntry( - shard, - kind == CultNetDatabaseChangeKind.Reconciled ? CultNetDatabaseChangeKind.Updated : kind, - key, - descriptor.SchemaId, - document, - previous, - new CultNetShardLogEntryMessage - { - ChangeKind = previous == null ? "added" : "updated", - Put = message - }); - PublishUntyped( - descriptor.DocumentType, - kind, - key, - descriptor.SchemaId, - shard, - document, - previous); + void PublishCommittedPut() + { + if (reconcilesPrediction) + _predictedDocuments.Remove(predictionKey); + AppendMutationLogEntry( + shard, + kind == CultNetDatabaseChangeKind.Reconciled ? CultNetDatabaseChangeKind.Updated : kind, + key, + descriptor.SchemaId, + document, + previous, + new CultNetShardLogEntryMessage + { + ChangeKind = previous == null ? "added" : "updated", + Put = message + }); + PublishUntyped( + descriptor.DocumentType, + kind, + key, + descriptor.SchemaId, + shard, + document, + previous); + } + PublishCommittedPut(); return document; } @@ -951,26 +968,30 @@ namespace GameCult.Networking } _cache.Remove(key); - AppendMutationLogEntry( - shard, - CultNetDatabaseChangeKind.Removed, - key, - descriptor.SchemaId, - document: null, - previousDocument: previous, - wireEntry: new CultNetShardLogEntryMessage - { - ChangeKind = "removed", - Delete = message - }); - PublishUntyped( - descriptor.DocumentType, - CultNetDatabaseChangeKind.Removed, - key, - descriptor.SchemaId, - shard, - document: null, - previousDocument: previous); + void PublishCommittedDelete() + { + AppendMutationLogEntry( + shard, + CultNetDatabaseChangeKind.Removed, + key, + descriptor.SchemaId, + document: null, + previousDocument: previous, + wireEntry: new CultNetShardLogEntryMessage + { + ChangeKind = "removed", + Delete = message + }); + PublishUntyped( + descriptor.DocumentType, + CultNetDatabaseChangeKind.Removed, + key, + descriptor.SchemaId, + shard, + document: null, + previousDocument: previous); + } + PublishCommittedDelete(); return Task.CompletedTask; } diff --git a/src/GameCult.Networking/CultNetDatabaseSubscriptionClient.cs b/src/GameCult.Networking/CultNetDatabaseSubscriptionClient.cs index 39cc4cf3..db1e2907 100644 --- a/src/GameCult.Networking/CultNetDatabaseSubscriptionClient.cs +++ b/src/GameCult.Networking/CultNetDatabaseSubscriptionClient.cs @@ -248,18 +248,12 @@ namespace GameCult.Networking }); } - private async Task DeleteAsync(CultNetDatabaseChangeRawMessage message) + private Task DeleteAsync(CultNetDatabaseChangeRawMessage message) { - if (string.IsNullOrWhiteSpace(message.RecordKey) || string.IsNullOrWhiteSpace(message.SchemaId)) return; - var descriptor = _documents.ResolveDescriptorForSchemaId(message.SchemaId!); - var handle = Activator.CreateInstance( - typeof(CultRecordHandle<>).MakeGenericType(descriptor.DocumentType), - new object[] { new CultRecordKey(message.RecordKey!) }); - var task = (Task)typeof(CultCache) - .GetMethod(nameof(CultCache.DeleteAsync), BindingFlags.Public | BindingFlags.Instance)! - .MakeGenericMethod(descriptor.DocumentType) - .Invoke(_cache, new[] { handle })!; - await task.ConfigureAwait(false); + if (string.IsNullOrWhiteSpace(message.RecordKey) || string.IsNullOrWhiteSpace(message.SchemaId)) return Task.CompletedTask; + _documents.ResolveDescriptorForSchemaId(message.SchemaId!); + _cache.Remove(new CultRecordKey(message.RecordKey!)); + return Task.CompletedTask; } private void Enqueue(Func operation) diff --git a/src/GameCult.Networking/CultNetDatabaseSubscriptionServer.cs b/src/GameCult.Networking/CultNetDatabaseSubscriptionServer.cs index c5fe47bd..3d284556 100644 --- a/src/GameCult.Networking/CultNetDatabaseSubscriptionServer.cs +++ b/src/GameCult.Networking/CultNetDatabaseSubscriptionServer.cs @@ -13,6 +13,8 @@ namespace GameCult.Networking { /// /// Publishes live database changes through any schema-v0 server transport. + /// Optional record projection runs after authorization and owns both initial + /// snapshot and live-update delivery for a peer. /// public sealed class CultNetDatabaseSubscriptionServer : IDisposable { @@ -20,18 +22,32 @@ namespace GameCult.Networking private readonly CultNetDatabase _database; private readonly Func _subscribe; private readonly Func _unsubscribe; + private readonly Func? _authorizeRequest; + private readonly Func? _authorizeRecord; + private readonly Func? _projectRecord; private readonly ICultNetSchemaServerPeerLifecycle? _peerLifecycle; private readonly ConcurrentDictionary _subscriptions = new ConcurrentDictionary(SubscriptionKeyComparer.Instance); private readonly ConcurrentDictionary _requests = new ConcurrentDictionary(SubscriptionKeyComparer.Instance); + private readonly ConcurrentDictionary _projections = + new ConcurrentDictionary(SubscriptionKeyComparer.Instance); + private readonly object _lifecycleGate = new(); private bool _disposed; /// Attaches live database subscription handlers to a schema server. - public CultNetDatabaseSubscriptionServer(ICultNetSchemaServer server, CultNetDatabase database) + public CultNetDatabaseSubscriptionServer( + ICultNetSchemaServer server, + CultNetDatabase database, + Func? authorizeRequest = null, + Func? authorizeRecord = null, + Func? projectRecord = null) { _server = server ?? throw new ArgumentNullException(nameof(server)); _database = database ?? throw new ArgumentNullException(nameof(database)); + _authorizeRequest = authorizeRequest; + _authorizeRecord = authorizeRecord; + _projectRecord = projectRecord; _subscribe = HandleSubscribeAsync; _unsubscribe = HandleUnsubscribeAsync; _server.OnCultNet(_subscribe); @@ -48,34 +64,48 @@ namespace GameCult.Networking /// public event Action? DemandChanged; + /// + /// Re-evaluates every live peer projection against current authorization and source state. + /// Records that lost visibility are explicitly removed and body demand is withdrawn while + /// the request is unauthorized. The subscription intent remains available for a later + /// authorized generation on the same established peer. + /// + public void Reconcile() + { + lock (_lifecycleGate) + { + if (_disposed) return; + foreach (var entry in _requests.ToArray()) + Reconcile(entry.Key, entry.Value); + } + } + /// Detaches handlers and releases all active watches. public void Dispose() { - if (_disposed) return; - _disposed = true; - _server.RemoveCultNetMessageListener(_subscribe); - _server.RemoveCultNetMessageListener(_unsubscribe); - if (_peerLifecycle != null) - _peerLifecycle.PeerDisconnected -= HandlePeerDisconnected; - foreach (var entry in _subscriptions) + lock (_lifecycleGate) { - entry.Value.Dispose(); - if (_requests.TryRemove(entry.Key, out var request)) - PublishDemand(request, entry.Key, active: false); + if (_disposed) return; + _disposed = true; + _server.RemoveCultNetMessageListener(_subscribe); + _server.RemoveCultNetMessageListener(_unsubscribe); + if (_peerLifecycle != null) + _peerLifecycle.PeerDisconnected -= HandlePeerDisconnected; + foreach (var entry in _subscriptions.ToArray()) + Withdraw(entry.Key, sendRemovals: false, forgetRequest: true); + _subscriptions.Clear(); + _requests.Clear(); + _projections.Clear(); } - _subscriptions.Clear(); - _requests.Clear(); } private void HandlePeerDisconnected(ICultNetSchemaServerPeer peer) { - foreach (var key in _subscriptions.Keys.Where(candidate => - SubscriptionKeyComparer.SamePeer(candidate.Peer, peer)).ToArray()) + lock (_lifecycleGate) { - if (_subscriptions.TryRemove(key, out var subscription)) - subscription.Dispose(); - if (_requests.TryRemove(key, out var request)) - PublishDemand(request, key, active: false); + foreach (var key in _subscriptions.Keys.Where(candidate => + SubscriptionKeyComparer.SamePeer(candidate.Peer, peer)).ToArray()) + Withdraw(key, sendRemovals: false, forgetRequest: true); } } @@ -89,39 +119,49 @@ namespace GameCult.Networking peer.SendCultNet(new CultNetErrorMessage { Error = "Database subscription requires a subscriptionId or messageId." }); return Task.CompletedTask; } - - var key = new SubscriptionKey(peer, subscriptionId); - if (_requests.TryGetValue(key, out var previous)) - PublishDemand(previous, key, active: false); - _subscriptions.AddOrUpdate( - key, - _ => Watch(request, subscriptionId, peer), - (_, current) => - { - current.Dispose(); - return Watch(request, subscriptionId, peer); - }); - _requests[key] = request; - PublishDemand(request, key, active: true); - if (request.IncludeSnapshot) + if (_authorizeRequest?.Invoke(request, peer) == false) { - peer.SendCultNet(_database.Documents.CreateRawSnapshotResponse( - _database.Cache, - request.MessageId, - new CultNetSnapshotRequestMessage - { - MessageId = request.MessageId, - SchemaIds = request.SchemaIds, - RecordKeys = request.RecordKeys - })); + peer.SendCultNet(new CultNetErrorMessage { Error = "Database subscription is not authorized for this peer." }); + return Task.CompletedTask; } - else + + lock (_lifecycleGate) { - peer.SendCultNet(new CultNetSnapshotResponseRawMessage + var key = new SubscriptionKey(peer, subscriptionId); + Withdraw(key, sendRemovals: true, forgetRequest: true); + var projection = new SubscriptionProjectionState(); + _projections[key] = projection; + _requests[key] = request; + try { - MessageId = request.MessageId, - Documents = Array.Empty() - }); + _subscriptions[key] = Watch(request, subscriptionId, peer, key); + if (request.IncludeSnapshot) + { + var snapshot = CreateProjectedSnapshot(request, peer); + foreach (var entry in snapshot.BySourceRecordKey) + projection.DeliveredBySourceRecordKey[entry.Key] = entry.Value; + peer.SendCultNet(new CultNetSnapshotResponseRawMessage + { + MessageId = request.MessageId, + Documents = snapshot.Documents + }); + } + else + { + peer.SendCultNet(new CultNetSnapshotResponseRawMessage + { + MessageId = request.MessageId, + Documents = Array.Empty() + }); + } + PublishDemand(request, key, active: true); + projection.DemandActive = true; + } + catch + { + Withdraw(key, sendRemovals: false, forgetRequest: true); + throw; + } } return Task.CompletedTask; } @@ -134,9 +174,8 @@ namespace GameCult.Networking if (!string.IsNullOrWhiteSpace(subscriptionId)) { var key = new SubscriptionKey(peer, subscriptionId); - if (_subscriptions.TryRemove(key, out var subscription)) subscription.Dispose(); - if (_requests.TryRemove(key, out var subscribed)) - PublishDemand(subscribed, key, active: false); + lock (_lifecycleGate) + Withdraw(key, sendRemovals: false, forgetRequest: true); } return Task.CompletedTask; } @@ -179,19 +218,232 @@ namespace GameCult.Networking private IDisposable Watch( CultNetDatabaseSubscribeMessage request, string subscriptionId, - ICultNetSchemaServerPeer peer) + ICultNetSchemaServerPeer peer, + SubscriptionKey key) { return _database.WatchAllChanges().Subscribe(change => { - var message = CreateChange(change, request, subscriptionId); - if (message != null) peer.SendCultNet(message); + lock (_lifecycleGate) + { + if (_disposed || !_requests.ContainsKey(key) || !_projections.TryGetValue(key, out var projection)) + return; + if (_authorizeRequest?.Invoke(request, peer) == false) + { + Reconcile(key, request); + return; + } + + var sourceRecordKey = ResolveChangeRecordKey(change); + var message = CreateAuthorizedChange(change, request, subscriptionId, peer); + var projected = message?.Document; + if (projected != null && _projectRecord != null) + projected = _projectRecord(request, peer, projected); + ApplyProjectedChange( + projection, + sourceRecordKey, + projected, + peer, + subscriptionId); + } }); } + private void Reconcile(SubscriptionKey key, CultNetDatabaseSubscribeMessage request) + { + if (!_projections.TryGetValue(key, out var projection)) return; + var authorized = _authorizeRequest?.Invoke(request, key.Peer) != false; + var next = authorized + ? CreateProjectedSnapshot(request, key.Peer) + : ProjectedSnapshot.Empty; + + foreach (var previous in projection.DeliveredBySourceRecordKey.ToArray()) + { + if (!next.BySourceRecordKey.TryGetValue(previous.Key, out var current)) + { + SendRemoval(key.Peer, key.Id, previous.Value); + continue; + } + if (!string.Equals(previous.Value.RecordKey, current.RecordKey, StringComparison.Ordinal) || + !string.Equals(previous.Value.SchemaId, current.SchemaId, StringComparison.Ordinal)) + { + SendRemoval(key.Peer, key.Id, previous.Value); + SendUpsert(key.Peer, key.Id, current, added: true); + } + else if (!Equivalent(previous.Value, current)) + { + SendUpsert(key.Peer, key.Id, current, added: false); + } + } + foreach (var current in next.BySourceRecordKey) + { + if (!projection.DeliveredBySourceRecordKey.ContainsKey(current.Key)) + SendUpsert(key.Peer, key.Id, current.Value, added: true); + } + + projection.DeliveredBySourceRecordKey.Clear(); + foreach (var current in next.BySourceRecordKey) + projection.DeliveredBySourceRecordKey[current.Key] = current.Value; + + if (projection.DemandActive != authorized) + { + projection.DemandActive = authorized; + PublishDemand(request, key, authorized); + } + } + + private ProjectedSnapshot CreateProjectedSnapshot( + CultNetDatabaseSubscribeMessage request, + ICultNetSchemaServerPeer peer) + { + var snapshot = _database.Documents.CreateRawSnapshotResponse( + _database.Cache, + request.MessageId, + new CultNetSnapshotRequestMessage + { + MessageId = request.MessageId, + SchemaIds = request.SchemaIds, + RecordKeys = request.RecordKeys + }); + var bySourceRecordKey = new Dictionary(StringComparer.Ordinal); + var projectedRecordKeys = new HashSet(StringComparer.Ordinal); + foreach (var source in snapshot.Documents) + { + var sourceRecordKey = source.RecordKey; + if (_authorizeRecord?.Invoke(request, peer, sourceRecordKey, source.SchemaId) == false) + continue; + var projected = _projectRecord == null ? source : _projectRecord(request, peer, source); + if (projected == null) continue; + if (!projectedRecordKeys.Add(projected.RecordKey)) + throw new InvalidOperationException( + $"Database subscription projection produced duplicate record key '{projected.RecordKey}'."); + bySourceRecordKey[sourceRecordKey] = projected; + } + return new ProjectedSnapshot(bySourceRecordKey); + } + + private void ApplyProjectedChange( + SubscriptionProjectionState projection, + string sourceRecordKey, + CultNetRawDocumentRecord? current, + ICultNetSchemaServerPeer peer, + string subscriptionId) + { + projection.DeliveredBySourceRecordKey.TryGetValue(sourceRecordKey, out var previous); + if (current == null) + { + if (previous != null) + { + projection.DeliveredBySourceRecordKey.Remove(sourceRecordKey); + SendRemoval(peer, subscriptionId, previous); + } + return; + } + if (previous == null) + { + projection.DeliveredBySourceRecordKey[sourceRecordKey] = current; + SendUpsert(peer, subscriptionId, current, added: true); + return; + } + if (!string.Equals(previous.RecordKey, current.RecordKey, StringComparison.Ordinal) || + !string.Equals(previous.SchemaId, current.SchemaId, StringComparison.Ordinal)) + { + SendRemoval(peer, subscriptionId, previous); + projection.DeliveredBySourceRecordKey[sourceRecordKey] = current; + SendUpsert(peer, subscriptionId, current, added: true); + return; + } + if (!Equivalent(previous, current)) + { + projection.DeliveredBySourceRecordKey[sourceRecordKey] = current; + SendUpsert(peer, subscriptionId, current, added: false); + } + } + + private static string ResolveChangeRecordKey(object change) + { + var value = change.GetType().GetProperty("Key")?.GetValue(change); + return value is CultRecordKey key ? key.Value : string.Empty; + } + + private static bool Equivalent(CultNetRawDocumentRecord left, CultNetRawDocumentRecord right) => + string.Equals(left.SchemaId, right.SchemaId, StringComparison.Ordinal) && + string.Equals(left.SchemaName, right.SchemaName, StringComparison.Ordinal) && + string.Equals(left.SchemaVersion, right.SchemaVersion, StringComparison.Ordinal) && + string.Equals(left.SchemaContentHash, right.SchemaContentHash, StringComparison.Ordinal) && + string.Equals(left.RecordKey, right.RecordKey, StringComparison.Ordinal) && + string.Equals(left.PayloadEncoding, right.PayloadEncoding, StringComparison.Ordinal) && + left.Payload.SequenceEqual(right.Payload) && + string.Equals(left.SourceRuntimeId, right.SourceRuntimeId, StringComparison.Ordinal) && + string.Equals(left.SourceAgentId, right.SourceAgentId, StringComparison.Ordinal) && + string.Equals(left.SourceRole, right.SourceRole, StringComparison.Ordinal); + + private static void SendUpsert( + ICultNetSchemaServerPeer peer, + string subscriptionId, + CultNetRawDocumentRecord document, + bool added) => + peer.SendCultNet(new CultNetDatabaseChangeRawMessage + { + MessageId = Guid.NewGuid().ToString("N"), + SubscriptionId = subscriptionId, + ChangeKind = added ? "added" : "updated", + Document = document + }); + + private static void SendRemoval( + ICultNetSchemaServerPeer peer, + string subscriptionId, + CultNetRawDocumentRecord document) => + peer.SendCultNet(new CultNetDatabaseChangeRawMessage + { + MessageId = Guid.NewGuid().ToString("N"), + SubscriptionId = subscriptionId, + ChangeKind = "removed", + RecordKey = document.RecordKey, + SchemaId = document.SchemaId + }); + + private void Withdraw(SubscriptionKey key, bool sendRemovals, bool forgetRequest) + { + if (_subscriptions.TryRemove(key, out var subscription)) + subscription.Dispose(); + if (_projections.TryRemove(key, out var projection)) + { + if (sendRemovals) + { + foreach (var delivered in projection.DeliveredBySourceRecordKey.Values) + SendRemoval(key.Peer, key.Id, delivered); + } + if (projection.DemandActive && _requests.TryGetValue(key, out var activeRequest)) + PublishDemand(activeRequest, key, active: false); + } + if (forgetRequest) + _requests.TryRemove(key, out _); + } + + private CultNetDatabaseChangeRawMessage? CreateAuthorizedChange( + object change, + CultNetDatabaseSubscribeMessage request, + string subscriptionId, + ICultNetSchemaServerPeer peer) + { + return CreateChangeCore(change, request, subscriptionId, (recordKey, schemaId) => + _authorizeRecord?.Invoke(request, peer, recordKey, schemaId) != false); + } + private CultNetDatabaseChangeRawMessage? CreateChange( object change, CultNetDatabaseSubscribeMessage request, string subscriptionId) + { + return CreateChangeCore(change, request, subscriptionId, (_, _) => true); + } + + private CultNetDatabaseChangeRawMessage? CreateChangeCore( + object change, + CultNetDatabaseSubscribeMessage request, + string subscriptionId, + Func authorizeRecord) { var changeType = change.GetType(); var key = (CultRecordKey)(changeType.GetProperty("Key")?.GetValue(change) ?? new CultRecordKey("")); @@ -206,6 +458,8 @@ namespace GameCult.Networking var schemaId = ResolveWireSchemaId(previous, (string?)changeType.GetProperty("SchemaId")?.GetValue(change) ?? ""); if (!MatchesRequestedSchema(request.SchemaIds, previous, schemaId)) return null; + if (!authorizeRecord(key.Value, schemaId)) + return null; return new CultNetDatabaseChangeRawMessage { MessageId = Guid.NewGuid().ToString("N"), @@ -219,6 +473,8 @@ namespace GameCult.Networking var raw = CreateRawRecord(key, document); if (!MatchesRequestedSchema(request.SchemaIds, document, raw.SchemaId)) return null; + if (!authorizeRecord(key.Value, raw.SchemaId)) + return null; return new CultNetDatabaseChangeRawMessage { MessageId = Guid.NewGuid().ToString("N"), @@ -258,6 +514,27 @@ namespace GameCult.Networking return ((CultNetDocumentPutRawMessage)put!).Document; } + private sealed class SubscriptionProjectionState + { + public Dictionary DeliveredBySourceRecordKey { get; } = + new(StringComparer.Ordinal); + public bool DemandActive { get; set; } + } + + private sealed class ProjectedSnapshot + { + public ProjectedSnapshot(Dictionary bySourceRecordKey) + { + BySourceRecordKey = bySourceRecordKey; + Documents = bySourceRecordKey.Values.ToArray(); + } + + public static ProjectedSnapshot Empty { get; } = + new(new Dictionary(StringComparer.Ordinal)); + public Dictionary BySourceRecordKey { get; } + public CultNetRawDocumentRecord[] Documents { get; } + } + private sealed class SubscriptionKey { public SubscriptionKey(ICultNetSchemaServerPeer peer, string id) diff --git a/src/GameCult.Networking/CultNetDocumentRegistry.cs b/src/GameCult.Networking/CultNetDocumentRegistry.cs index 8017ad2f..4a5d314e 100644 --- a/src/GameCult.Networking/CultNetDocumentRegistry.cs +++ b/src/GameCult.Networking/CultNetDocumentRegistry.cs @@ -80,7 +80,8 @@ namespace GameCult.Networking Func? payloadDeserializer = null) where T : class { - var descriptor = (registry ?? CultDocumentRegistry.Shared).GetRequired(); + var documents = registry ?? CultDocumentRegistry.Shared; + var descriptor = documents.GetRequired(); return new CultNetDocumentBinding( typeof(T), string.IsNullOrWhiteSpace(schemaId) ? descriptor.SchemaId : schemaId!, @@ -89,11 +90,11 @@ namespace GameCult.Networking var typed = (T)document; return payloadSerializer != null ? payloadSerializer(typed) - : CultDocumentMessagePackSerialization.Serialize(typed); + : CultDocumentMessagePackSerialization.SerializeUntyped(typed, typeof(T), documents); }, payload => payloadDeserializer != null ? payloadDeserializer(payload) - : CultDocumentMessagePackSerialization.Deserialize(payload)); + : (T)CultDocumentMessagePackSerialization.DeserializeUntyped(typeof(T), payload, documents)); } /// @@ -111,7 +112,8 @@ namespace GameCult.Networking throw new ArgumentNullException(nameof(documentType)); } - var descriptor = (registry ?? CultDocumentRegistry.Shared).GetRequired(documentType); + var documents = registry ?? CultDocumentRegistry.Shared; + var descriptor = documents.GetRequired(documentType); return new CultNetDocumentBinding( documentType, string.IsNullOrWhiteSpace(schemaId) ? descriptor.SchemaId : schemaId!, @@ -125,11 +127,11 @@ namespace GameCult.Networking return payloadSerializer != null ? payloadSerializer(document) - : CultDocumentMessagePackSerialization.SerializeUntyped(document, documentType); + : CultDocumentMessagePackSerialization.SerializeUntyped(document, documentType, documents); }, payload => payloadDeserializer != null ? payloadDeserializer(payload) - : CultDocumentMessagePackSerialization.DeserializeUntyped(documentType, payload)); + : CultDocumentMessagePackSerialization.DeserializeUntyped(documentType, payload, documents)); } } @@ -224,6 +226,9 @@ namespace GameCult.Networking Document = new CultNetRawDocumentRecord { SchemaId = binding.SchemaId, + SchemaName = descriptor.SchemaName, + SchemaVersion = descriptor.SchemaVersion, + SchemaContentHash = descriptor.ContentHash, RecordKey = handle.Key.Value, StoredAt = storedAt, PayloadEncoding = "messagepack", @@ -292,6 +297,9 @@ namespace GameCult.Networking documents.Add(new CultNetRawDocumentRecord { SchemaId = binding.SchemaId, + SchemaName = descriptor.SchemaName, + SchemaVersion = descriptor.SchemaVersion, + SchemaContentHash = descriptor.ContentHash, RecordKey = key, StoredAt = storedAt, PayloadEncoding = "messagepack", diff --git a/src/GameCult.Networking/CultNetLocal.cs b/src/GameCult.Networking/CultNetLocal.cs index e884cd8d..87942ce8 100644 --- a/src/GameCult.Networking/CultNetLocal.cs +++ b/src/GameCult.Networking/CultNetLocal.cs @@ -100,7 +100,7 @@ namespace GameCult.Networking /// public Task FlushAsync(bool soft = false) { - return Cache.FlushAsync(soft); + return Cache.FlushAsync(); } /// diff --git a/src/GameCult.Networking/CultNetOperationServer.cs b/src/GameCult.Networking/CultNetOperationServer.cs new file mode 100644 index 00000000..27b52c55 --- /dev/null +++ b/src/GameCult.Networking/CultNetOperationServer.cs @@ -0,0 +1,358 @@ +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using MessagePack; + +namespace GameCult.Networking +{ + /// Portable failure payload for operation routing and envelope validation. + [MessagePackObject] + public sealed class CultNetOperationFailure + { + /// Gets or sets the stable machine-readable failure code. + [Key("code")] public string Code { get; set; } = "operation-error"; + + /// Gets or sets the human-readable failure description. + [Key("message")] public string Message { get; set; } = "CultNet operation failed."; + } + + /// + /// Typed application input delivered through one CultNet operation envelope. + /// The message id is the caller's idempotency key; the application transaction + /// remains responsible for applying it exactly once. + /// + public sealed class CultNetOperationContext + { + internal CultNetOperationContext(CultNetOperationRequestMessage envelope, TRequest value) + { + Envelope = envelope ?? throw new ArgumentNullException(nameof(envelope)); + Value = value; + } + + /// Gets the decoded application request. + public TRequest Value { get; } + + /// Gets the durable caller-supplied idempotency key. + public string IdempotencyKey => Envelope.MessageId; + + /// Gets the calling runtime identity, when supplied. + public string? SourceRuntimeId => Envelope.SourceRuntimeId; + + /// Gets the optional target runtime identity. + public string? TargetRuntimeId => Envelope.TargetRuntimeId; + + /// Gets the original transport-neutral envelope for diagnostics. + public CultNetOperationRequestMessage Envelope { get; } + } + + /// Provider-authored typed result for one CultNet operation. + public sealed class CultNetOperationReply + { + /// Creates one provider-authored typed reply. + public CultNetOperationReply( + string status, + TResponse value, + IEnumerable? diagnostics = null) + { + Status = RequireNonEmpty(status, nameof(status)); + Value = value; + Diagnostics = diagnostics == null ? Array.Empty() : new List(diagnostics).ToArray(); + } + + /// Gets the provider-authored operation status. + public string Status { get; } + + /// Gets the typed response payload. + public TResponse Value { get; } + + /// Gets provider-authored diagnostics. + public IReadOnlyList Diagnostics { get; } + + /// Creates an accepted typed reply. + public static CultNetOperationReply Accepted(TResponse value) => new("accepted", value); + + /// Creates a rejected typed reply whose payload remains schema-valid. + public static CultNetOperationReply Rejected( + TResponse value, + params string[] diagnostics) => new("rejected", value, diagnostics); + + private static string RequireNonEmpty(string value, string paramName) => + string.IsNullOrWhiteSpace(value) + ? throw new ArgumentException("Value must be non-empty.", paramName) + : value.Trim(); + } + + /// + /// Transport-neutral typed dispatcher for CultNet application operations. + /// It owns envelope validation and serialization only; registered handlers own + /// domain validation, idempotency, state mutation, and durable receipts. + /// + public sealed class CultNetOperationServer : IDisposable + { + /// Schema used by correlated framework-level operation failures. + public const string FailureSchemaId = "gamecult.cultnet.operation_failure.v1"; + + private const string PayloadEncoding = "messagepack-base64"; + private readonly ICultNetSchemaServer _server; + private readonly string? _sourceRuntimeId; + private readonly Dictionary _bindings = new(); + private readonly Func _handler; + private bool _disposed; + + /// Attaches one typed operation dispatcher to a schema server. + public CultNetOperationServer(ICultNetSchemaServer server, string? sourceRuntimeId = null) + { + _server = server ?? throw new ArgumentNullException(nameof(server)); + _sourceRuntimeId = string.IsNullOrWhiteSpace(sourceRuntimeId) ? null : sourceRuntimeId!.Trim(); + _handler = HandleAsync; + _server.OnCultNet(_handler); + } + + /// + /// Registers one typed operation. Duplicate service/operation routes are rejected. + /// + public CultNetOperationServer Register( + string serviceId, + string operation, + string requestSchema, + string responseSchema, + Func, Task>> handler) + { + ThrowIfDisposed(); + var route = new Route(serviceId, operation); + if (_bindings.ContainsKey(route)) + throw new InvalidOperationException($"CultNet operation '{route}' is already registered."); + _bindings.Add(route, new Binding( + route, + requestSchema, + responseSchema, + handler ?? throw new ArgumentNullException(nameof(handler)), + _sourceRuntimeId)); + return this; + } + + /// + /// Registers one typed operation whose successful handler result is always accepted. + /// + public CultNetOperationServer Register( + string serviceId, + string operation, + string requestSchema, + string responseSchema, + Func, Task> handler) + { + if (handler == null) throw new ArgumentNullException(nameof(handler)); + return Register( + serviceId, + operation, + requestSchema, + responseSchema, + async context => CultNetOperationReply.Accepted(await handler(context).ConfigureAwait(false))); + } + + /// + public void Dispose() + { + if (_disposed) return; + _disposed = true; + _server.RemoveCultNetMessageListener(_handler); + _bindings.Clear(); + } + + private async Task HandleAsync(CultNetOperationRequestMessage request, ICultNetSchemaServerPeer peer) + { + if (_disposed) return; + if (request == null) throw new ArgumentNullException(nameof(request)); + if (peer == null) throw new ArgumentNullException(nameof(peer)); + if (_sourceRuntimeId != null && + !string.Equals(request.TargetRuntimeId, _sourceRuntimeId, StringComparison.Ordinal)) + { + peer.SendCultNet(Failure( + request, + "denied", + "target-runtime-mismatch", + $"Operation targeted runtime '{request.TargetRuntimeId ?? "(missing)"}', not '{_sourceRuntimeId}'.")); + return; + } + + Route route; + try + { + route = new Route(request.ServiceId, request.Operation); + } + catch (ArgumentException error) + { + peer.SendCultNet(Failure(request, "invalid", "invalid-route", error.Message)); + return; + } + + if (!_bindings.TryGetValue(route, out var binding)) + { + peer.SendCultNet(Failure( + request, + "unsupported", + "operation-not-registered", + $"CultNet operation '{route}' is not registered.")); + return; + } + + try + { + peer.SendCultNet(await binding.InvokeAsync(request).ConfigureAwait(false)); + } + catch (RequestException error) + { + peer.SendCultNet(Failure(request, "invalid", error.Code, error.Message)); + } + catch (Exception error) + { + peer.SendCultNet(Failure( + request, + "error", + "handler-failed", + $"CultNet operation '{route}' failed: {error.Message}")); + } + } + + private CultNetOperationResponseMessage Failure( + CultNetOperationRequestMessage request, + string status, + string code, + string message) + { + return new CultNetOperationResponseMessage + { + MessageId = request.MessageId ?? "", + ServiceId = request.ServiceId ?? "", + Operation = request.Operation ?? "", + Status = status, + PayloadSchema = FailureSchemaId, + PayloadEncoding = PayloadEncoding, + Payload = Convert.ToBase64String(MessagePackSerializer.Serialize( + new CultNetOperationFailure { Code = code, Message = message }, + CultNetSchemaMessageSerialization.Options)), + Diagnostics = new[] { message }, + SourceRuntimeId = _sourceRuntimeId + }; + } + + private void ThrowIfDisposed() + { + if (_disposed) throw new ObjectDisposedException(nameof(CultNetOperationServer)); + } + + private interface IBinding + { + Task InvokeAsync(CultNetOperationRequestMessage request); + } + + private sealed class Binding : IBinding + { + private readonly Route _route; + private readonly string _requestSchema; + private readonly string _responseSchema; + private readonly Func, Task>> _handler; + private readonly string? _sourceRuntimeId; + + public Binding( + Route route, + string requestSchema, + string responseSchema, + Func, Task>> handler, + string? sourceRuntimeId) + { + _route = route; + _requestSchema = RequireNonEmpty(requestSchema, nameof(requestSchema)); + _responseSchema = RequireNonEmpty(responseSchema, nameof(responseSchema)); + _handler = handler; + _sourceRuntimeId = sourceRuntimeId; + } + + public async Task InvokeAsync(CultNetOperationRequestMessage request) + { + if (string.IsNullOrWhiteSpace(request.MessageId)) + throw new RequestException("missing-idempotency-key", "CultNet operation omitted its idempotency key."); + if (!string.Equals(request.PayloadSchema, _requestSchema, StringComparison.Ordinal)) + throw new RequestException( + "request-schema-mismatch", + $"CultNet operation '{_route}' expected payload schema '{_requestSchema}', got '{request.PayloadSchema}'."); + if (!string.Equals(request.PayloadEncoding, PayloadEncoding, StringComparison.Ordinal)) + throw new RequestException( + "request-encoding-mismatch", + $"CultNet operation '{_route}' expected payload encoding '{PayloadEncoding}', got '{request.PayloadEncoding}'."); + + TRequest value; + try + { + value = MessagePackSerializer.Deserialize( + Convert.FromBase64String(request.Payload), + CultNetSchemaMessageSerialization.Options); + } + catch (FormatException error) + { + throw new RequestException("request-payload-invalid", "CultNet operation payload was not valid base64.", error); + } + catch (MessagePackSerializationException error) + { + throw new RequestException("request-payload-invalid", "CultNet operation payload was not valid MessagePack.", error); + } + if (value == null) throw new RequestException("request-payload-null", "CultNet operation payload decoded to null."); + + var reply = await _handler(new CultNetOperationContext(request, value)).ConfigureAwait(false) + ?? throw new InvalidOperationException("CultNet operation handler returned no reply."); + if (reply.Value == null) throw new InvalidOperationException("CultNet operation handler returned a null payload."); + return new CultNetOperationResponseMessage + { + MessageId = request.MessageId, + ServiceId = _route.ServiceId, + Operation = _route.Operation, + Status = reply.Status, + PayloadSchema = _responseSchema, + PayloadEncoding = PayloadEncoding, + Payload = Convert.ToBase64String(MessagePackSerializer.Serialize( + reply.Value, + CultNetSchemaMessageSerialization.Options)), + Diagnostics = new List(reply.Diagnostics).ToArray(), + SourceRuntimeId = _sourceRuntimeId + }; + } + } + + private sealed class RequestException : Exception + { + public RequestException(string code, string message, Exception? inner = null) + : base(message, inner) + { + Code = code; + } + + public string Code { get; } + } + + private readonly struct Route : IEquatable + { + public Route(string serviceId, string operation) + { + ServiceId = RequireNonEmpty(serviceId, nameof(serviceId)); + Operation = RequireNonEmpty(operation, nameof(operation)); + } + + public string ServiceId { get; } + public string Operation { get; } + + public bool Equals(Route other) => + string.Equals(ServiceId, other.ServiceId, StringComparison.Ordinal) && + string.Equals(Operation, other.Operation, StringComparison.Ordinal); + + public override bool Equals(object? value) => value is Route route && Equals(route); + public override int GetHashCode() => + (StringComparer.Ordinal.GetHashCode(ServiceId) * 397) ^ StringComparer.Ordinal.GetHashCode(Operation); + public override string ToString() => ServiceId + "/" + Operation; + } + + private static string RequireNonEmpty(string value, string paramName) => + string.IsNullOrWhiteSpace(value) + ? throw new ArgumentException("Value must be non-empty.", paramName) + : value.Trim(); + } +} diff --git a/src/GameCult.Networking/CultNetSchemaClient.cs b/src/GameCult.Networking/CultNetSchemaClient.cs index 4071cd1b..5c6117aa 100644 --- a/src/GameCult.Networking/CultNetSchemaClient.cs +++ b/src/GameCult.Networking/CultNetSchemaClient.cs @@ -38,6 +38,17 @@ namespace GameCult.Networking where T : ICultNetSchemaMessage; } + /// + /// Schema client that connects to one exact advertised URI, including its path. + /// Host/port-only connection is retained for legacy transports; rendezvous and + /// route-aware clients should prefer this contract when available. + /// + public interface ICultNetUriSchemaClient : ICultNetSchemaClient + { + /// Connects to the exact advertised schema endpoint. + Task ConnectAsync(Uri endpoint, CancellationToken cancellationToken = default); + } + /// /// Exposes terminal failure of a schema client's background transport work. /// diff --git a/src/GameCult.Networking/CultNetSchemaMessageSerialization.cs b/src/GameCult.Networking/CultNetSchemaMessageSerialization.cs index 386bc99b..e712beef 100644 --- a/src/GameCult.Networking/CultNetSchemaMessageSerialization.cs +++ b/src/GameCult.Networking/CultNetSchemaMessageSerialization.cs @@ -72,6 +72,8 @@ namespace GameCult.Networking CultNetSchemaVersions.SimulationConsensusCandidate => Deserialize(payload), CultNetSchemaVersions.VerseCatalogRequest => Deserialize(payload), CultNetSchemaVersions.VerseCatalogResponse => Deserialize(payload), + CultNetSchemaVersions.SessionOpen => Deserialize(payload), + CultNetSchemaVersions.SessionAccepted => Deserialize(payload), CultNetSchemaVersions.PeerExchangeRequest => Deserialize(payload), CultNetSchemaVersions.PeerExchangeResponse => Deserialize(payload), CultNetSchemaVersions.ContentChunkRequest => Deserialize(payload), diff --git a/src/GameCult.Networking/CultNetSchemaMessages.cs b/src/GameCult.Networking/CultNetSchemaMessages.cs index 9a8800fc..56370207 100644 --- a/src/GameCult.Networking/CultNetSchemaMessages.cs +++ b/src/GameCult.Networking/CultNetSchemaMessages.cs @@ -135,6 +135,10 @@ namespace GameCult.Networking /// CultMesh Verse catalog response contract identifier. /// public const string VerseCatalogResponse = "cultmesh.verse_catalog_response.v0"; + /// CultMesh session identity request contract identifier. + public const string SessionOpen = "cultmesh.session_open.v2"; + /// CultMesh verified session identity response contract identifier. + public const string SessionAccepted = "cultmesh.session_accepted.v2"; /// /// CultMesh peer exchange request contract identifier. /// @@ -608,6 +612,18 @@ namespace GameCult.Networking /// [Key("schemaId")] public string SchemaId { get; set; } = string.Empty; /// + /// Gets or sets the stable schema family name when the publisher has a typed descriptor. + /// + [Key("schemaName")] public string? SchemaName { get; set; } + /// + /// Gets or sets the portable versioned schema contract when the publisher has a typed descriptor. + /// + [Key("schemaVersion")] public string? SchemaVersion { get; set; } + /// + /// Gets or sets the canonical schema content hash when the publisher has a typed descriptor. + /// + [Key("schemaContentHash")] public string? SchemaContentHash { get; set; } + /// /// Gets or sets the record key. /// [Key("recordKey")] public string RecordKey { get; set; } = string.Empty; @@ -1167,6 +1183,37 @@ namespace GameCult.Networking } } + /// Requests an identity-bound CultMesh session before application traffic begins. + [MessagePackObject] + public class CultMeshSessionOpenMessage : ICultNetSchemaMessage + { + [Key("schemaVersion")] public string SchemaVersion { get; set; } = CultNetSchemaVersions.SessionOpen; + [Key("messageId")] public string MessageId { get; set; } = string.Empty; + [Key("sourceRuntimeId")] public string SourceRuntimeId { get; set; } = string.Empty; + [Key("verseId")] public string VerseId { get; set; } = string.Empty; + [Key("authorityRuntimeId")] public string AuthorityRuntimeId { get; set; } = string.Empty; + [Key("protocolId")] public string ProtocolId { get; set; } = string.Empty; + [Key("routeGeneration")] public string RouteGeneration { get; set; } = string.Empty; + [Key("clientNonce")] public string ClientNonce { get; set; } = string.Empty; + } + + /// Accepts or rejects a CultMesh session with the server's actual identity. + [MessagePackObject] + public class CultMeshSessionAcceptedMessage : ICultNetSchemaMessage + { + [Key("schemaVersion")] public string SchemaVersion { get; set; } = CultNetSchemaVersions.SessionAccepted; + [Key("messageId")] public string MessageId { get; set; } = string.Empty; + [Key("accepted")] public bool Accepted { get; set; } + [Key("verseId")] public string VerseId { get; set; } = string.Empty; + [Key("authorityRuntimeId")] public string AuthorityRuntimeId { get; set; } = string.Empty; + [Key("protocolId")] public string ProtocolId { get; set; } = string.Empty; + [Key("routeGeneration")] public string RouteGeneration { get; set; } = string.Empty; + [Key("clientNonce")] public string ClientNonce { get; set; } = string.Empty; + [Key("providerKeyId")] public string ProviderKeyId { get; set; } = string.Empty; + [Key("providerSignature")] public string ProviderSignature { get; set; } = string.Empty; + [Key("error")] public string? Error { get; set; } + } + /// /// Requests known CultMesh Verses from a peer. /// @@ -1242,6 +1289,11 @@ namespace GameCult.Networking /// [Key("authorityRuntimeIds")] public string[] AuthorityRuntimeIds { get; set; } = Array.Empty(); /// + /// Gets or sets exact authoritative-runtime route bindings. This is the + /// routing authority; the parallel legacy arrays are projections only. + /// + [Key("authorityRoutes")] public CultMeshAuthorityRouteMessage[] AuthorityRoutes { get; set; } = Array.Empty(); + /// /// Gets or sets the parent Verse id for overlays or branches. /// [Key("parentVerseId")] public string? ParentVerseId { get; set; } @@ -1251,6 +1303,30 @@ namespace GameCult.Networking [Key("description")] public string? Description { get; set; } } + /// Wire representation of one authority-bound CultMesh route. + [MessagePackObject] + public class CultMeshAuthorityRouteMessage + { + [Key("authorityRuntimeId")] public string AuthorityRuntimeId { get; set; } = string.Empty; + [Key("endpoint")] public string Endpoint { get; set; } = string.Empty; + [Key("protocolIds")] public string[] ProtocolIds { get; set; } = Array.Empty(); + [Key("priority")] public int Priority { get; set; } + [Key("generation")] public string Generation { get; set; } = string.Empty; + [Key("certificate")] public CultMeshRouteCertificateMessage? Certificate { get; set; } + } + + [MessagePackObject] + public class CultMeshRouteCertificateMessage + { + [Key("providerKeyId")] public string ProviderKeyId { get; set; } = string.Empty; + [Key("providerPublicKeyX")] public string ProviderPublicKeyX { get; set; } = string.Empty; + [Key("providerPublicKeyY")] public string ProviderPublicKeyY { get; set; } = string.Empty; + [Key("odinKeyId")] public string OdinKeyId { get; set; } = string.Empty; + [Key("issuedAtUnixMilliseconds")] public long IssuedAtUnixMilliseconds { get; set; } + [Key("expiresAtUnixMilliseconds")] public long ExpiresAtUnixMilliseconds { get; set; } + [Key("signature")] public string Signature { get; set; } = string.Empty; + } + /// /// Wire representation of CultMesh Verse compatibility. /// diff --git a/src/GameCult.Networking/CultNetSchemaRegistry.cs b/src/GameCult.Networking/CultNetSchemaRegistry.cs index 037c0d84..4fc68a13 100644 --- a/src/GameCult.Networking/CultNetSchemaRegistry.cs +++ b/src/GameCult.Networking/CultNetSchemaRegistry.cs @@ -370,6 +370,27 @@ namespace GameCult.Networking WireContracts = new[] { CultNetWireContracts.SchemaV0 } }, new SchemaResourceSpec + { + FileName = "cultnet.database-subscribe.schema.json", + Kind = "wire_message", + SchemaVersion = CultNetSchemaVersions.DatabaseSubscribe, + WireContracts = new[] { CultNetWireContracts.SchemaV0 } + }, + new SchemaResourceSpec + { + FileName = "cultnet.database-unsubscribe.schema.json", + Kind = "wire_message", + SchemaVersion = CultNetSchemaVersions.DatabaseUnsubscribe, + WireContracts = new[] { CultNetWireContracts.SchemaV0 } + }, + new SchemaResourceSpec + { + FileName = "cultnet.database-change-raw.schema.json", + Kind = "wire_message", + SchemaVersion = CultNetSchemaVersions.DatabaseChangeRaw, + WireContracts = new[] { CultNetWireContracts.SchemaV0 } + }, + new SchemaResourceSpec { FileName = "cultnet.schema-descriptor.schema.json", Kind = "shared_contract", diff --git a/src/GameCult.Networking/CultNetSchemaServerGroup.cs b/src/GameCult.Networking/CultNetSchemaServerGroup.cs new file mode 100644 index 00000000..91a8484d --- /dev/null +++ b/src/GameCult.Networking/CultNetSchemaServerGroup.cs @@ -0,0 +1,97 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; + +namespace GameCult.Networking +{ + /// + /// Presents one logical CultNet schema server over several physical transports. + /// It owns handler fan-out and peer-lifecycle aggregation, but never owns or + /// disposes the transport servers themselves. + /// + public sealed class CultNetSchemaServerGroup : + ICultNetSchemaServer, + ICultNetSchemaServerPeerLifecycle, + IDisposable + { + private readonly IReadOnlyList _servers; + private readonly IReadOnlyList _lifecycles; + private bool _disposed; + + /// Creates one logical server from two or more transport servers. + public CultNetSchemaServerGroup(params ICultNetSchemaServer[] servers) + { + if (servers == null) throw new ArgumentNullException(nameof(servers)); + if (servers.Length < 2) + throw new ArgumentException("A CultNet schema server group requires at least two transports.", nameof(servers)); + if (servers.Any(server => server == null)) + throw new ArgumentException("A CultNet schema server group cannot contain a null transport.", nameof(servers)); + for (var left = 0; left < servers.Length; left++) + for (var right = left + 1; right < servers.Length; right++) + { + if (ReferenceEquals(servers[left], servers[right])) + throw new ArgumentException("A CultNet schema server group cannot contain the same transport twice.", nameof(servers)); + } + + _servers = Array.AsReadOnly(servers.ToArray()); + _lifecycles = Array.AsReadOnly(servers.OfType().ToArray()); + foreach (var lifecycle in _lifecycles) lifecycle.PeerDisconnected += ForwardPeerDisconnected; + } + + /// + public event Action? PeerDisconnected; + + /// + public void OnCultNet(Func callback) + where TMessage : ICultNetSchemaMessage + { + if (callback == null) throw new ArgumentNullException(nameof(callback)); + ThrowIfDisposed(); + var registered = 0; + try + { + foreach (var server in _servers) + { + server.OnCultNet(callback); + registered++; + } + } + catch + { + for (var index = 0; index < registered; index++) + _servers[index].RemoveCultNetMessageListener(callback); + throw; + } + } + + /// + public void RemoveCultNetMessageListener(Delegate callback) + where TMessage : ICultNetSchemaMessage + { + if (callback == null) throw new ArgumentNullException(nameof(callback)); + if (_disposed) return; + foreach (var server in _servers) server.RemoveCultNetMessageListener(callback); + } + + /// + /// Detaches lifecycle forwarding. The physical transports retain their + /// independent ownership and must be disposed by their creator. + /// + public void Dispose() + { + if (_disposed) return; + _disposed = true; + foreach (var lifecycle in _lifecycles) lifecycle.PeerDisconnected -= ForwardPeerDisconnected; + PeerDisconnected = null; + } + + private void ForwardPeerDisconnected(ICultNetSchemaServerPeer peer) => + PeerDisconnected?.Invoke(peer); + + private void ThrowIfDisposed() + { + if (_disposed) throw new ObjectDisposedException(nameof(CultNetSchemaServerGroup)); + } + } +} diff --git a/src/GameCult.Networking/CultNetTransport.cs b/src/GameCult.Networking/CultNetTransport.cs index fc72b2d4..f549f735 100644 --- a/src/GameCult.Networking/CultNetTransport.cs +++ b/src/GameCult.Networking/CultNetTransport.cs @@ -723,6 +723,10 @@ namespace GameCult.Networking /// public IReadOnlyList Delivered { get; set; } = Array.Empty(); /// + /// Gets or sets queued reliable packets admitted by newly received acknowledgements. + /// + public IReadOnlyList ReadyToSend { get; set; } = Array.Empty(); + /// /// Gets or sets an optional immediate reply packet. /// public CultNetRudpPacket? Reply { get; set; } @@ -865,6 +869,9 @@ namespace GameCult.Networking /// public sealed class CultNetRudpSession { + /// Maximum reliable packets admitted to the wire before acknowledgements advance the window. + public const int ReliableSendWindowPackets = 32; + private const int ReceivedSequenceWindow = 4096; private sealed class PendingReliablePacket { public CultNetRudpPacket Packet { get; set; } = new CultNetRudpPacket(); @@ -887,14 +894,17 @@ namespace GameCult.Networking } private uint _nextSequence; + private readonly Dictionary _nextSequencedByChannel = new Dictionary(StringComparer.Ordinal); private ushort _nextFragmentId = 1; private readonly int? _maxPendingReliablePackets; private volatile bool _connected; private long? _lastReceivedAtMs; private uint? _highestReceivedSequence; private readonly HashSet _receivedSequences = new HashSet(); + private readonly Dictionary _latestSequencedByChannel = new Dictionary(StringComparer.Ordinal); private readonly object _pendingReliableGate = new object(); private readonly Dictionary _pendingReliable = new Dictionary(); + private readonly Queue _queuedReliable = new Queue(); private readonly Dictionary _orderedNextSequenceByChannel = new Dictionary(StringComparer.Ordinal); private readonly Dictionary> _orderedBuffers = new Dictionary>(StringComparer.Ordinal); @@ -947,6 +957,18 @@ namespace GameCult.Networking } } + /// Gets the number of reliable packets waiting for admission to the wire. + public int QueuedReliablePacketCount + { + get { lock (_pendingReliableGate) return _queuedReliable.Count; } + } + + /// Gets all reliable packets that have not yet been acknowledged. + public int OutstandingReliablePacketCount + { + get { lock (_pendingReliableGate) return _pendingReliable.Count + _queuedReliable.Count; } + } + /// /// Creates a reliable ordered connect packet. /// @@ -982,6 +1004,12 @@ namespace GameCult.Networking /// public CultNetRudpPacket Send(string channelId, byte[] payload, CultNetRudpSendOptions? options = null) { + options ??= new CultNetRudpSendOptions(); + lock (_pendingReliableGate) + { + if (options.Reliable && _pendingReliable.Count >= ReliableSendWindowPackets) + throw new InvalidOperationException("RUDP reliable send window is full; receive acknowledgements before sending."); + } return SendMany(channelId, payload, options).First(); } @@ -996,6 +1024,10 @@ namespace GameCult.Networking } options ??= new CultNetRudpSendOptions(); + if (options.Ordered && !options.Reliable) + { + throw new InvalidOperationException("RUDP ordered delivery requires reliability."); + } payload ??= Array.Empty(); if (maxFragmentBytes.HasValue && maxFragmentBytes.Value <= 0) { @@ -1029,13 +1061,11 @@ namespace GameCult.Networking fragmentId, (ushort)index, (ushort)fragmentCount); - if (fragmentPacket.Reliable) - { - TrackReliable(fragmentPacket, options.NowMs); - } packets.Add(fragmentPacket); } - return packets; + return options.Reliable + ? AdmitReliablePackets(packets, options.NowMs) + : packets; } EnsureReliableCapacity(options.Reliable ? 1 : 0); @@ -1048,7 +1078,7 @@ namespace GameCult.Networking options.Sequenced); if (packet.Reliable) { - TrackReliable(packet, options.NowMs); + return AdmitReliablePackets(new[] { packet }, options.NowMs); } return new[] { packet }; } @@ -1060,6 +1090,7 @@ namespace GameCult.Networking { RequireConnection(packet); ApplyAcknowledgements(packet); + var readyToSend = PromoteQueuedReliable(nowMs); _lastReceivedAtMs = nowMs; var expectedSequenceIfUninitialized = _highestReceivedSequence.HasValue ? _highestReceivedSequence.Value + 1 @@ -1069,14 +1100,14 @@ namespace GameCult.Networking { RememberReceived(packet.Sequence); _connected = true; - return new CultNetRudpReceiveResult(); + return new CultNetRudpReceiveResult { ReadyToSend = readyToSend }; } if (packet.PacketType == CultNetRudpPacketType.Ping) { - RememberReceived(packet.Sequence); return new CultNetRudpReceiveResult { + ReadyToSend = readyToSend, Reply = CreatePacket( CultNetRudpPacketType.Pong, "control", @@ -1089,11 +1120,10 @@ namespace GameCult.Networking if (packet.PacketType == CultNetRudpPacketType.Ack || packet.PacketType == CultNetRudpPacketType.Pong) { - if (packet.PacketType == CultNetRudpPacketType.Pong) - RememberReceived(packet.Sequence); return new CultNetRudpReceiveResult { - Delivered = DrainUnblockedOrdered(), + Delivered = Array.Empty(), + ReadyToSend = readyToSend, Pong = packet.PacketType == CultNetRudpPacketType.Pong, PongPayload = packet.PacketType == CultNetRudpPacketType.Pong ? packet.Payload ?? Array.Empty() @@ -1103,10 +1133,10 @@ namespace GameCult.Networking if (packet.PacketType == CultNetRudpPacketType.Disconnect) { - RememberReceived(packet.Sequence); _connected = false; return new CultNetRudpReceiveResult { + ReadyToSend = readyToSend, Disconnected = true, DisconnectReason = packet.Payload ?? Array.Empty() }; @@ -1114,24 +1144,43 @@ namespace GameCult.Networking if (packet.PacketType != CultNetRudpPacketType.Data) { - return new CultNetRudpReceiveResult(); + return new CultNetRudpReceiveResult { ReadyToSend = readyToSend }; } - var duplicate = _receivedSequences.Contains(packet.Sequence); - RememberReceived(packet.Sequence); + var duplicate = packet.Reliable + && (_receivedSequences.Contains(packet.Sequence) + || (_highestReceivedSequence.HasValue + && packet.Sequence < _highestReceivedSequence.Value + && _highestReceivedSequence.Value - packet.Sequence >= ReceivedSequenceWindow)); + if (packet.Reliable) + { + RememberReceived(packet.Sequence); + } if (duplicate) { - return new CultNetRudpReceiveResult(); + return new CultNetRudpReceiveResult { ReadyToSend = readyToSend }; } var reassembled = Reassemble(packet); if (reassembled == null) { - return new CultNetRudpReceiveResult(); + return new CultNetRudpReceiveResult { ReadyToSend = readyToSend }; + } + + if (!reassembled.Ordered && packet.Sequenced) + { + var newestSequence = reassembled.NextSequence - 1; + if (_latestSequencedByChannel.TryGetValue(reassembled.Frame.ChannelId, out var latestSequence) + && newestSequence <= latestSequence) + { + return new CultNetRudpReceiveResult { ReadyToSend = readyToSend }; + } + _latestSequencedByChannel[reassembled.Frame.ChannelId] = newestSequence; } return new CultNetRudpReceiveResult { + ReadyToSend = readyToSend, Delivered = reassembled.Ordered ? DeliverOrdered(reassembled.Frame, reassembled.NextSequence, expectedSequenceIfUninitialized) : new[] { reassembled.Frame } @@ -1161,6 +1210,18 @@ namespace GameCult.Networking }; } + /// + /// Creates a cumulative acknowledgement when the received packet is inside the + /// acknowledgement horizon, or an exact acknowledgement for an older retransmit. + /// + public CultNetRudpPacket CreateAckForReceived(uint receivedSequence) + { + var (ack, _) = AckState(); + return ack >= receivedSequence && ack - receivedSequence <= 32 + ? CreateAck() + : CreateAck(receivedSequence); + } + /// /// Creates a packet carrying a keepalive ping payload. /// @@ -1238,8 +1299,24 @@ namespace GameCult.Networking ushort fragmentIndex, ushort fragmentCount) { - var sequence = _nextSequence; - _nextSequence = checked(_nextSequence + 1); + uint sequence; + if (reliable) + { + sequence = _nextSequence; + _nextSequence = checked(_nextSequence + 1); + } + else if (sequenced) + { + if (!_nextSequencedByChannel.TryGetValue(channelId, out sequence)) + { + sequence = 1; + } + _nextSequencedByChannel[channelId] = checked(sequence + 1); + } + else + { + sequence = 0; + } var (ack, ackMask) = AckState(); return new CultNetRudpPacket { @@ -1271,6 +1348,54 @@ namespace GameCult.Networking } } + private IReadOnlyList AdmitReliablePackets( + IReadOnlyList packets, + long nowMs) + { + lock (_pendingReliableGate) + { + var available = Math.Max(0, ReliableSendWindowPackets - _pendingReliable.Count); + var ready = new List(Math.Min(available, packets.Count)); + foreach (var packet in packets) + { + if (ready.Count < available) + { + _pendingReliable[packet.Sequence] = new PendingReliablePacket + { + Packet = ClonePacket(packet), + LastSentAtMs = nowMs + }; + ready.Add(packet); + } + else + { + _queuedReliable.Enqueue(ClonePacket(packet)); + } + } + return ready; + } + } + + private IReadOnlyList PromoteQueuedReliable(long nowMs) + { + lock (_pendingReliableGate) + { + var available = Math.Max(0, ReliableSendWindowPackets - _pendingReliable.Count); + var ready = new List(Math.Min(available, _queuedReliable.Count)); + while (ready.Count < available && _queuedReliable.Count > 0) + { + var packet = _queuedReliable.Dequeue(); + _pendingReliable[packet.Sequence] = new PendingReliablePacket + { + Packet = ClonePacket(packet), + LastSentAtMs = nowMs + }; + ready.Add(packet); + } + return ready; + } + } + private void EnsureReliableCapacity(int packetCount) { if (packetCount == 0 || !_maxPendingReliablePackets.HasValue) @@ -1280,7 +1405,7 @@ namespace GameCult.Networking lock (_pendingReliableGate) { - if (_pendingReliable.Count + packetCount > _maxPendingReliablePackets.Value) + if (_pendingReliable.Count + _queuedReliable.Count + packetCount > _maxPendingReliablePackets.Value) { throw new InvalidOperationException("RUDP reliable send queue is full."); } @@ -1309,6 +1434,13 @@ namespace GameCult.Networking { _highestReceivedSequence = sequence; } + if (_receivedSequences.Count > ReceivedSequenceWindow) + { + var keepFrom = (_highestReceivedSequence ?? sequence) >= ReceivedSequenceWindow - 1 + ? (_highestReceivedSequence ?? sequence) - (uint)(ReceivedSequenceWindow - 1) + : 0; + _receivedSequences.RemoveWhere(received => received < keepFrom); + } } private (uint Ack, uint AckMask) AckState() @@ -1464,17 +1596,6 @@ namespace GameCult.Networking return delivered; } - private IReadOnlyList DrainUnblockedOrdered() - { - var delivered = new List(); - foreach (var channelId in _orderedBuffers.Keys.ToArray()) - { - SkipReceivedNonChannelSequences(channelId); - delivered.AddRange(DrainOrdered(channelId)); - } - return delivered; - } - private void SkipReceivedNonChannelSequences(string channelId) { if (!_orderedNextSequenceByChannel.TryGetValue(channelId, out var next)) @@ -1698,6 +1819,46 @@ namespace GameCult.Networking _stats.FramesSent++; } + /// + /// Pumps acknowledgements and resends until every queued reliable packet is acknowledged. + /// + public void FlushReliable(TimeSpan? timeout = null) + { + var deadline = DateTimeOffset.UtcNow.Add(timeout ?? TimeSpan.FromSeconds(30)); + var preserved = new List(); + while (_deliveredFrames.Count > 0) + preserved.Add(_deliveredFrames.Dequeue()); + try + { + while (true) + { + lock (_sessionGate) + { + if (_session.OutstandingReliablePacketCount == 0) + return; + } + if (DateTimeOffset.UtcNow >= deadline) + { + int outstanding; + lock (_sessionGate) + outstanding = _session.OutstandingReliablePacketCount; + throw new TimeoutException($"RUDP reliable flush timed out with {outstanding} packets outstanding."); + } + if (TryReceiveOnce(out var delivered) && delivered != null) + preserved.Add(delivered); + PollResends(); + Thread.Sleep(5); + } + } + finally + { + while (_deliveredFrames.Count > 0) + preserved.Add(_deliveredFrames.Dequeue()); + foreach (var frame in preserved) + _deliveredFrames.Enqueue(frame); + } + } + /// /// Sends a reliable ordered schema-channel payload. /// @@ -1947,9 +2108,10 @@ namespace GameCult.Networking result = _session.Receive(packet, NowMs()); if (result.Reply != null) SendPacket(result.Reply); + SendPackets(result.ReadyToSend); if (packet.PacketType == CultNetRudpPacketType.Accept || packet.PacketType == CultNetRudpPacketType.Data) - acknowledgement = _session.CreateAck(packet.Sequence); + acknowledgement = _session.CreateAckForReceived(packet.Sequence); } if (result.Pong) { @@ -2126,7 +2288,7 @@ namespace GameCult.Networking /// /// Gets the number of reliable packets still awaiting acknowledgement. /// - public int PendingReliablePacketCount { get { lock (SessionGate) return Session.PendingReliableSequences.Count; } } + public int PendingReliablePacketCount { get { lock (SessionGate) return Session.OutstandingReliablePacketCount; } } /// /// Gets the last transport-level remote disconnect reason, if one was received. /// @@ -2356,8 +2518,9 @@ namespace GameCult.Networking result = existingPeer.Session.Receive(packet, NowMs()); if (result.Reply != null) SendPacket(existingPeer.RemoteEndPoint, result.Reply); + SendPackets(existingPeer, result.ReadyToSend); if (packet.PacketType == CultNetRudpPacketType.Data) - acknowledgement = existingPeer.Session.CreateAck(packet.Sequence); + acknowledgement = existingPeer.Session.CreateAckForReceived(packet.Sequence); } if (result.Disconnected) { diff --git a/src/GameCult.Networking/GameCult.Networking.csproj b/src/GameCult.Networking/GameCult.Networking.csproj index f275484f..0d63f94c 100644 --- a/src/GameCult.Networking/GameCult.Networking.csproj +++ b/src/GameCult.Networking/GameCult.Networking.csproj @@ -7,10 +7,8 @@ - + diff --git a/src/GameCult.Networking/PlayerData.cs b/src/GameCult.Networking/PlayerData.cs index 997db848..ade82c0f 100644 --- a/src/GameCult.Networking/PlayerData.cs +++ b/src/GameCult.Networking/PlayerData.cs @@ -8,6 +8,7 @@ namespace GameCult.Networking /// Basic player account document used by the sample networking stack. /// [CultDocument("gamecult.player_data", "gamecult.player_data.v2")] + [MessagePackObject] public class PlayerData { /// diff --git a/src/GameCult.Networking/README.md b/src/GameCult.Networking/README.md index c7c4792f..de8496d3 100644 --- a/src/GameCult.Networking/README.md +++ b/src/GameCult.Networking/README.md @@ -390,6 +390,48 @@ peer lookup and handshake before returning the same RUDP transport. Packet semantics, schema-channel send/receive helpers, and reconnect state still live in `GameCult.Networking`. +## Typed Operations + +`CultNetOperationServer` attaches typed application handlers to any +`ICultNetSchemaServer`. It owns route/schema validation, MessagePack envelope +encoding, correlation, and framework failure replies. The application handler +owns domain validation, state mutation, idempotency, and durable receipts: + +```csharp +using var operations = new CultNetOperationServer(schemaServer, "counter.provider") + .Register( + "counter", + "counter.increment", + "counter.increment_request.v1", + "counter.increment_receipt.v1", + command => IncrementExactlyOnceAsync( + command.IdempotencyKey, + command.Value)); +``` + +Returning `CultNetOperationReply.Rejected(...)` preserves a schema-valid +domain receipt and status. Unknown routes, malformed payloads, and schema +mismatches return the correlated `gamecult.cultnet.operation_failure.v1` +payload; `CultMeshClient.InvokeAsync` raises +`CultMeshRemoteOperationException` immediately instead of timing out. The +browser client raises the equivalent `CultMeshBrowserOperationError` with the +same status and failure code. + +## Browser WebSocket Host + +`GameCult.Networking.WebSockets` keeps ASP.NET Core hosting out of the +netstandard CultNet core while adapting an authenticated binary WebSocket to +the same `ICultNetSchemaServer` and `ICultNetSchemaClient` ports. A host calls +`UseWebSockets()` and maps `MapCultNetWebSocket(...)` with an authorization +predicate. Anonymous use requires the explicit `AllowAnonymousDevelopment` +opt-in. Text frames and messages over the configured size bound are closed +before schema dispatch. + +The matching `CultNetWebSocketSchemaClient` is suitable for C# headless and +service consumers. Browser applications use `cultmesh-browser`, which owns +stable Verse/provider identity, document leases, resubscription, and operation +correlation above the raw WebSocket route. + ## Important Constraints - `Server` currently centers on `PlayerData` as the built-in account model. diff --git a/src/GameCult.Unity/Assets/Caching/Editor/CultCacheStudioDrawers.cs b/src/GameCult.Unity/Assets/Caching/Editor/CultCacheStudioDrawers.cs new file mode 100644 index 00000000..12eed25e --- /dev/null +++ b/src/GameCult.Unity/Assets/Caching/Editor/CultCacheStudioDrawers.cs @@ -0,0 +1,444 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using GameCult.Caching; +using UnityEditor; +using UnityEngine; +using Object = UnityEngine.Object; + +namespace GameCult.Unity.Caching.Editor +{ + // An IMGUI drawer claimed through [CultInspectorDrawer]. A change returns the new value and sets GUI.changed, as IMGUI + // controls do; only a frame that reports a change is saved. Mutating value in place without GUI.changed is not saved + // reliably: it lingers in the edit copy until some later change saves it. A drawer that throws discards the whole + // frame's edit copy, so nothing it or any other drawer did that frame is saved. type is the value's declared type. + // member is the member the value belongs to, also for list elements and dictionary keys and values, and null for a + // bare value. inspector.DrawDefault hands the value back to built-in drawing; inspector.DrawValue draws a sub-value + // with claims. + public interface ICultInspectorDrawer + { + object Draw(CultInspector inspector, string label, Type type, object value, MemberInfo member); + } + + // The IMGUI lowering of CultInspectorModel. The model decides members, metadata, shapes, drawer claims and edit + // rules; this class draws them and hands every change back through the model. + public sealed class CultInspector + { + private static CultInspectorDrawerClaims _claims; + private static readonly Dictionary DrawerInstances = new Dictionary(); + private static GUIStyle _errorStyle; + + private readonly Dictionary _foldouts = new Dictionary(StringComparer.Ordinal); + private readonly Dictionary _notices = new Dictionary(StringComparer.Ordinal); + private string _path = string.Empty; + private Type _document; + private bool _drawerFailed; + + internal CultInspector(CultInspectorModel model) + { + Model = model; + } + + public CultInspectorModel Model { get; } + + public IReadOnlyList Records { get; internal set; } = Array.Empty(); + + private static GUIStyle ErrorStyle => _errorStyle ??= new GUIStyle(EditorStyles.label) { normal = { textColor = new Color(1f, .35f, .3f) } }; + + private static CultInspectorDrawerClaims Claims + { + get + { + if (_claims != null) return _claims; + _claims = new CultInspectorDrawerClaims(TypeCache.GetTypesWithAttribute(), typeof(ICultInspectorDrawer)); + foreach (var error in _claims.Errors) Debug.LogError(error); + return _claims; + } + } + + // False when a drawer threw: the edit copy may hold part of its in-place mutation and must be discarded, not saved. + internal bool DrawDocument(CultInspectorEdit edit) + { + _drawerFailed = false; + _document = edit.Document.GetType(); + DrawMembers(edit.Document, edit.Source.Descriptor.DocumentType, edit.Source.Key.Value); + return !_drawerFailed; + } + + // Draws a sub-value of the value being drawn, claims applied. + public object DrawValue(string label, Type type, object value, MemberInfo member) + { + return DrawValue(label, type, value, member, _path + "/" + label); + } + + // Built-in drawing for a value, ignoring drawer claims on it. + public object DrawDefault(string label, Type type, object value, MemberInfo member) + { + var shape = Model.ShapeOf(type); + var metadata = Model.MetadataOf(member); + switch (shape.Kind) + { + case CultInspectorValueKind.String: + return DrawString(label, value as string, metadata); + case CultInspectorValueKind.Integer: + return DrawInteger(label, type, value, metadata); + case CultInspectorValueKind.Float: + if (type == typeof(double)) return EditorGUILayout.DoubleField(label, value is double d ? d : 0d); + var f = value is float single ? single : 0f; + return metadata.Range == null ? EditorGUILayout.FloatField(label, f) : EditorGUILayout.Slider(label, f, metadata.Range.Min, metadata.Range.Max); + case CultInspectorValueKind.Bool: + return EditorGUILayout.Toggle(label, value is bool b && b); + case CultInspectorValueKind.Enum: + var current = value as Enum ?? (Enum)Enum.ToObject(type, 0); + return type.IsDefined(typeof(FlagsAttribute), false) ? EditorGUILayout.EnumFlagsField(label, current) : EditorGUILayout.EnumPopup(label, current); + case CultInspectorValueKind.RecordRef: + return DrawRecordRef(label, type, value); + case CultInspectorValueKind.List: + return DrawList(label, shape, value, member); + case CultInspectorValueKind.Dictionary: + return DrawDictionary(label, shape, value as IDictionary, member); + case CultInspectorValueKind.Union: + return DrawUnion(label, shape, value); + case CultInspectorValueKind.Nested: + return DrawNested(label, type, value); + default: + return typeof(Object).IsAssignableFrom(type) + ? EditorGUILayout.ObjectField(label, value as Object, type, false) + : ErrorRow(label, shape.Reason, value); + } + } + + private object DrawValue(string label, Type type, object value, MemberInfo member, string path) + { + var outer = _path; + _path = path; + try + { + var claim = Claims.Resolve(type, member); + if (claim.Conflict != null) return ErrorRow(label, "no drawer: " + claim.Conflict, value); + if (claim.Drawer == null) return DrawDefault(label, type, value, member); + var changed = GUI.changed; + try + { + return Drawer(claim.Drawer).Draw(this, label, type, value, member); + } + catch (ExitGUIException) + { + throw; + } + catch (Exception exception) + { + GUI.changed = changed; + _drawerFailed = true; + return ErrorRow(label, claim.Drawer.Name + " failed: " + exception.Message + " Nothing edited this frame was saved.", value); + } + } + finally + { + _path = outer; + } + } + + private static ICultInspectorDrawer Drawer(Type type) + { + if (!DrawerInstances.TryGetValue(type, out var drawer)) + DrawerInstances[type] = drawer = (ICultInspectorDrawer)Activator.CreateInstance(type); + return drawer; + } + + private void DrawMembers(object target, Type type, string path) + { + foreach (var member in Model.MembersOf(type)) + { + if (member.Metadata.Hidden) continue; + using (new EditorGUI.DisabledScope(member.IsReadOnly)) + { + EditorGUI.BeginChangeCheck(); + var next = DrawValue(LabelOf(member), member.ValueType, member.GetValue(target), member.Member, path + "." + member.Name); + if (EditorGUI.EndChangeCheck() && !member.IsReadOnly) member.SetValue(target, next); + } + } + } + + private static string DrawString(string label, string value, CultInspectorMetadata metadata) + { + if (metadata.AssetPath != null) + { + var assetType = metadata.AssetPath.AssetType != null && typeof(Object).IsAssignableFrom(metadata.AssetPath.AssetType) ? metadata.AssetPath.AssetType : typeof(Object); + var asset = string.IsNullOrEmpty(value) ? null : AssetDatabase.LoadAssetAtPath(value, assetType); + var next = EditorGUILayout.ObjectField(label, asset, assetType, false); + return next == asset ? value : next == null ? string.Empty : AssetDatabase.GetAssetPath(next); + } + + var textArea = metadata.TextArea; + if (textArea == null) return EditorGUILayout.TextField(label, value ?? string.Empty); + EditorGUILayout.LabelField(label); + return EditorGUILayout.TextArea(value ?? string.Empty, + GUILayout.MinHeight(Mathf.Max(textArea.MinLines, 1) * EditorGUIUtility.singleLineHeight), + GUILayout.MaxHeight(Mathf.Max(textArea.MaxLines, textArea.MinLines, 1) * EditorGUIUtility.singleLineHeight)); + } + + private object DrawInteger(string label, Type type, object value, CultInspectorMetadata metadata) + { + var current = value == null ? 0L : Convert.ToInt64(value); + if (type == typeof(int) && metadata.Range != null) + return EditorGUILayout.IntSlider(label, (int)current, Mathf.RoundToInt(metadata.Range.Min), Mathf.RoundToInt(metadata.Range.Max)); + var next = EditorGUILayout.LongField(label, current); + return next == current && value != null ? value : Model.NarrowInteger(type, next); + } + + private object DrawRecordRef(string label, Type type, object value) + { + var key = CultInspectorModel.RecordKey(value); + var candidates = Model.RecordCandidates(type, Records); + var index = 0; + var names = new string[candidates.Count + 1]; + for (var i = 0; i < candidates.Count; i++) + { + names[i + 1] = CultInspectorModel.RecordLabel(candidates[i]); + if (candidates[i].Key.Value == key) index = i + 1; + } + + names[0] = index == 0 && key.Length > 0 ? "Missing " + key : "None"; + var next = key; + using (new EditorGUILayout.HorizontalScope()) + { + var picked = EditorGUILayout.Popup(label, index, names); + var indent = EditorGUI.indentLevel; + EditorGUI.indentLevel = 0; + var typed = EditorGUILayout.DelayedTextField(key, GUILayout.Width(120)); + EditorGUI.indentLevel = indent; + if (picked != index) next = picked == 0 ? string.Empty : candidates[picked - 1].Key.Value; + else if (typed != key) next = typed; + } + + return next == key ? value : Model.CreateRecordRef(type, next); + } + + private object DrawDictionary(string label, CultInspectorShape shape, IDictionary dictionary, MemberInfo member) + { + var path = _path; + var entries = new List>(); + if (dictionary != null) + { + var enumerator = dictionary.GetEnumerator(); + while (enumerator.MoveNext()) entries.Add(new KeyValuePair(enumerator.Key, enumerator.Value)); + } + + if (!Foldout(path, label + " (" + entries.Count + ")")) return dictionary; + EditorGUI.indentLevel++; + var changed = false; + var remove = -1; + for (var i = 0; i < entries.Count; i++) + { + using (new EditorGUILayout.HorizontalScope()) + { + using (new EditorGUILayout.VerticalScope()) + { + EditorGUI.BeginChangeCheck(); + var key = DrawValue("Key", shape.KeyType, entries[i].Key, member, path + "{" + i + "}.key"); + var item = DrawValue("Value", shape.ValueType, entries[i].Value, member, path + "{" + i + "}.value"); + if (EditorGUI.EndChangeCheck()) + { + key = Model.ReplaceKey(_document, shape.Type, entries.Select(e => e.Key).ToArray(), i, key, out var notice); + Notice(path, notice); + entries[i] = new KeyValuePair(key, item); + changed = true; + } + } + + if (GUILayout.Button("-", GUILayout.Width(20))) remove = i; + } + } + + if (remove >= 0) + { + entries.RemoveAt(remove); + changed = true; + } + + if (Add(shape.ValueType, path, out var created)) + { + var fresh = Model.FreshKey(_document, shape.Type, entries.Select(e => e.Key).ToArray(), Records, out var notice); + Notice(path, notice); + if (fresh != null) + { + entries.Add(new KeyValuePair(fresh, created)); + changed = true; + } + } + + DrawNotice(path); + EditorGUI.indentLevel--; + return changed ? Model.BuildDictionary(shape.Type, entries) : dictionary; + } + + private object DrawList(string label, CultInspectorShape shape, object value, MemberInfo member) + { + var path = _path; + var element = shape.ElementType; + var items = new List(); + if (value is IEnumerable enumerable) + foreach (var item in enumerable) items.Add(item); + + if (!Foldout(path, label + " [" + items.Count + "]")) return value; + EditorGUI.indentLevel++; + var changed = false; + var remove = -1; + for (var i = 0; i < items.Count; i++) + { + using (new EditorGUILayout.HorizontalScope()) + { + using (new EditorGUILayout.VerticalScope()) + { + EditorGUI.BeginChangeCheck(); + var next = DrawValue("Element " + i, element, items[i], member, path + "[" + i + "]"); + if (EditorGUI.EndChangeCheck()) + { + items[i] = next; + changed = true; + } + } + + if (GUILayout.Button("-", GUILayout.Width(20))) remove = i; + } + } + + if (remove >= 0) + { + items.RemoveAt(remove); + changed = true; + } + + if (Add(element, path, out var created)) + { + items.Add(created); + changed = true; + } + + DrawNotice(path); + EditorGUI.indentLevel--; + return changed ? Model.BuildList(shape.Type, items) : value; + } + + private object DrawUnion(string label, CultInspectorShape shape, object value) + { + var path = _path; + var choices = shape.UnionChoices; + var index = value == null ? 0 : IndexOf(choices, value.GetType()) + 1; + var names = choices.Select(u => u.Name).Prepend(value == null || index > 0 ? "None" : value.GetType().Name + " (not a declared union)").ToArray(); + + bool expanded; + using (new EditorGUILayout.HorizontalScope()) + { + expanded = Foldout(path, label, GUILayout.Width(EditorGUIUtility.labelWidth)); + var indent = EditorGUI.indentLevel; + EditorGUI.indentLevel = 0; + var picked = EditorGUILayout.Popup(index, names); + EditorGUI.indentLevel = indent; + if (picked != index) + { + string notice = null; + value = picked == 0 ? null : Model.CreateElement(shape.Type, choices[picked - 1], out notice) ?? value; + Notice(path, notice); + } + } + + DrawNotice(path); + if (!expanded || value == null) return value; + EditorGUI.indentLevel++; + DrawMembers(value, value.GetType(), path); + EditorGUI.indentLevel--; + return value; + } + + private object DrawNested(string label, Type type, object value) + { + var path = _path; + if (value == null) + { + using (new EditorGUILayout.HorizontalScope()) + { + EditorGUILayout.LabelField(label, "null"); + var created = Model.CreateDefault(type); + if (created != null && GUILayout.Button("Create", GUILayout.Width(56))) value = created; + } + + return value; + } + + if (!Foldout(path, label)) return value; + EditorGUI.indentLevel++; + DrawMembers(value, type, path); + EditorGUI.indentLevel--; + return value; + } + + // Foldouts stay usable inside disabled scopes so read-only data can still be browsed. Expanding is view state: + // it never counts as a change, so it never upserts. + private bool Foldout(string path, string label, params GUILayoutOption[] options) + { + _foldouts.TryGetValue(path, out var expanded); + var enabled = GUI.enabled; + var changed = GUI.changed; + GUI.enabled = true; + expanded = EditorGUI.Foldout(EditorGUILayout.GetControlRect(false, options), expanded, label, true); + GUI.enabled = enabled; + GUI.changed = changed; + _foldouts[path] = expanded; + return expanded; + } + + // The add control under a list or dictionary: a button when the model offers the element type itself, a popup of + // a union's declared subtypes otherwise. True with the model's new value once one is picked and made. + private bool Add(Type elementType, string path, out object created) + { + created = null; + var choices = Model.ElementChoices(elementType); + Type choice; + if (choices.Count == 1 && choices[0] == elementType) + { + choice = GUILayout.Button("Add") ? elementType : null; + } + else + { + var picked = EditorGUILayout.Popup("Add", 0, choices.Select(c => c.Name).Prepend("...").ToArray()); + choice = picked > 0 ? choices[picked - 1] : null; + } + + if (choice == null) return false; + created = Model.CreateElement(elementType, choice, out var notice); + Notice(path, notice); + return notice == null; + } + + // The model's notice for the control at path, kept until that control's next answer. + private void Notice(string path, string notice) + { + if (notice == null) _notices.Remove(path); + else _notices[path] = notice; + } + + private void DrawNotice(string path) + { + if (_notices.TryGetValue(path, out var notice)) EditorGUILayout.HelpBox(notice, MessageType.Warning); + } + + private static object ErrorRow(string label, string message, object value) + { + EditorGUILayout.LabelField(label, message, ErrorStyle); + return value; + } + + private static int IndexOf(IReadOnlyList types, Type type) + { + for (var i = 0; i < types.Count; i++) + if (types[i] == type) return i; + return -1; + } + + private static string LabelOf(CultInspectorMember member) => member.Metadata.Label ?? ObjectNames.NicifyVariableName(member.Name); + } +} diff --git a/src/GameCult.Unity/Assets/Caching/Editor/CultCacheStudioDrawers.cs.meta b/src/GameCult.Unity/Assets/Caching/Editor/CultCacheStudioDrawers.cs.meta new file mode 100644 index 00000000..a0d1bbf1 --- /dev/null +++ b/src/GameCult.Unity/Assets/Caching/Editor/CultCacheStudioDrawers.cs.meta @@ -0,0 +1,2 @@ +fileFormatVersion: 2 +guid: 06d55de1c539404b9a935871a8a3a334 diff --git a/src/GameCult.Unity/Assets/Caching/Editor/CultCacheStudioWindow.cs b/src/GameCult.Unity/Assets/Caching/Editor/CultCacheStudioWindow.cs index e50ff1e6..6c9fedf8 100644 --- a/src/GameCult.Unity/Assets/Caching/Editor/CultCacheStudioWindow.cs +++ b/src/GameCult.Unity/Assets/Caching/Editor/CultCacheStudioWindow.cs @@ -1,37 +1,38 @@ using System; -using System.Collections; -using System.Collections.Generic; -using System.Globalization; using System.IO; using System.Linq; -using System.Reflection; -using System.Threading.Tasks; -using GameCult.Unity.Caching; +using GameCult.Caching; +using GameCult.Caching.MessagePack; using UnityEditor; using UnityEngine; -using Object = UnityEngine.Object; namespace GameCult.Unity.Caching.Editor { - /// - /// Unity editor window for inspecting and editing CultCache .cc stores. - /// public sealed class CultCacheStudioWindow : EditorWindow { private const string LastPathKey = "GameCult.CultCacheStudio.LastPath"; - private readonly Dictionary _foldouts = new Dictionary(StringComparer.Ordinal); - private object _cache; + private CultCache _cache; + private CultInspectorModel _model; + private CultInspector _inspector; + private CultInspectorEdit _edit; + private CultStoredDocument[] _records = Array.Empty(); private string _path = string.Empty; + private bool _directory; + private bool _readOnly; private string _search = string.Empty; - private string _selectedKey = string.Empty; private Type _selectedType; + private string _selectedKey; private Vector2 _typeScroll; private Vector2 _recordScroll; private Vector2 _inspectorScroll; - private string _status = "Open a .cc file to begin."; + private string _status = "Open a CultCache store to begin."; private MessageType _statusType = MessageType.Info; + private bool ReadOnly => _cache.BackingStores.Any(store => store.IsReadOnly); + + private static string ProjectRoot => Path.GetDirectoryName(Path.GetFullPath(Application.dataPath)); + [MenuItem("GameCult/CultCache Studio")] public static void Open() { @@ -43,34 +44,27 @@ namespace GameCult.Unity.Caching.Editor _path = EditorPrefs.GetString(LastPathKey, string.Empty); } + // Disposing never flushes; unsaved edits are dropped on close and on script reload. + private void OnDisable() + { + CloseStore(); + } + private void OnGUI() { DrawToolbar(); - EditorGUILayout.Space(6); - - if (!CultCacheBridge.IsAvailable) - { - EditorGUILayout.HelpBox( - "CultCache Studio needs GameCult.Caching and GameCult.Caching.MessagePack loaded in this Unity project.", - MessageType.Warning); - } - - if (!string.IsNullOrWhiteSpace(_status)) - { - EditorGUILayout.HelpBox(_status, _statusType); - } - - if (_cache == null) - { - DrawClosedState(); - return; - } + if (!string.IsNullOrEmpty(_status)) EditorGUILayout.HelpBox(_status, _statusType); + if (_cache == null) return; + if (ReadOnly) + EditorGUILayout.HelpBox("Read-only store: " + _path + " was opened read-only. Adding, deleting, editing and saving are disabled.", MessageType.Warning); + _records = _cache.AllStoredDocuments.ToArray(); + _inspector.Records = _records; using (new EditorGUILayout.HorizontalScope()) { - DrawTypeColumn(); - DrawRecordColumn(); - DrawInspectorColumn(); + DrawTypes(); + DrawRecords(); + DrawInspector(); } } @@ -78,91 +72,59 @@ namespace GameCult.Unity.Caching.Editor { using (new EditorGUILayout.HorizontalScope(EditorStyles.toolbar)) { - GUILayout.Label("Path", GUILayout.Width(32)); _path = GUILayout.TextField(_path, EditorStyles.toolbarTextField, GUILayout.MinWidth(180)); - - using (new EditorGUI.DisabledScope(!CultCacheBridge.IsAvailable)) + _directory = GUILayout.Toggle(_directory, "Directory", EditorStyles.toolbarButton, GUILayout.Width(62)); + _readOnly = GUILayout.Toggle(_readOnly, "Read Only", EditorStyles.toolbarButton, GUILayout.Width(66)); + if (GUILayout.Button("Open", EditorStyles.toolbarButton, GUILayout.Width(44))) Browse(false); + using (new EditorGUI.DisabledScope(_readOnly || _directory)) { - if (GUILayout.Button("Open", EditorStyles.toolbarButton, GUILayout.Width(52))) - { - BrowseAndOpen(); - } - - if (GUILayout.Button("New", EditorStyles.toolbarButton, GUILayout.Width(44))) - { - BrowseAndCreate(); - } + if (GUILayout.Button(new GUIContent("New", "Creates a writable single-file store. Turn off Read Only and Directory to enable."), + EditorStyles.toolbarButton, GUILayout.Width(40))) + Browse(true); } using (new EditorGUI.DisabledScope(_cache == null)) { - if (GUILayout.Button("Reload", EditorStyles.toolbarButton, GUILayout.Width(58))) + if (GUILayout.Button("Reload", EditorStyles.toolbarButton, GUILayout.Width(52)) && ConfirmDiscard()) OpenStore(_path, false); + using (new EditorGUI.DisabledScope(_cache == null || ReadOnly || !_cache.IsDirty)) { - ReloadCurrent(); + if (GUILayout.Button("Save", EditorStyles.toolbarButton, GUILayout.Width(40))) Save(); } - if (GUILayout.Button("Save", EditorStyles.toolbarButton, GUILayout.Width(48))) + if (GUILayout.Button("Close", EditorStyles.toolbarButton, GUILayout.Width(44)) && ConfirmDiscard()) { - SaveCurrent(); + CloseStore(); + SetStatus("Closed " + _path + ".", MessageType.Info); + GUIUtility.ExitGUI(); } } GUILayout.FlexibleSpace(); if (_cache != null) - { - GUILayout.Label(CultCacheBridge.IsDirty(_cache) ? "Dirty" : "Saved", EditorStyles.miniLabel, GUILayout.Width(42)); - } + GUILayout.Label(ReadOnly ? "Read Only" : _cache.IsDirty ? "Dirty" : "Saved", EditorStyles.miniBoldLabel); } } - private void DrawClosedState() + private void DrawTypes() { - GUILayout.FlexibleSpace(); - using (new EditorGUILayout.HorizontalScope()) + using (new EditorGUILayout.VerticalScope(GUILayout.Width(240))) { - GUILayout.FlexibleSpace(); - using (new EditorGUILayout.VerticalScope(GUILayout.Width(420))) - { - EditorGUILayout.LabelField("CultCache Studio", EditorStyles.boldLabel); - EditorGUILayout.LabelField("Open or create a .cc file to inspect registered CultCache documents."); - EditorGUILayout.Space(8); - using (new EditorGUI.DisabledScope(!CultCacheBridge.IsAvailable)) - { - if (GUILayout.Button("Open .cc File", GUILayout.Height(32))) - { - BrowseAndOpen(); - } - - if (GUILayout.Button("Create .cc File", GUILayout.Height(28))) - { - BrowseAndCreate(); - } - } - } - - GUILayout.FlexibleSpace(); - } - - GUILayout.FlexibleSpace(); - } - - private void DrawTypeColumn() - { - using (new EditorGUILayout.VerticalScope(GUILayout.Width(250), GUILayout.ExpandHeight(true))) - { - EditorGUILayout.LabelField("Document Types", EditorStyles.boldLabel); _search = EditorGUILayout.TextField(_search, EditorStyles.toolbarSearchField); - _typeScroll = EditorGUILayout.BeginScrollView(_typeScroll, GUI.skin.box); - - foreach (var descriptor in FilteredDescriptors()) + _typeScroll = EditorGUILayout.BeginScrollView(_typeScroll); + foreach (var descriptor in _cache.Registry.AllDescriptors) { - var count = RecordsFor(descriptor.DocumentType).Count(); - var label = descriptor.SchemaName + " (" + count.ToString(CultureInfo.InvariantCulture) + ")"; - var style = _selectedType == descriptor.DocumentType ? EditorStyles.toolbarButton : EditorStyles.miniButton; - if (GUILayout.Button(label, style)) + var type = descriptor.DocumentType; + if (_search.Length > 0 && + descriptor.SchemaName.IndexOf(_search, StringComparison.OrdinalIgnoreCase) < 0 && + type.Name.IndexOf(_search, StringComparison.OrdinalIgnoreCase) < 0) + continue; + + var count = _model.RecordCandidates(typeof(CultRecordRef<>).MakeGenericType(type), _records).Count; + var label = descriptor.SchemaName + (descriptor.IsGlobal ? count == 0 ? " (absent)" : " (global)" : " (" + count + ")"); + if (GUILayout.Toggle(_selectedType == type, label, EditorStyles.miniButton) && _selectedType != type) { - _selectedType = descriptor.DocumentType; - _selectedKey = string.Empty; + _selectedType = type; + _selectedKey = null; } } @@ -170,445 +132,250 @@ namespace GameCult.Unity.Caching.Editor } } - private void DrawRecordColumn() + private void DrawRecords() { - using (new EditorGUILayout.VerticalScope(GUILayout.Width(280), GUILayout.ExpandHeight(true))) + using (new EditorGUILayout.VerticalScope(GUILayout.Width(260))) { + var descriptor = _selectedType == null ? null : _cache.Registry.GetRequired(_selectedType); + var selected = Selected(); + var cannotAdd = descriptor == null || ReadOnly || !_model.CanCreate(_selectedType); using (new EditorGUILayout.HorizontalScope()) { - EditorGUILayout.LabelField("Records", EditorStyles.boldLabel); - using (new EditorGUI.DisabledScope(_selectedType == null || _selectedType.GetConstructor(Type.EmptyTypes) == null)) + using (new EditorGUI.DisabledScope(cannotAdd || descriptor.IsGlobal)) { - if (GUILayout.Button("Add", GUILayout.Width(56))) - { - AddSelectedTypeRecord(); - } + if (GUILayout.Button("Add", EditorStyles.miniButtonLeft)) Add(); + } + + using (new EditorGUI.DisabledScope(selected == null || selected.Descriptor.IsGlobal || ReadOnly)) + { + if (GUILayout.Button("Duplicate", EditorStyles.miniButtonMid)) Duplicate(selected); + } + + using (new EditorGUI.DisabledScope(selected == null || ReadOnly)) + { + if (GUILayout.Button("Delete", EditorStyles.miniButtonRight)) Delete(selected); } } - _recordScroll = EditorGUILayout.BeginScrollView(_recordScroll, GUI.skin.box); - if (_selectedType == null) + _recordScroll = EditorGUILayout.BeginScrollView(_recordScroll); + if (descriptor == null) { EditorGUILayout.LabelField("Select a document type."); } else { - foreach (var record in RecordsFor(_selectedType)) + var records = _model.RecordCandidates(typeof(CultRecordRef<>).MakeGenericType(_selectedType), _records); + if (descriptor.IsGlobal && records.Count == 0) { - var style = string.Equals(_selectedKey, record.Key, StringComparison.Ordinal) - ? EditorStyles.toolbarButton - : EditorStyles.miniButton; - if (GUILayout.Button(GetRecordLabel(record), style)) + using (new EditorGUILayout.HorizontalScope()) { - _selectedKey = record.Key; + EditorGUILayout.LabelField("Global is absent."); + using (new EditorGUI.DisabledScope(cannotAdd)) + { + if (GUILayout.Button("Create", GUILayout.Width(56))) Add(); + } } } + + foreach (var record in records) + { + var label = CultInspectorModel.RecordLabel(record); + var text = record.Descriptor.DocumentType == _selectedType ? label : label + " <" + record.Descriptor.DocumentType.Name + ">"; + if (GUILayout.Toggle(_selectedKey == record.Key.Value, text, EditorStyles.miniButton)) _selectedKey = record.Key.Value; + } } EditorGUILayout.EndScrollView(); } } - private void DrawInspectorColumn() + private void DrawInspector() { - using (new EditorGUILayout.VerticalScope(GUILayout.ExpandWidth(true), GUILayout.ExpandHeight(true))) + using (new EditorGUILayout.VerticalScope(GUILayout.ExpandWidth(true))) { - var record = SelectedRecord(); - using (new EditorGUILayout.HorizontalScope()) - { - EditorGUILayout.LabelField("Inspector", EditorStyles.boldLabel); - using (new EditorGUI.DisabledScope(record == null)) - { - if (GUILayout.Button("Delete", GUILayout.Width(64))) - { - DeleteSelectedRecord(); - return; - } - } - } - - _inspectorScroll = EditorGUILayout.BeginScrollView(_inspectorScroll, GUI.skin.box); + var record = Selected(); if (record == null) { EditorGUILayout.LabelField("Select a record."); - EditorGUILayout.EndScrollView(); return; } - EditorGUILayout.SelectableLabel(record.Descriptor.SchemaName, EditorStyles.boldLabel, GUILayout.Height(EditorGUIUtility.singleLineHeight)); - EditorGUILayout.SelectableLabel("Key: " + record.Key, EditorStyles.miniLabel, GUILayout.Height(EditorGUIUtility.singleLineHeight)); - EditorGUILayout.SelectableLabel("Stored: " + record.StoredAt, EditorStyles.miniLabel, GUILayout.Height(EditorGUIUtility.singleLineHeight)); - EditorGUILayout.Space(8); + EditorGUILayout.LabelField(record.Descriptor.SchemaName + " " + record.Descriptor.DocumentType.Name, EditorStyles.boldLabel); + EditorGUILayout.SelectableLabel("Key " + record.Key.Value + " Stored " + record.StoredAt, EditorStyles.miniLabel, GUILayout.Height(EditorGUIUtility.singleLineHeight)); + _inspectorScroll = EditorGUILayout.BeginScrollView(_inspectorScroll); - EditorGUI.BeginChangeCheck(); - DrawDocument(record.Document, record.Descriptor.DocumentType, record.Descriptor.SchemaName); - if (EditorGUI.EndChangeCheck()) + // Drawers edit the edit's private copy. A commit spends the edit, admitted or refused, and the next frame + // begins a new one from whatever record the cache then holds. The copy is held here and kept only by a frame + // that drew to the end with no drawer failing: a claimed drawer that threw, a built-in drawing exception, or + // an ExitGUIException all leave _edit null, so a half-made in-place mutation is never committed later. + var edit = _edit != null && _edit.IsFor(record) ? _edit : _model.BeginEdit(record); + _edit = null; + var readOnly = ReadOnly; + using (new EditorGUI.DisabledScope(readOnly)) { - UpsertRecord(record); + EditorGUI.BeginChangeCheck(); + var drawn = _inspector.DrawDocument(edit); + var changed = EditorGUI.EndChangeCheck(); + if (drawn) + { + _edit = edit; + if (changed && !readOnly && !edit.Commit(_cache, out var error)) SetStatus(error, MessageType.Error); + } } EditorGUILayout.EndScrollView(); } } - private IEnumerable FilteredDescriptors() + private CultStoredDocument Selected() { - var descriptors = CultCacheBridge.GetDescriptors(_cache); - if (string.IsNullOrWhiteSpace(_search)) + return _selectedKey == null ? null : _records.FirstOrDefault(r => r.Key.Value == _selectedKey); + } + + // Dialogs start in the last store's folder, or the project root; never under Assets. + private void Browse(bool create) + { + if (!ConfirmDiscard()) return; + var last = string.IsNullOrEmpty(_path) ? null : Path.GetDirectoryName(FullPathOrNull(_path) ?? string.Empty); + var directory = !string.IsNullOrEmpty(last) && Directory.Exists(last) && !InsideAssets(last) ? last : ProjectRoot; + var path = create + ? EditorUtility.SaveFilePanel("New CultCache store", directory, "GameData", "cc") + : EditorUtility.OpenFilePanel("Open CultCache store", directory, _directory ? "" : "cc"); + if (!string.IsNullOrEmpty(path)) OpenStore(path, create); + GUIUtility.ExitGUI(); + } + + // New writes an empty single-file store outside Assets and never touches an existing path; directory stores and + // read-only opens are refused because neither writes a store on creation. Game data never lives under Assets: + // Unity would import and manage it, and CultCache exists so Unity does not manage game data. + private void OpenStore(string path, bool create) + { + var exists = File.Exists(path) || Directory.Exists(DirectoryMessagePackBackingStore.DefaultRecordDirectoryPath(path)); + var insideAssets = InsideAssets(path); + string refusal = null; + if (create && (_readOnly || _directory)) + refusal = "New creates writable single-file stores only; turn off Read Only and Directory. Nothing was created."; + else if (create && insideAssets) + refusal = "New refuses " + path + ": game data does not live under Assets, where Unity imports and manages files, and CultCache exists so Unity does not manage game data. Choose a folder outside " + Application.dataPath + ". Nothing was created."; + else if (create && exists) + refusal = path + " already exists; nothing was created or replaced. Use Open to edit it."; + else if (!create && !File.Exists(path)) + refusal = "No store at " + path + "."; + if (refusal != null) { - return descriptors; - } - - return descriptors.Where(descriptor => - descriptor.SchemaName.IndexOf(_search, StringComparison.OrdinalIgnoreCase) >= 0 || - descriptor.DocumentType.Name.IndexOf(_search, StringComparison.OrdinalIgnoreCase) >= 0); - } - - private IEnumerable RecordsFor(Type type) - { - return CultCacheBridge.GetStoredDocuments(_cache) - .Where(record => type.IsAssignableFrom(record.Descriptor.DocumentType)) - .OrderBy(GetRecordLabel, StringComparer.Ordinal); - } - - private RecordInfo SelectedRecord() - { - if (_cache == null || string.IsNullOrEmpty(_selectedKey)) - { - return null; - } - - return CultCacheBridge.GetStoredDocuments(_cache) - .FirstOrDefault(record => string.Equals(record.Key, _selectedKey, StringComparison.Ordinal)); - } - - private string GetRecordLabel(RecordInfo record) - { - var namedMember = record.Descriptor.Members.FirstOrDefault(member => member.IsName); - if (namedMember != null) - { - var member = FindInspectableMember(record.Descriptor.DocumentType, namedMember.MemberName); - if (member != null) - { - var value = member.GetValue(record.Document); - if (value != null && !string.IsNullOrWhiteSpace(value.ToString())) - { - return value + " [" + record.Key + "]"; - } - } - } - - return record.Key; - } - - private void DrawDocument(object target, Type type, string path) - { - foreach (var member in GetInspectableMembers(type)) - { - if (member.GetCustomAttribute() != null) - { - continue; - } - - var current = member.GetValue(target); - var readOnly = member.GetCustomAttribute() != null || member.IsReadOnly; - using (new EditorGUI.DisabledScope(readOnly)) - { - var next = DrawValue(GetLabel(member), member.MemberType, current, member, path + "." + member.Name); - if (!readOnly && !Equals(current, next)) - { - member.SetValue(target, next); - } - } - } - } - - private object DrawValue(string label, Type type, object value, InspectableMember member, string path) - { - if (type == typeof(string)) return DrawString(label, value as string, member); - if (type == typeof(int)) - { - var range = member.GetCustomAttribute(); - return range == null - ? EditorGUILayout.IntField(label, value == null ? 0 : (int)value) - : EditorGUILayout.IntSlider(label, value == null ? 0 : (int)value, Mathf.RoundToInt(range.Min), Mathf.RoundToInt(range.Max)); - } - - if (type == typeof(uint)) return (uint)Math.Max(0L, EditorGUILayout.LongField(label, value == null ? 0L : Convert.ToInt64(value, CultureInfo.InvariantCulture))); - if (type == typeof(long)) return EditorGUILayout.LongField(label, value == null ? 0L : (long)value); - if (type == typeof(float)) - { - var range = member.GetCustomAttribute(); - return range == null - ? EditorGUILayout.FloatField(label, value == null ? 0f : (float)value) - : EditorGUILayout.Slider(label, value == null ? 0f : (float)value, range.Min, range.Max); - } - - if (type == typeof(double)) return EditorGUILayout.DoubleField(label, value == null ? 0d : (double)value); - if (type == typeof(bool)) return EditorGUILayout.Toggle(label, value != null && (bool)value); - if (type.IsEnum) return EditorGUILayout.EnumPopup(label, value == null ? (Enum)Enum.GetValues(type).GetValue(0) : (Enum)value); - if (type == typeof(Vector2)) return EditorGUILayout.Vector2Field(label, value == null ? Vector2.zero : (Vector2)value); - if (type == typeof(Vector3)) return EditorGUILayout.Vector3Field(label, value == null ? Vector3.zero : (Vector3)value); - if (type == typeof(Color)) return EditorGUILayout.ColorField(label, value == null ? Color.white : (Color)value); - if (typeof(Object).IsAssignableFrom(type)) return EditorGUILayout.ObjectField(label, value as Object, type, false); - if (CultCacheBridge.IsCultRecordRef(type)) return CultCacheBridge.DrawRecordRef(label, type, value); - if (type.IsArray || typeof(IList).IsAssignableFrom(type)) return DrawList(label, type, value, member, path); - if (type.IsClass || IsMutableStruct(type)) return DrawNestedObject(label, type, value, path); - - using (new EditorGUI.DisabledScope(true)) - { - EditorGUILayout.TextField(label, value == null ? string.Empty : value.ToString()); - } - - return value; - } - - private string DrawString(string label, string value, InspectableMember member) - { - var assetPath = member.GetCustomAttribute(); - if (assetPath != null) - { - var assetType = assetPath.AssetType == null || !typeof(Object).IsAssignableFrom(assetPath.AssetType) - ? typeof(Object) - : assetPath.AssetType; - var asset = string.IsNullOrEmpty(value) ? null : AssetDatabase.LoadAssetAtPath(value, assetType); - var nextAsset = EditorGUILayout.ObjectField(label, asset, assetType, false); - return nextAsset == null ? string.Empty : AssetDatabase.GetAssetPath(nextAsset); - } - - var textArea = member.GetCustomAttribute(); - if (textArea != null) - { - EditorGUILayout.LabelField(label); - return EditorGUILayout.TextArea(value ?? string.Empty, GUILayout.MinHeight(Mathf.Max(textArea.MinLines, 1) * EditorGUIUtility.singleLineHeight)); - } - - return EditorGUILayout.TextField(label, value ?? string.Empty); - } - - private object DrawList(string label, Type type, object value, InspectableMember member, string path) - { - if (!Foldout(path, label)) return value; - - var elementType = GetListElementType(type); - var list = ToMutableList(elementType, value); - EditorGUI.indentLevel++; - var size = Mathf.Max(0, EditorGUILayout.IntField("Size", list.Count)); - while (list.Count < size) list.Add(CreateDefaultValue(elementType)); - while (list.Count > size) list.RemoveAt(list.Count - 1); - - for (var index = 0; index < list.Count; index++) - { - using (new EditorGUILayout.HorizontalScope()) - { - list[index] = DrawValue("Element " + index.ToString(CultureInfo.InvariantCulture), elementType, list[index], member, path + "[" + index.ToString(CultureInfo.InvariantCulture) + "]"); - if (GUILayout.Button("-", GUILayout.Width(24))) - { - list.RemoveAt(index); - index--; - } - } - } - - if (GUILayout.Button("Add " + elementType.Name)) list.Add(CreateDefaultValue(elementType)); - EditorGUI.indentLevel--; - return FromMutableList(type, elementType, list); - } - - private object DrawNestedObject(string label, Type type, object value, string path) - { - var expanded = Foldout(path, label); - if (value == null && type.GetConstructor(Type.EmptyTypes) != null) value = Activator.CreateInstance(type); - if (!expanded || value == null) return value; - EditorGUI.indentLevel++; - DrawDocument(value, type, path); - EditorGUI.indentLevel--; - return value; - } - - private bool Foldout(string key, string label) - { - bool expanded; - _foldouts.TryGetValue(key, out expanded); - expanded = EditorGUILayout.Foldout(expanded, label, true); - _foldouts[key] = expanded; - return expanded; - } - - private IEnumerable GetInspectableMembers(Type type) - { - var descriptor = CultCacheBridge.GetDescriptor(_cache, type); - var byName = descriptor.Members.ToDictionary(member => member.MemberName, StringComparer.Ordinal); - return type.GetMembers(BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic) - .Select(InspectableMember.TryCreate) - .Where(member => member != null && byName.ContainsKey(member.Name)) - .OrderBy(member => member.GetCustomAttribute() == null - ? byName[member.Name].Slot - : member.GetCustomAttribute().Order) - .ThenBy(member => byName[member.Name].Slot) - .ThenBy(member => member.Name, StringComparer.Ordinal); - } - - private static InspectableMember FindInspectableMember(Type type, string name) - { - return type.GetMembers(BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic) - .Select(InspectableMember.TryCreate) - .FirstOrDefault(member => member != null && string.Equals(member.Name, name, StringComparison.Ordinal)); - } - - private static string GetLabel(InspectableMember member) - { - var label = member.GetCustomAttribute(); - return label == null || string.IsNullOrWhiteSpace(label.Label) - ? ObjectNames.NicifyVariableName(member.Name) - : label.Label; - } - - private static Type GetListElementType(Type type) - { - if (type.IsArray) return type.GetElementType(); - return type.IsGenericType ? type.GetGenericArguments()[0] : typeof(object); - } - - private static IList ToMutableList(Type elementType, object value) - { - var result = (IList)Activator.CreateInstance(typeof(List<>).MakeGenericType(elementType)); - var source = value as IEnumerable; - if (source != null) - { - foreach (var item in source) result.Add(item); - } - - return result; - } - - private static object FromMutableList(Type listType, Type elementType, IList list) - { - if (listType.IsArray) - { - var array = Array.CreateInstance(elementType, list.Count); - for (var index = 0; index < list.Count; index++) array.SetValue(list[index], index); - return array; - } - - if (listType.IsAssignableFrom(list.GetType())) return list; - var target = listType.GetConstructor(Type.EmptyTypes) == null ? null : Activator.CreateInstance(listType) as IList; - if (target == null) return list; - foreach (var item in list) target.Add(item); - return target; - } - - private static object CreateDefaultValue(Type type) - { - if (type == typeof(string)) return string.Empty; - if (type.IsValueType) return Activator.CreateInstance(type); - return type.GetConstructor(Type.EmptyTypes) == null ? null : Activator.CreateInstance(type); - } - - private static bool IsMutableStruct(Type type) - { - return type.IsValueType && !type.IsPrimitive && !type.IsEnum; - } - - private void BrowseAndOpen() - { - var start = string.IsNullOrEmpty(_path) ? Application.dataPath : Path.GetDirectoryName(_path); - var selected = EditorUtility.OpenFilePanel("Open CultCache", start, "cc"); - if (!string.IsNullOrEmpty(selected)) - { - _path = selected; - OpenPath(true); - } - } - - private void BrowseAndCreate() - { - var start = string.IsNullOrEmpty(_path) ? Application.dataPath : Path.GetDirectoryName(_path); - var selected = EditorUtility.SaveFilePanel("Create CultCache", start, "GameData", "cc"); - if (!string.IsNullOrEmpty(selected)) - { - _path = selected; - OpenPath(false); - SaveCurrent(); - } - } - - private void OpenPath(bool pullOnOpen) - { - try - { - _cache = CultCacheBridge.Open(_path, pullOnOpen); - _selectedType = CultCacheBridge.GetDescriptors(_cache).FirstOrDefault()?.DocumentType; - _selectedKey = string.Empty; - EditorPrefs.SetString(LastPathKey, _path); - SetStatus("Opened " + _path, MessageType.Info); - } - catch (Exception ex) - { - _cache = null; - SetStatus("Failed to open CultCache: " + ex.GetBaseException().Message, MessageType.Error); - } - } - - private void ReloadCurrent() - { - if (string.IsNullOrWhiteSpace(_path)) - { - SetStatus("No .cc path selected.", MessageType.Warning); + SetStatus(refusal, MessageType.Error); return; } - OpenPath(true); + CloseStore(); + try + { + _cache = CultCacheMessagePack.Create(path, new CultCacheOpenOptions { ReadOnly = !create && _readOnly, UseDirectoryStore = !create && _directory }); + if (create) + { + _cache.BackingStores[0].PushAll(); + if (!File.Exists(path)) throw new IOException("the store wrote no file"); + } + + _model = CultCacheMessagePack.CreateInspectorModel(_cache.Registry); + _inspector = new CultInspector(_model); + _path = path; + EditorPrefs.SetString(LastPathKey, path); + var opened = (create ? "Created " : "Opened ") + path + (_cache.BackingStores[0] is DirectoryMessagePackBackingStore ? " (directory store)." : "."); + if (insideAssets) + SetStatus(opened + " This store is under Assets, where Unity imports and manages files; game data belongs outside Assets.", MessageType.Warning); + else + SetStatus(opened, MessageType.Info); + } + catch (Exception exception) + { + CloseStore(); + SetStatus("Failed to open " + path + ": " + exception.GetBaseException().Message, MessageType.Error); + } } - private void SaveCurrent() + private static bool InsideAssets(string path) + { + var full = FullPathOrNull(path); + var assets = FullPathOrNull(Application.dataPath)?.TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar); + return full != null && assets != null && + (full.Equals(assets, StringComparison.OrdinalIgnoreCase) || + full.StartsWith(assets + Path.DirectorySeparatorChar, StringComparison.OrdinalIgnoreCase)); + } + + private static string FullPathOrNull(string path) { try { - CultCacheBridge.Flush(_cache); - SetStatus("Saved " + _path, MessageType.Info); + return Path.GetFullPath(path); } - catch (Exception ex) + catch (Exception) { - SetStatus("Failed to save CultCache: " + ex.GetBaseException().Message, MessageType.Error); + return null; } } - private void AddSelectedTypeRecord() + private void CloseStore() + { + _cache?.Dispose(); + _cache = null; + _model = null; + _inspector = null; + _edit = null; + _records = Array.Empty(); + _selectedKey = null; + } + + private bool ConfirmDiscard() + { + return _cache == null || !_cache.IsDirty || + EditorUtility.DisplayDialog("CultCache Studio", "Discard unsaved changes to " + _path + "?", "Discard", "Cancel"); + } + + private void Save() + { + Run("Saved " + _path + ".", () => _cache.FlushAsync().GetAwaiter().GetResult()); + } + + private void Add() + { + var type = _selectedType; + // The model decides whether the type can be made; its notice is the refusal. + Run("Added " + type.Name + ".", () => _selectedKey = _cache.UpsertAsync(type, + _model.CreateElement(type, type, out var notice) ?? throw new InvalidOperationException(notice)).GetAwaiter().GetResult().Value); + GUIUtility.ExitGUI(); + } + + private void Duplicate(CultStoredDocument record) + { + var type = record.Descriptor.DocumentType; + Run("Duplicated " + CultInspectorModel.RecordLabel(record) + ".", () => + _selectedKey = _cache.UpsertAsync(type, _model.Clone(record.Document, type)).GetAwaiter().GetResult().Value); + GUIUtility.ExitGUI(); + } + + private void Delete(CultStoredDocument record) + { + var label = CultInspectorModel.RecordLabel(record); + if (!EditorUtility.DisplayDialog("CultCache Studio", "Delete " + label + "?", "Delete", "Cancel")) return; + Run("Deleted " + label + ".", () => + { + _cache.Remove(record.Key); + _selectedKey = null; + }); + GUIUtility.ExitGUI(); + } + + private void Run(string success, Action action) { try { - var document = Activator.CreateInstance(_selectedType); - _selectedKey = CultCacheBridge.Upsert(_cache, _selectedType, document, null); - SetStatus("Added " + _selectedType.Name + ".", MessageType.Info); + action(); + if (success != null) SetStatus(success, MessageType.Info); } - catch (Exception ex) + catch (Exception exception) { - SetStatus("Failed to add record: " + ex.GetBaseException().Message, MessageType.Error); - } - } - - private void DeleteSelectedRecord() - { - if (string.IsNullOrEmpty(_selectedKey)) return; - if (!EditorUtility.DisplayDialog("Delete CultCache Record", "Delete record " + _selectedKey + "?", "Delete", "Cancel")) return; - if (CultCacheBridge.Remove(_cache, _selectedKey)) - { - _selectedKey = string.Empty; - SetStatus("Deleted record.", MessageType.Info); - } - } - - private void UpsertRecord(RecordInfo record) - { - try - { - CultCacheBridge.Upsert(_cache, record.Descriptor.DocumentType, record.Document, record.Key); - SetStatus("Edited " + record.Descriptor.SchemaName + ". Save to persist.", MessageType.Info); - } - catch (Exception ex) - { - SetStatus("Failed to apply edit: " + ex.GetBaseException().Message, MessageType.Error); + SetStatus(exception.GetBaseException().Message, MessageType.Error); } } @@ -618,230 +385,5 @@ namespace GameCult.Unity.Caching.Editor _statusType = type; Repaint(); } - - private sealed class InspectableMember - { - private readonly FieldInfo _field; - private readonly PropertyInfo _property; - - private InspectableMember(FieldInfo field) - { - _field = field; - Name = field.Name; - MemberType = field.FieldType; - IsReadOnly = field.IsInitOnly || field.IsLiteral; - } - - private InspectableMember(PropertyInfo property) - { - _property = property; - Name = property.Name; - MemberType = property.PropertyType; - IsReadOnly = property.GetSetMethod(true) == null; - } - - public string Name { get; } - public Type MemberType { get; } - public bool IsReadOnly { get; } - - public static InspectableMember TryCreate(MemberInfo member) - { - var field = member as FieldInfo; - if (field != null && !field.IsStatic) return new InspectableMember(field); - var property = member as PropertyInfo; - return property != null && property.GetIndexParameters().Length == 0 && property.GetGetMethod(true) != null - ? new InspectableMember(property) - : null; - } - - public T GetCustomAttribute() where T : Attribute - { - return _field != null ? _field.GetCustomAttribute(true) : _property.GetCustomAttribute(true); - } - - public object GetValue(object target) - { - return _field != null ? _field.GetValue(target) : _property.GetValue(target, null); - } - - public void SetValue(object target, object value) - { - if (_field != null) _field.SetValue(target, value); - else _property.SetValue(target, value, null); - } - } - - private static class CultCacheBridge - { - private static readonly Type CacheType = FindType("GameCult.Caching.CultCache"); - private static readonly Type OpenOptionsType = FindType("GameCult.Caching.MessagePack.CultCacheOpenOptions"); - private static readonly Type MessagePackType = FindType("GameCult.Caching.MessagePack.CultCacheMessagePack"); - private static readonly Type RecordKeyType = FindType("GameCult.Caching.CultRecordKey"); - - public static bool IsAvailable => CacheType != null && OpenOptionsType != null && MessagePackType != null && RecordKeyType != null; - - public static object Open(string path, bool pullOnOpen) - { - EnsureAvailable(); - var options = Activator.CreateInstance(OpenOptionsType); - OpenOptionsType.GetProperty("PullOnOpen").SetValue(options, pullOnOpen, null); - var method = MessagePackType.GetMethod("OpenAsync", new[] { typeof(string), OpenOptionsType }); - var task = (Task)method.Invoke(null, new[] { path, options }); - task.GetAwaiter().GetResult(); - return task.GetType().GetProperty("Result").GetValue(task, null); - } - - public static bool IsDirty(object cache) - { - return cache != null && (bool)CacheType.GetProperty("IsDirty").GetValue(cache, null); - } - - public static IEnumerable GetDescriptors(object cache) - { - if (cache == null) return Enumerable.Empty(); - var registry = CacheType.GetProperty("Registry").GetValue(cache, null); - var descriptors = (IEnumerable)registry.GetType().GetProperty("AllDescriptors").GetValue(registry, null); - return descriptors.Cast().Select(DescriptorInfo.From).ToArray(); - } - - public static DescriptorInfo GetDescriptor(object cache, Type documentType) - { - var registry = CacheType.GetProperty("Registry").GetValue(cache, null); - var method = registry.GetType().GetMethod("GetRequired", new[] { typeof(Type) }); - return DescriptorInfo.From(method.Invoke(registry, new object[] { documentType })); - } - - public static IEnumerable GetStoredDocuments(object cache) - { - if (cache == null) return Enumerable.Empty(); - var records = (IEnumerable)CacheType.GetProperty("AllStoredDocuments").GetValue(cache, null); - return records.Cast().Select(RecordInfo.From).ToArray(); - } - - public static string Upsert(object cache, Type documentType, object document, string key) - { - var method = CacheType.GetMethod("UpsertAsync", new[] { typeof(Type), typeof(object), typeof(Nullable<>).MakeGenericType(RecordKeyType) }); - var keyObject = string.IsNullOrEmpty(key) ? null : CreateRecordKey(key); - var task = (Task)method.Invoke(cache, new[] { documentType, document, keyObject }); - task.GetAwaiter().GetResult(); - var result = task.GetType().GetProperty("Result").GetValue(task, null); - return GetKeyValue(result); - } - - public static bool Remove(object cache, string key) - { - var method = CacheType.GetMethod("Remove", new[] { RecordKeyType }); - return (bool)method.Invoke(cache, new[] { CreateRecordKey(key) }); - } - - public static void Flush(object cache) - { - var method = CacheType.GetMethod("FlushAsync", new[] { typeof(bool) }); - var task = (Task)method.Invoke(cache, new object[] { false }); - task.GetAwaiter().GetResult(); - } - - public static bool IsCultRecordRef(Type type) - { - return type.IsGenericType && type.GetGenericTypeDefinition().FullName == "GameCult.Caching.CultRecordRef`1"; - } - - public static object DrawRecordRef(string label, Type type, object value) - { - var keyProperty = type.GetProperty("Key"); - var currentKey = keyProperty == null || value == null ? string.Empty : GetKeyValue(keyProperty.GetValue(value, null)); - var nextKey = EditorGUILayout.TextField(label, currentKey); - return string.Equals(currentKey, nextKey, StringComparison.Ordinal) - ? value - : Activator.CreateInstance(type, CreateRecordKey(nextKey)); - } - - private static object CreateRecordKey(string key) - { - return Activator.CreateInstance(RecordKeyType, key ?? string.Empty); - } - - private static string GetKeyValue(object key) - { - return key == null ? string.Empty : (string)RecordKeyType.GetProperty("Value").GetValue(key, null); - } - - private static void EnsureAvailable() - { - if (!IsAvailable) - { - throw new InvalidOperationException("GameCult.Caching and GameCult.Caching.MessagePack are not loaded."); - } - } - - private static Type FindType(string fullName) - { - return AppDomain.CurrentDomain.GetAssemblies() - .Where(assembly => !assembly.IsDynamic) - .Select(assembly => assembly.GetType(fullName, false)) - .FirstOrDefault(type => type != null); - } - } - - private sealed class DescriptorInfo - { - public object Raw; - public Type DocumentType; - public string SchemaName; - public CatalogMemberInfo[] Members; - - public static DescriptorInfo From(object raw) - { - var type = raw.GetType(); - var catalog = type.GetMethod("ToCatalogEntry").Invoke(raw, null); - var members = (IEnumerable)catalog.GetType().GetProperty("Members").GetValue(catalog, null); - return new DescriptorInfo - { - Raw = raw, - DocumentType = (Type)type.GetProperty("DocumentType").GetValue(raw, null), - SchemaName = (string)type.GetProperty("SchemaName").GetValue(raw, null), - Members = members.Cast().Select(CatalogMemberInfo.From).ToArray() - }; - } - } - - private sealed class CatalogMemberInfo - { - public string MemberName; - public int Slot; - public bool IsName; - - public static CatalogMemberInfo From(object raw) - { - var type = raw.GetType(); - return new CatalogMemberInfo - { - MemberName = (string)type.GetProperty("MemberName").GetValue(raw, null), - Slot = (int)type.GetProperty("Slot").GetValue(raw, null), - IsName = (bool)type.GetProperty("IsName").GetValue(raw, null) - }; - } - } - - private sealed class RecordInfo - { - public string Key; - public string StoredAt; - public DescriptorInfo Descriptor; - public object Document; - - public static RecordInfo From(object raw) - { - var type = raw.GetType(); - var key = type.GetProperty("Key").GetValue(raw, null); - return new RecordInfo - { - Key = (string)key.GetType().GetProperty("Value").GetValue(key, null), - StoredAt = (string)type.GetProperty("StoredAt").GetValue(raw, null), - Descriptor = DescriptorInfo.From(type.GetProperty("Descriptor").GetValue(raw, null)), - Document = type.GetProperty("Document").GetValue(raw, null) - }; - } - } } } diff --git a/src/GameCult.Unity/Assets/Caching/Editor/GameCult.Unity.Caching.Editor.asmdef b/src/GameCult.Unity/Assets/Caching/Editor/GameCult.Unity.Caching.Editor.asmdef index be5589c6..0a5b140d 100644 --- a/src/GameCult.Unity/Assets/Caching/Editor/GameCult.Unity.Caching.Editor.asmdef +++ b/src/GameCult.Unity/Assets/Caching/Editor/GameCult.Unity.Caching.Editor.asmdef @@ -2,7 +2,7 @@ "name": "GameCult.Unity.Caching.Editor", "rootNamespace": "GameCult.Unity.Caching.Editor", "references": [ - "GameCult.Unity.Caching" + "GameCult.CultLib" ], "includePlatforms": [ "Editor" diff --git a/src/GameCult.Unity/Assets/Caching/README.md b/src/GameCult.Unity/Assets/Caching/README.md index 84b24084..a5ea3d92 100644 --- a/src/GameCult.Unity/Assets/Caching/README.md +++ b/src/GameCult.Unity/Assets/Caching/README.md @@ -1,35 +1,129 @@ # GameCult CultCache Unity Tools -`org.gamecult.caching.unity` provides Unity editor tooling for opening, -inspecting, editing, and saving CultCache `.cc` stores. +`org.gamecult.caching.unity` is the Unity editor surface over CultCache: open a +store, browse its documents, edit them, save. -The package is intentionally a Unity editor surface over `GameCult.Caching`. -CultCache owns document identity, schema descriptors, persistence, dirty state, -and flush behavior. The editor window owns selection, presentation, and edit -commands. +CultCache owns document identity, schema descriptors, persistence, dirty state +and flush. `CultInspectorModel` (in `GameCult.Caching`) owns what an inspector +shows and which edits it may make. The Studio window owns selection, the +toolbar and IMGUI widgets. It references `GameCult.Caching` and +`GameCult.Caching.MessagePack` at compile time through the +`org.gamecult.cultlib` package (assembly `GameCult.CultLib`). ## Use -1. Reference `GameCult.Caching`, `GameCult.Caching.MessagePack`, and this UPM - package from the Unity project that declares your `[CultDocument]` classes. -2. Open Unity's `GameCult/CultCache Studio` menu. -3. Open or create a `.cc` file. -4. Select a document type, select a record, edit fields, then save explicitly. +1. Add this package; it depends on `org.gamecult.cultlib`. +2. Open `GameCult/CultCache Studio`. +3. `Open` a `.cc` store. `Directory` opens a directory store (a manifest beside + its `.records` folder; an existing one is detected without the toggle). + `Read Only` opens the store read-only. `New` creates an empty writable + single-file store; it is disabled while `Read Only` or `Directory` is on and + refuses a path that already exists. +4. Pick a document type, then a record. Edits upsert into the cache as you make + them; the toolbar shows `Dirty` until `Save` flushes the store. -The editor discovers registered CultCache document descriptors from the same -registry used by runtime code. It does not require document classes to inherit -from Unity types or storage base classes. +Game data stores do not live under the project's `Assets` folder: Unity would +import and manage them, and CultCache exists so Unity does not manage game +data. `New` refuses a path under `Assets`, `Open` warns about one, and the file +dialogs start in the last store's folder or the project root. + +Document types come from the cache's `CultDocumentRegistry`. A type lists its +own records and those of its subclasses, by `[CultName]` with the key as +fallback. `Add`, `Duplicate` and `Delete` act on the selected type and record. +A `[CultGlobal]` type with no record shows as absent with a `Create` action; the +Studio never creates one on its own. + +Opening a store never writes it, directory stores included. Closing or +reloading scripts drops unsaved edits. + +## Inspection Model + +`CultInspectorModel` is engine-free and lives beside the inspector attributes +in `GameCult.Caching`; `CultCacheMessagePack.CreateInspectorModel` builds it +over a `.cc` store's codec. It gives the member list of a type (registry +catalog slots, `CultInspector*` metadata, whether a member is assignable), the +shape of any value (string, integer, float, bool, enum, `CultRecordRef` with +its candidate records, rank-1 list or array, dictionary, `[Union]` with only +its declared subtypes, nested keyed object, struct edited in place through its +public fields, or its settable properties when it has none, or unsupported with a reason), drawer claim resolution +(`CultInspectorDrawerClaims`), and the edit decisions: the key a dictionary +entry keeps and the notice when a key is refused (null, an empty record +reference, a duplicate: serialized the same as another key under the owning +document's serializer or equal by the rebuilt dictionary type's default +comparer, or a key that document cannot serialize), the fresh key an added entry gets, what a new list +element, dictionary value or union pick may be, and how an integer edit clamps +to its type. Edits work on a copy (`CultInspectorEdit`): a refused upsert +leaves the cached object unchanged. The Studio is one lowering of that model +and owns only widgets and layout; a runtime CultUI panel needs only its own +renderer and drawers to show and edit the same documents under the same rules. + +## Drawers + +Built in: every model shape above. A struct without `[Key]` members is edited +in place. With public fields it folds out to those fields only (readonly ones +read-only), so aliasing properties such as `Quaternion.eulerAngles` or +`Rect.min` add no rows; without public fields it folds out to its public +properties with a setter, `init` ones read-only (record structs and CultMath +matrices work as written). Get-only properties are never shown. Unity object +references draw as object fields. A value no drawer claims and no shape covers +shows a red error row with the model's reason, as does a multi-dimensional +array. + +A project adds or overrides a drawer with an editor class. `Claimed` is either +a value type (an open generic definition claims every closed form) or an +attribute type, which claims every member carrying it. Attribute claims win +over type claims, which win over built-in drawing. A claim two drawers make is +used by neither: both are logged and the member shows the error row. + +A drawer returns the new value and lets its IMGUI controls set `GUI.changed`; +only a frame that reports a change is saved. Mutating the value in place +without reporting a change is not saved reliably. A drawer that throws shows +an error row and discards that frame's edit copy, so nothing edited in that +frame is saved. + +```csharp +using System; +using System.Reflection; +using GameCult.Caching; +using GameCult.Unity.Caching.Editor; +using UnityEditor; +using UnityEngine; + +// Lives beside the documents; an attribute needs no Unity reference. +public sealed class ColorAttribute : Attribute +{ +} + +// Draws a "#RRGGBBAA" string member as a color field. +[CultInspectorDrawer(typeof(ColorAttribute))] +public sealed class ColorDrawer : ICultInspectorDrawer +{ + public object Draw(CultInspector inspector, string label, Type type, object value, MemberInfo member) + { + if (type != typeof(string)) + return inspector.DrawDefault(label, type, value, member); + ColorUtility.TryParseHtmlString(value as string ?? string.Empty, out var color); + var next = EditorGUILayout.ColorField(label, color); + return next == color ? value : "#" + ColorUtility.ToHtmlStringRGBA(next); + } +} +``` + +`Draw` receives the value's declared type and the member it belongs to, also +for list elements and dictionary keys and values, so an attribute drawer +checks the type it is handed. `inspector.DrawDefault` hands a value back to +built-in drawing; `inspector.DrawValue` draws a sub-value with claims applied. ## Inspector Annotations -Domain classes can opt into nicer Unity editor rendering with attributes from -`GameCult.Unity.Caching`: +The annotations live in `GameCult.Caching`, so headless shared models carry +them without referencing Unity. ```csharp using GameCult.Caching; -using GameCult.Unity.Caching; using MessagePack; +[MessagePackObject] [CultDocument("game.item", "game.item.v1")] public sealed class ItemData { @@ -47,27 +141,20 @@ public sealed class ItemData public string Notes = string.Empty; [Key(3)] - [CultInspectorAssetPath(typeof(UnityEngine.Texture2D))] + [CultInspectorAssetPath] public string IconPath = string.Empty; } ``` -Available annotations: - - `CultInspectorLabel` - `CultInspectorHidden` - `CultInspectorReadOnly` - `CultInspectorOrder` - `CultInspectorTextArea` - `CultInspectorRange` -- `CultInspectorAssetPath` - -## Current Scope - -The first studio pass supports local `.cc` files, explicit save/reload, adding -records with parameterless constructors, deleting records, primitive fields, -enums, strings, lists/arrays, nested objects, Unity object references, asset -path pickers, and `CultRecordRef` key editing. +- `CultInspectorAssetPath` (optional engine asset `Type`, named by the + consumer; the Studio uses it when it is a `UnityEngine.Object` type) +- `CultInspectorDrawer` (on drawer classes) CultMesh collaboration should feed the same CultCache mutation surface rather than becoming a second owner for document state. diff --git a/src/GameCult.Unity/Assets/Caching/Runtime.meta b/src/GameCult.Unity/Assets/Caching/Runtime.meta deleted file mode 100644 index 5142c828..00000000 --- a/src/GameCult.Unity/Assets/Caching/Runtime.meta +++ /dev/null @@ -1,3 +0,0 @@ -fileFormatVersion: 2 -guid: 5a7d06cb8f1e491785b3ae24034f1121 -timeCreated: 1779630000 diff --git a/src/GameCult.Unity/Assets/Caching/Runtime/CultCacheInspectorAttributes.cs.meta b/src/GameCult.Unity/Assets/Caching/Runtime/CultCacheInspectorAttributes.cs.meta deleted file mode 100644 index ecc92c68..00000000 --- a/src/GameCult.Unity/Assets/Caching/Runtime/CultCacheInspectorAttributes.cs.meta +++ /dev/null @@ -1,3 +0,0 @@ -fileFormatVersion: 2 -guid: 8871d3eb346242e3b0acfd6da5108676 -timeCreated: 1779630000 diff --git a/src/GameCult.Unity/Assets/Caching/Runtime/GameCult.Unity.Caching.asmdef b/src/GameCult.Unity/Assets/Caching/Runtime/GameCult.Unity.Caching.asmdef deleted file mode 100644 index 6e7e12a6..00000000 --- a/src/GameCult.Unity/Assets/Caching/Runtime/GameCult.Unity.Caching.asmdef +++ /dev/null @@ -1,14 +0,0 @@ -{ - "name": "GameCult.Unity.Caching", - "rootNamespace": "GameCult.Unity.Caching", - "references": [], - "includePlatforms": [], - "excludePlatforms": [], - "allowUnsafeCode": false, - "overrideReferences": false, - "precompiledReferences": [], - "autoReferenced": true, - "defineConstraints": [], - "versionDefines": [], - "noEngineReferences": true -} diff --git a/src/GameCult.Unity/Assets/Caching/Runtime/GameCult.Unity.Caching.asmdef.meta b/src/GameCult.Unity/Assets/Caching/Runtime/GameCult.Unity.Caching.asmdef.meta deleted file mode 100644 index 00ae3475..00000000 --- a/src/GameCult.Unity/Assets/Caching/Runtime/GameCult.Unity.Caching.asmdef.meta +++ /dev/null @@ -1,7 +0,0 @@ -fileFormatVersion: 2 -guid: ab6636c1402c4c579df9c07101542f13 -AssemblyDefinitionImporter: - externalObjects: {} - userData: - assetBundleName: - assetBundleVariant: diff --git a/src/GameCult.Unity/Assets/Caching/package.json b/src/GameCult.Unity/Assets/Caching/package.json index b08a3ce5..dff80c97 100644 --- a/src/GameCult.Unity/Assets/Caching/package.json +++ b/src/GameCult.Unity/Assets/Caching/package.json @@ -1,7 +1,7 @@ { "name": "org.gamecult.caching.unity", "displayName": "GameCult CultCache Unity Tools", - "version": "1.0.0", + "version": "1.2.0", "unity": "2021.3", "description": "Unity editor tooling for inspecting and editing CultCache .cc stores.", "keywords": [ @@ -9,6 +9,9 @@ "Editor", "Data" ], + "dependencies": { + "org.gamecult.cultlib": "1.0.57" + }, "license": "MIT", "category": "Scripting" } diff --git a/src/GameCult.Unity/Assets/UI/UI.txt b/src/GameCult.Unity/Assets/UI/UI.txt deleted file mode 100644 index 8789f73a..00000000 --- a/src/GameCult.Unity/Assets/UI/UI.txt +++ /dev/null @@ -1,2917 +0,0 @@ - - -// E:\Projects\CultLib\src\GameCult.Unity\Assets\UI\Attributes.cs - -using System; - -namespace GameCult.Unity.UI -{ - [AttributeUsage(AttributeTargets.All, Inherited = false)] - public class InspectableAttribute : Attribute { } - - [AttributeUsage(AttributeTargets.Field | AttributeTargets.Property)] - public abstract class PreferredInspectorAttribute : InspectableAttribute { } - - public class InspectableReadOnlyLabelAttribute : PreferredInspectorAttribute { } - - public class InspectableRangedFloatAttribute : PreferredInspectorAttribute - { - public readonly float Min, Max; - - public InspectableRangedFloatAttribute(float min, float max) - { - Min = min; - Max = max; - } - } - - public class InspectableRangedIntAttribute : PreferredInspectorAttribute - { - public readonly int Min, Max; - - public InspectableRangedIntAttribute(int min, int max) - { - Min = min; - Max = max; - } - } - - public class InspectableIncrementIntAttribute : PreferredInspectorAttribute - { - public readonly int Min, Max; - - public InspectableIncrementIntAttribute(int min, int max) - { - Min = min; - Max = max; - } - } -} - -// E:\Projects\CultLib\src\GameCult.Unity\Assets\UI\ColorConversion.cs - -using Unity.Mathematics; -using UnityEngine; -using static Unity.Mathematics.math; - -namespace GameCult.Unity.UI -{ - // Color conversion functions from https://www.chilliant.com/rgb2hsv.html - public static class ColorConversion - { - private static float Epsilon = 1e-10f; - - public static float3 HUEtoRGB(in float h) - { - float R = abs(h * 6 - 3) - 1; - float G = 2 - abs(h * 6 - 2); - float B = 2 - abs(h * 6 - 4); - return saturate(float3(R,G,B)); - } - - public static float3 RGBtoHCV(in float3 rgb) - { - // Based on work by Sam Hocevar and Emil Persson - float4 P = (rgb.y < rgb.z) ? float4(rgb.zy, -1.0f, 2.0f/3.0f) : float4(rgb.yz, 0.0f, -1.0f/3.0f); - float4 Q = (rgb.x < P.x) ? float4(P.xyw, rgb.x) : float4(rgb.x, P.yzx); - float C = Q.x - min(Q.w, Q.y); - float H = abs((Q.w - Q.y) / (6 * C + Epsilon) + Q.z); - return float3(H, C, Q.x); - } - - public static float3 HSVtoRGB(in float3 hsv) - { - float3 rgb = HUEtoRGB(hsv.x); - return ((rgb - 1) * hsv.y + 1) * hsv.z; - } - - public static float3 HSLtoRGB(in float3 hsl) - { - float3 RGB = HUEtoRGB(hsl.x); - float C = (1 - abs(2 * hsl.z - 1)) * hsl.y; - return (RGB - 0.5f) * C + hsl.z; - } - - // The weights of RGB contributions to luminance. - // Should sum to unity. - private static float3 HCYwts = float3(0.299f, 0.587f, 0.114f); - - public static float3 HCYtoRGB(float3 hcy) - { - float3 RGB = HUEtoRGB(hcy.x); - float Z = dot(RGB, HCYwts); - if (hcy.z < Z) - { - hcy.y *= hcy.z / Z; - } - else if (Z < 1) - { - hcy.y *= (1 - hcy.z) / (1 - Z); - } - return (RGB - Z) * hcy.y + hcy.z; - } - - public static float3 RGBtoHSV(in float3 rgb) - { - float3 HCV = RGBtoHCV(rgb); - float S = HCV.y / (HCV.z + Epsilon); - return float3(HCV.x, S, HCV.z); - } - - public static float3 RGBtoHSL(in float3 rgb) - { - float3 HCV = RGBtoHCV(rgb); - float L = HCV.z - HCV.y * 0.5f; - float S = HCV.y / (1 - abs(L * 2 - 1) + Epsilon); - return float3(HCV.x, S, L); - } - - public static float3 RGBtoHCY(in float3 rgb) - { - // Corrected by David Schaeffer - float3 HCV = RGBtoHCV(rgb); - float Y = dot(rgb, HCYwts); - float Z = dot(HUEtoRGB(HCV.x), HCYwts); - if (Y < Z) - { - HCV.y *= Z / (Epsilon + Y); - } - else - { - HCV.y *= (1 - Z) / (Epsilon + 1 - Y); - } - return float3(HCV.x, HCV.y, Y); - } - - public static Color ToColor(this float3 color) => new Color(color.x, color.y, color.z); - public static float3 ToFloat3(this Color color) => new float3(color.r, color.g, color.b); - } -} - -// E:\Projects\CultLib\src\GameCult.Unity\Assets\UI\DisplayOptions.cs - -using TMPro; - -namespace GameCult.Unity.UI -{ - public class DisplayOptions - { - public TextAlignmentOptions? TextAlignment { get; } - public VerticalAlignmentOptions? VerticalAlignment { get; } - public float? LayoutPriority { get; } - public float? LayoutHeight { get; } - public float? TextSize { get; } - public bool Interactable { get; } - public bool PlaceInContext { get; } - - public DisplayOptions(bool placeInContext = true, - bool interactable = true, - float? layoutHeight = null, - float? textSize = null, - float? layoutPriority = null, - VerticalAlignmentOptions? verticalAlignment = null, - TextAlignmentOptions? textAlignment = null) - { - PlaceInContext = placeInContext; - LayoutHeight = layoutHeight; - Interactable = interactable; - TextSize = textSize; - LayoutPriority = layoutPriority; - VerticalAlignment = verticalAlignment; - TextAlignment = textAlignment; - } - } -} - -// E:\Projects\CultLib\src\GameCult.Unity\Assets\UI\Generator.cs - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Linq.Expressions; -using System.Reflection; -using GameCult.Unity.UI.Components; -using UnityEngine; -using ZLinq; - -namespace GameCult.Unity.UI -{ - public class Generator : ResolverComponent, IUIContext - { - [SerializeField] private float inspectorLabelPriority = 3; - [SerializeField] private float inspectorFieldPriority = 2; - [SerializeField] private ReflectiveResolver? resolver; - [SerializeField] private RectTransform? contentRoot; - protected event Action? OnPropertyAdded; - protected event Action? RefreshPropertyValues; - - protected List Elements { get; } = new(); - public IUIResolver Resolver => resolver ?? throw new NullReferenceException("Generator doesn't have an assigned resolver."); - public Action? Refresh - { - get => RefreshPropertyValues; - set => RefreshPropertyValues = value; - } - - public RectTransform ContentRoot => contentRoot ?? (RectTransform)transform; - public Canvas Canvas => GetComponentInParent(); - public RectTransform CanvasRoot => (RectTransform)Canvas.transform; - - public void Inspect(object obj, bool inspectableOnly = false, bool recursive = true) - { - var members = obj.GetType().GetMembers(BindingFlags.Public | BindingFlags.Instance); - foreach (var member in members) - { - var inspectable = member.GetCustomAttribute(); - if (inspectable == null && inspectableOnly) continue; - var preferred = inspectable as PreferredInspectorAttribute; - switch (member) - { - case FieldInfo field: - { - var type = field.FieldType; - var handler = Resolver.ResolveField(type, preferred); - // If there's a handler for this field and attribute type, use that - if (handler != null) - { - var group = Resolver.Resolve()?.Configure(this); - if (group == null) continue; - Resolver.Resolve