GameCult/CultLib
Watch
9
0
Fork
You've already forked CultLib
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge main at cultlib-unity-v2.0.0 (42dce361) into C2a: one owner per collision

Browse source 
Identity: C2a's StoredSchemaId/StoredVersion owns; main's PersistedSchemaId, PersistedIdentity and both MarkWritten are deleted. Stamp rule: ReadStore judges the stamp before the body. Flush over damage refuses (merge-r1-precedence). Mesh single-file writes route through SingleFileMessagePackBackingStore. TS: store-file.ts commits, C2a's header handling kept. Rust: C2a's readable_store_format in front of the grown decoder. Build not yet verified at this commit.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
Meta Vens 2026-10-22 20:21:31 +02:00
commit f2b3569822
210 changed files with 27332 additions and 899 deletions
Download patch file Download diff file Expand all files Collapse all files

41
.github/workflows/cultmesh-portability.yml vendored
View file

@ -101,8 +101,10 @@ jobs:
# pull requests: cultnet-interop.yml is windows-latest and main-only, so
# until these steps existed the shared verifier's tests never ran on
# Linux at all, and the browser client's never ran in CI on any OS.
# packages/cultnet-ts verifies both vector files; cultmesh-browser proves
# the browser client reads the shared rules rather than its own.
# packages/cultnet-ts verifies both vector files (content.test.ts judges both
# C# content files, the Rust one and its own); cultmesh-browser proves
# the browser client reads the shared rules rather than its own, and that its content reader releases only verified
# bytes, requesting the same URLs as the C# HTTPS connector.
# No workflow ran cultcache-ts's tests, and its test script listed two files by hand while a third sat compiled
# and unrun. The wiring check fails when any TypeScript test file is run by no script, and the cache's own suite
# runs here beside the packages built on it.
@ -112,21 +114,29 @@ jobs:
- name: Verify the CultCache document owner with the TypeScript cache
run: npm run test --workspace packages/cultcache-ts
# Packs the four TypeScript packages and installs them into a clean consumer, so a package that builds but does
# not install or load fails here.
- name: Verify the packed TypeScript packages install and load from a clean consumer
run: npm run test:pack:ts
- name: Verify the CultMesh authority rules with the TypeScript verifier
run: npm run test --workspace packages/cultnet-ts
- name: Verify the browser client against the shared TypeScript verifier
run: npm run test --workspace packages/cultmesh-browser
# The realtime frame codec. Its byte parity rests on two vector files,
# cultmesh-browser's test compiles test/interop, which imports cultmesh-ts/content, so cultmesh-ts builds first,
# as in scripts/run-typescript-workspaces.mjs.
# The realtime frame codec and the HTTP content chunk provider (content.test.ts,
# its answers over node:http). The codec's byte parity rests on two vector files,
# contracts/cultmesh/realtime-frame-vectors.json written by the C#
# reference and decoded by this step, and realtime-frame-vectors.ts-written.json
# written by this codec and decoded by the C# step below. Neither half ran
# in CI on any OS until these two steps existed, so a codec that agreed
# only with itself would have gone green.
- name: Verify the realtime frame codec with the TypeScript codec
- name: Verify the realtime frame codec and the content provider with the TypeScript runtime
run: npm run test --workspace packages/cultmesh-ts
- name: Verify the browser client against the shared TypeScript verifier
run: npm run test --workspace packages/cultmesh-browser
# Both directions of the CultMesh authority vectors: the C#-written
# file (contracts/cultmesh/authority-route-vectors.json) and the
# TypeScript-signed file (authority-route-vectors.ts-signed.json) are
@ -147,3 +157,20 @@ jobs:
dotnet test tests/GameCult.Mesh.Tests/GameCult.Mesh.Tests.csproj
--filter FullyQualifiedName~RealtimeWireProtocol
--logger "console;verbosity=minimal"
# The content plane: the reference writes the manifest and record-key
# vectors and judges the manifests the TypeScript packer wrote
# (content-vectors.ts-written.json). This suite also holds the C#/Rust
# content parity, which no CI step ran before this one. CULTNET_WRITE_VECTORS
# is never set in CI: the committed files are the contract.
- name: Verify the content vectors with the C# reference
run: >-
dotnet test tests/GameCult.Mesh.Tests/GameCult.Mesh.Tests.csproj
--filter FullyQualifiedName~CultMeshContentVectorTests
--logger "console;verbosity=minimal"
# The cross-process HTTPS content parity: the C# transfer service over the unmodified HTTPS connector and the
# TypeScript reader fetch from the TypeScript provider in its own process. Both refuse bytes tampered in flight,
# and the provider refuses a corrupt store.
- name: Verify HTTPS content parity across C#, the TypeScript provider and the TypeScript reader
run: npm run test:interop --workspace packages/cultmesh-browser

702
contracts/cultmesh/cdn-record-key-vectors.json Normal file
View file

@ -0,0 +1,702 @@
{
"artifacts": [
{
"artifactIdUnits": [
101,
117,
114,
101,
107,
97,
47,
99,
111,
110,
116,
101,
110,
116,
45,
118,
101,
99,
116,
111,
114
],
"versionUnits": [],
"contentHash": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
"recordKey": "mesh:cdn:artifact:eureka-content-vector:unversioned:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
},
{
"artifactIdUnits": [
32,
32,
77,
105,
120,
101,
100,
32,
67,
97,
115,
101,
47,
80,
97,
116,
104,
32,
32
],
"versionUnits": [
32,
32,
32
],
"contentHash": "SHA256:E3B0C44298FC1C149AFBF4C8996FB92427AE41E4649B934CA495991B7852B855",
"recordKey": "mesh:cdn:artifact:mixed-case-path:unversioned:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
},
{
"artifactIdUnits": [
304,
115,
116,
97,
110,
98,
117,
108
],
"versionUnits": [
49,
46,
48,
46,
48
],
"contentHash": " e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 ",
"recordKey": "mesh:cdn:artifact:\u0130stanbul:1.0.0:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
},
{
"artifactIdUnits": [
913,
931
],
"versionUnits": [
86,
50,
32,
66,
101,
116,
97
],
"contentHash": " sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
"recordKey": "mesh:cdn:artifact:\u03B1\u03C3:v2-beta:sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
},
{
"artifactIdUnits": [
65279,
98,
111,
109
],
"versionUnits": [
252
],
"contentHash": "sha256:sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
"recordKey": "mesh:cdn:artifact:bom:\u00FC:sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
},
{
"artifactIdUnits": [
133,
110,
101,
108,
133
],
"versionUnits": [],
"contentHash": "Sha256:\u0130\u03A3\uD801\uDC00",
"recordKey": "mesh:cdn:artifact:nel:unversioned:\u0130\u03C3\uD801\uDC28"
},
{
"artifactIdUnits": [
26085,
26412,
35486,
47,
36039,
29987
],
"versionUnits": [
32,
32,
32
],
"contentHash": "sha256:",
"recordKey": "mesh:cdn:artifact:\u65E5\u672C\u8A9E-\u8CC7\u7523:unversioned:"
},
{
"artifactIdUnits": [
55357,
56898,
32,
115,
109,
105,
108,
101
],
"versionUnits": [
49,
46,
48,
46,
48
],
"contentHash": "\u017Fha256:abc",
"recordKey": "mesh:cdn:artifact:smile:1.0.0:\u017Fha256:abc"
},
{
"artifactIdUnits": [
45,
45,
45
],
"versionUnits": [
86,
50,
32,
66,
101,
116,
97
],
"contentHash": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
"recordKey": "mesh:cdn:artifact:cb3f91d54eee30e53e35b2b99905f70f169ed549fd78909d3dac2defc9ed8d3b:v2-beta:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
},
{
"artifactIdUnits": [
97,
55296,
98
],
"versionUnits": [
252
],
"contentHash": "SHA256:E3B0C44298FC1C149AFBF4C8996FB92427AE41E4649B934CA495991B7852B855",
"recordKey": "mesh:cdn:artifact:a-b:\u00FC:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
},
{
"artifactIdUnits": [
55296
],
"versionUnits": [],
"contentHash": " e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 ",
"recordKey": "mesh:cdn:artifact:83d544ccc223c057d2bf80d3f2a32982c32c3c0db8e2674820da5064783fb097:unversioned:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
},
{
"artifactIdUnits": [
120,
178,
121
],
"versionUnits": [
32,
32,
32
],
"contentHash": " sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
"recordKey": "mesh:cdn:artifact:x-y:unversioned:sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
},
{
"artifactIdUnits": [
1635,
100
],
"versionUnits": [
49,
46,
48,
46,
48
],
"contentHash": "sha256:sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
"recordKey": "mesh:cdn:artifact:\u0663d:1.0.0:sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
},
{
"artifactIdUnits": [
55297,
56320,
65
],
"versionUnits": [
86,
50,
32,
66,
101,
116,
97
],
"contentHash": "Sha256:\u0130\u03A3\uD801\uDC00",
"recordKey": "mesh:cdn:artifact:a:v2-beta:\u0130\u03C3\uD801\uDC28"
},
{
"artifactIdUnits": [
32,
32,
45,
45,
45,
32,
32
],
"versionUnits": [
252
],
"contentHash": "sha256:",
"recordKey": "mesh:cdn:artifact:fcd2a2fb27957aa9a10d2d9eca16ccbc9c35a76933b03f470211d327f6946c3f:\u00FC:"
},
{
"artifactIdUnits": [
9398
],
"versionUnits": [],
"contentHash": "\u017Fha256:abc",
"recordKey": "mesh:cdn:artifact:7d9633ae307dae04a8b362df41dca4b38a653823d69f27a259e7b6928508d3a6:unversioned:\u017Fha256:abc"
},
{
"artifactIdUnits": [
8555
],
"versionUnits": [
32,
32,
32
],
"contentHash": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
"recordKey": "mesh:cdn:artifact:8c0e55c55f47de02406174ec976279a977e2e73ea5aec8ab90f94c40a6ab2d60:unversioned:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
},
{
"artifactIdUnits": [
101,
117,
114,
101,
107,
97,
47,
99,
111,
110,
116,
101,
110,
116,
45,
118,
101,
99,
116,
111,
114
],
"versionUnits": [],
"contentHash": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
"recordKey": "mesh:cdn:artifact:eureka-content-vector:unversioned:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
},
{
"artifactIdUnits": [
101,
117,
114,
101,
107,
97,
47,
99,
111,
110,
116,
101,
110,
116,
45,
118,
101,
99,
116,
111,
114
],
"versionUnits": [
32,
32,
32
],
"contentHash": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
"recordKey": "mesh:cdn:artifact:eureka-content-vector:unversioned:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
},
{
"artifactIdUnits": [
101,
117,
114,
101,
107,
97,
47,
99,
111,
110,
116,
101,
110,
116,
45,
118,
101,
99,
116,
111,
114
],
"versionUnits": [
49,
46,
48,
46,
48
],
"contentHash": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
"recordKey": "mesh:cdn:artifact:eureka-content-vector:1.0.0:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
},
{
"artifactIdUnits": [
101,
117,
114,
101,
107,
97,
47,
99,
111,
110,
116,
101,
110,
116,
45,
118,
101,
99,
116,
111,
114
],
"versionUnits": [
86,
50,
32,
66,
101,
116,
97
],
"contentHash": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
"recordKey": "mesh:cdn:artifact:eureka-content-vector:v2-beta:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
},
{
"artifactIdUnits": [
101,
117,
114,
101,
107,
97,
47,
99,
111,
110,
116,
101,
110,
116,
45,
118,
101,
99,
116,
111,
114
],
"versionUnits": [
252
],
"contentHash": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
"recordKey": "mesh:cdn:artifact:eureka-content-vector:\u00FC:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
},
{
"artifactIdUnits": [
101,
117,
114,
101,
107,
97,
47,
99,
111,
110,
116,
101,
110,
116,
45,
118,
101,
99,
116,
111,
114
],
"versionUnits": [],
"contentHash": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
"recordKey": "mesh:cdn:artifact:eureka-content-vector:unversioned:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
},
{
"artifactIdUnits": [
101,
117,
114,
101,
107,
97,
47,
99,
111,
110,
116,
101,
110,
116,
45,
118,
101,
99,
116,
111,
114
],
"versionUnits": [],
"contentHash": "SHA256:E3B0C44298FC1C149AFBF4C8996FB92427AE41E4649B934CA495991B7852B855",
"recordKey": "mesh:cdn:artifact:eureka-content-vector:unversioned:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
},
{
"artifactIdUnits": [
101,
117,
114,
101,
107,
97,
47,
99,
111,
110,
116,
101,
110,
116,
45,
118,
101,
99,
116,
111,
114
],
"versionUnits": [],
"contentHash": " e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 ",
"recordKey": "mesh:cdn:artifact:eureka-content-vector:unversioned:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
},
{
"artifactIdUnits": [
101,
117,
114,
101,
107,
97,
47,
99,
111,
110,
116,
101,
110,
116,
45,
118,
101,
99,
116,
111,
114
],
"versionUnits": [],
"contentHash": " sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
"recordKey": "mesh:cdn:artifact:eureka-content-vector:unversioned:sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
},
{
"artifactIdUnits": [
101,
117,
114,
101,
107,
97,
47,
99,
111,
110,
116,
101,
110,
116,
45,
118,
101,
99,
116,
111,
114
],
"versionUnits": [],
"contentHash": "sha256:sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
"recordKey": "mesh:cdn:artifact:eureka-content-vector:unversioned:sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
},
{
"artifactIdUnits": [
101,
117,
114,
101,
107,
97,
47,
99,
111,
110,
116,
101,
110,
116,
45,
118,
101,
99,
116,
111,
114
],
"versionUnits": [],
"contentHash": "Sha256:\u0130\u03A3\uD801\uDC00",
"recordKey": "mesh:cdn:artifact:eureka-content-vector:unversioned:\u0130\u03C3\uD801\uDC28"
},
{
"artifactIdUnits": [
101,
117,
114,
101,
107,
97,
47,
99,
111,
110,
116,
101,
110,
116,
45,
118,
101,
99,
116,
111,
114
],
"versionUnits": [],
"contentHash": "sha256:",
"recordKey": "mesh:cdn:artifact:eureka-content-vector:unversioned:"
},
{
"artifactIdUnits": [
101,
117,
114,
101,
107,
97,
47,
99,
111,
110,
116,
101,
110,
116,
45,
118,
101,
99,
116,
111,
114
],
"versionUnits": [],
"contentHash": "\u017Fha256:abc",
"recordKey": "mesh:cdn:artifact:eureka-content-vector:unversioned:\u017Fha256:abc"
}
],
"chunks": [
{
"chunkHash": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
"recordKey": "mesh:cdn:chunk:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
},
{
"chunkHash": "SHA256:E3B0C44298FC1C149AFBF4C8996FB92427AE41E4649B934CA495991B7852B855",
"recordKey": "mesh:cdn:chunk:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
},
{
"chunkHash": " e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 ",
"recordKey": "mesh:cdn:chunk:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
},
{
"chunkHash": " sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
"recordKey": "mesh:cdn:chunk:sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
},
{
"chunkHash": "sha256:sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
"recordKey": "mesh:cdn:chunk:sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
},
{
"chunkHash": "Sha256:\u0130\u03A3\uD801\uDC00",
"recordKey": "mesh:cdn:chunk:\u0130\u03C3\uD801\uDC28"
},
{
"chunkHash": "sha256:",
"recordKey": "mesh:cdn:chunk:"
},
{
"chunkHash": "\u017Fha256:abc",
"recordKey": "mesh:cdn:chunk:\u017Fha256:abc"
}
]
}

1873
contracts/cultmesh/content-acceptance-vectors.json Normal file
View file

File diff suppressed because it is too large Load diff

120
contracts/cultmesh/content-vectors.ts-written.json Normal file
View file

@ -0,0 +1,120 @@
{
"vectors": [
{
"label": "ts multi-chunk body",
"kind": "manifest",
"input": {
"artifactId": "Pixel Heroes/Hüt ☃",
"kind": "asset",
"version": "",
"mimeType": "application/x-cultnet-ts-vector",
"createdAtUtc": "2026-10-19T00:00:00.0000000+00:00",
"chunkSizeBytes": 131072,
"tags": [
"a",
"b"
],
"metadata": [
[
"10",
"x"
],
[
"alpha",
"y"
],
[
"2",
"z"
]
],
"body": {
"length": 777777,
"multiplier": 7,
"modulus": 256
}
},
"recordKey": "mesh:cdn:artifact:pixel-heroes-hüt:unversioned:cf8a453a36981321977355448d24df49bcefee22be6e3730b9140e87c29013f9",
"messagePackBase64": "mrVQaXhlbCBIZXJvZXMvSMO8dCDimIOlYXNzZXSg2UBjZjhhNDUzYTM2OTgxMzIxOTc3MzU1NDQ4ZDI0ZGY0OWJjZWZlZTIyYmU2ZTM3MzBiOTE0MGU4N2MyOTAxM2Y5zgAL3jG/YXBwbGljYXRpb24veC1jdWx0bmV0LXRzLXZlY3RvctkhMjAyNi0xMC0xOVQwMDowMDowMC4wMDAwMDAwKzAwOjAwlpTZQDM3N2M3YjBjYzIyOWY0NzA1NjcwZDYzZDU0NGMyNzViNjYzZGY2ZThjODMxYjA5ODQ2YTVjNDhjY2EwOGI4NDgAzgACAADZT21lc2g6Y2RuOmNodW5rOjM3N2M3YjBjYzIyOWY0NzA1NjcwZDYzZDU0NGMyNzViNjYzZGY2ZThjODMxYjA5ODQ2YTVjNDhjY2EwOGI4NDiU2UAzNzdjN2IwY2MyMjlmNDcwNTY3MGQ2M2Q1NDRjMjc1YjY2M2RmNmU4YzgzMWIwOTg0NmE1YzQ4Y2NhMDhiODQ4zgACAADOAAIAANlPbWVzaDpjZG46Y2h1bms6Mzc3YzdiMGNjMjI5ZjQ3MDU2NzBkNjNkNTQ0YzI3NWI2NjNkZjZlOGM4MzFiMDk4NDZhNWM0OGNjYTA4Yjg0OJTZQDM3N2M3YjBjYzIyOWY0NzA1NjcwZDYzZDU0NGMyNzViNjYzZGY2ZThjODMxYjA5ODQ2YTVjNDhjY2EwOGI4NDjOAAQAAM4AAgAA2U9tZXNoOmNkbjpjaHVuazozNzdjN2IwY2MyMjlmNDcwNTY3MGQ2M2Q1NDRjMjc1YjY2M2RmNmU4YzgzMWIwOTg0NmE1YzQ4Y2NhMDhiODQ4lNlAMzc3YzdiMGNjMjI5ZjQ3MDU2NzBkNjNkNTQ0YzI3NWI2NjNkZjZlOGM4MzFiMDk4NDZhNWM0OGNjYTA4Yjg0OM4ABgAAzgACAADZT21lc2g6Y2RuOmNodW5rOjM3N2M3YjBjYzIyOWY0NzA1NjcwZDYzZDU0NGMyNzViNjYzZGY2ZThjODMxYjA5ODQ2YTVjNDhjY2EwOGI4NDiU2UAzNzdjN2IwY2MyMjlmNDcwNTY3MGQ2M2Q1NDRjMjc1YjY2M2RmNmU4YzgzMWIwOTg0NmE1YzQ4Y2NhMDhiODQ4zgAIAADOAAIAANlPbWVzaDpjZG46Y2h1bms6Mzc3YzdiMGNjMjI5ZjQ3MDU2NzBkNjNkNTQ0YzI3NWI2NjNkZjZlOGM4MzFiMDk4NDZhNWM0OGNjYTA4Yjg0OJTZQGEwYmI2OWY0NDg2MWViM2RiNTYwMjZhMWFiYmY5MWZmYTYyMTMyNTA4ZDVhMDZlMGQwZGRjNWNjMjZiZTMzZmPOAAoAAM4AAd4x2U9tZXNoOmNkbjpjaHVuazphMGJiNjlmNDQ4NjFlYjNkYjU2MDI2YTFhYmJmOTFmZmE2MjEzMjUwOGQ1YTA2ZTBkMGRkYzVjYzI2YmUzM2ZjkqFhoWKDojEwoXilYWxwaGGheaEyoXo="
},
{
"label": "ts empty body",
"kind": "manifest",
"input": {
"artifactId": "Pixel Heroes/Hüt ☃",
"kind": "asset",
"version": "",
"mimeType": "application/x-cultnet-ts-vector",
"createdAtUtc": "2026-10-19T00:00:00.0000000+00:00",
"chunkSizeBytes": 131072,
"tags": [
"a",
"b"
],
"metadata": [
[
"10",
"x"
],
[
"alpha",
"y"
],
[
"2",
"z"
]
],
"body": {
"length": 0,
"multiplier": 7,
"modulus": 256
}
},
"recordKey": "mesh:cdn:artifact:pixel-heroes-hüt:unversioned:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
"messagePackBase64": "mrVQaXhlbCBIZXJvZXMvSMO8dCDimIOlYXNzZXSg2UBlM2IwYzQ0Mjk4ZmMxYzE0OWFmYmY0Yzg5OTZmYjkyNDI3YWU0MWU0NjQ5YjkzNGNhNDk1OTkxYjc4NTJiODU1AL9hcHBsaWNhdGlvbi94LWN1bHRuZXQtdHMtdmVjdG9y2SEyMDI2LTEwLTE5VDAwOjAwOjAwLjAwMDAwMDArMDA6MDCRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAA2U9tZXNoOmNkbjpjaHVuazplM2IwYzQ0Mjk4ZmMxYzE0OWFmYmY0Yzg5OTZmYjkyNDI3YWU0MWU0NjQ5YjkzNGNhNDk1OTkxYjc4NTJiODU1kqFhoWKDojEwoXilYWxwaGGheaEyoXo="
},
{
"label": "ts single chunk, versioned",
"kind": "manifest",
"input": {
"artifactId": "Pixel Heroes/Hüt ☃",
"kind": "asset",
"version": "V1 Beta",
"mimeType": "application/x-cultnet-ts-vector",
"createdAtUtc": "2026-10-19T00:00:00.0000000+00:00",
"chunkSizeBytes": 4194304,
"tags": [],
"metadata": [],
"body": {
"length": 1000,
"multiplier": 7,
"modulus": 256
}
},
"recordKey": "mesh:cdn:artifact:pixel-heroes-hüt:v1-beta:89f4ff56a25dd1db06a4ce6033603775d705fb96f30f8693733fef602a1ca532",
"messagePackBase64": "mrVQaXhlbCBIZXJvZXMvSMO8dCDimIOlYXNzZXSnVjEgQmV0YdlAODlmNGZmNTZhMjVkZDFkYjA2YTRjZTYwMzM2MDM3NzVkNzA1ZmI5NmYzMGY4NjkzNzMzZmVmNjAyYTFjYTUzMs0D6L9hcHBsaWNhdGlvbi94LWN1bHRuZXQtdHMtdmVjdG9y2SEyMDI2LTEwLTE5VDAwOjAwOjAwLjAwMDAwMDArMDA6MDCRlNlAODlmNGZmNTZhMjVkZDFkYjA2YTRjZTYwMzM2MDM3NzVkNzA1ZmI5NmYzMGY4NjkzNzMzZmVmNjAyYTFjYTUzMgDNA+jZT21lc2g6Y2RuOmNodW5rOjg5ZjRmZjU2YTI1ZGQxZGIwNmE0Y2U2MDMzNjAzNzc1ZDcwNWZiOTZmMzBmODY5MzczM2ZlZjYwMmExY2E1MzKQgA=="
},
{
"label": "ts whitespace version, padded kind and mime",
"kind": "manifest",
"input": {
"artifactId": "Pixel Heroes/Hüt ☃",
"kind": " build ",
"version": " ",
"mimeType": " text/plain; charset=utf-8 ",
"createdAtUtc": "2026-10-19T00:00:00.0000000+00:00",
"chunkSizeBytes": 4194304,
"tags": [],
"metadata": [],
"body": {
"length": 1000,
"multiplier": 7,
"modulus": 256
}
},
"recordKey": "mesh:cdn:artifact:pixel-heroes-hüt:unversioned:89f4ff56a25dd1db06a4ce6033603775d705fb96f30f8693733fef602a1ca532",
"messagePackBase64": "mrVQaXhlbCBIZXJvZXMvSMO8dCDimIOnIGJ1aWxkIKIgINlAODlmNGZmNTZhMjVkZDFkYjA2YTRjZTYwMzM2MDM3NzVkNzA1ZmI5NmYzMGY4NjkzNzMzZmVmNjAyYTFjYTUzMs0D6L0gIHRleHQvcGxhaW47IGNoYXJzZXQ9dXRmLTggINkhMjAyNi0xMC0xOVQwMDowMDowMC4wMDAwMDAwKzAwOjAwkZTZQDg5ZjRmZjU2YTI1ZGQxZGIwNmE0Y2U2MDMzNjAzNzc1ZDcwNWZiOTZmMzBmODY5MzczM2ZlZjYwMmExY2E1MzIAzQPo2U9tZXNoOmNkbjpjaHVuazo4OWY0ZmY1NmEyNWRkMWRiMDZhNGNlNjAzMzYwMzc3NWQ3MDVmYjk2ZjMwZjg2OTM3MzNmZWY2MDJhMWNhNTMykIA="
}
]
}

63
docs/cultlib-gaps-map.md
View file

@ -27,6 +27,10 @@ copy of the solver and the warm/cold sweep, `suite` a scratch copy of `packages/
key tests read C#-written bytes instead of literals.
7. `decode-stack-safety` (section D): cultcache-rs's one decode door grows its stack at every container level
(ruling `decode-stack-mechanism-ruled`). Independent of 5 and 6; thing2's contract-rs merge waits on it.
8. `expect-persisted-schema` (section E): an observed entry keeps the schema id it was persisted under and
conditional commit compares that (ruling `aetheria-release:ruling:expect-fix-in-cultcache`). C# only. The
release that carries it to Aetheria follows question `cultlib-gaps:question:expect-fix-release-lane`; the
Aetheria pin bump is an Aetheria follow-on, not this cut.
## B. BoundedLeastSquares stopping
@ -684,6 +688,65 @@ door-checked `rmpv::Value`s that way (follow-up `rs-redecode-outside-door-r2`).
Rejected inside the ruling: the derive placement (above), and writing the wrapper by hand (D9: established
tooling, one owner upstream).
## E. Expect under compatible schema drift (cut `expect-persisted-schema`)
Pinned base: `origin/main` at `b549d731` (2026-10-21). Imagination pass `imagination-expect`, session
`self-2026-10-19-ag7`. Probe `probe1` (`ProbeExpectDrift.cs`, written into the clone by the job command, never
committed) ran on Yggdrasil, dotnet sdk 10.0, against `b549d731`. Fixture: a record written under
`DriftWide` (`tests.variant_drift.v1`) and read by a cache whose registry holds only `DriftNarrow` (v2) loads
as CompatibleDrift.
### Body facts
- **E1. The defect.** `CultCommitRequest.ConditionsHold` (`CultCache.cs:1677-1696`) compares the durable
record's `SchemaId` with the observed entry's `Descriptor.SchemaId` (`Expect` captures it at `:1772`;
`ExpectUnchanged` reads it at `:1694`). `ToStoredDocument` (`:3212-3268`) resolves a persisted record to the
local descriptor and drops `record.SchemaId`. So the observed side holds the local id and the durable side
the persisted id. Probe: unchanged drifted record, `Expect` + `Upsert` commits **False** on the single-file
store (P1) and on the directory store (P7). Same code at Aetheria's pin `45c2f40` (`CultCache.cs:1147-1207`).
- **E2. What must still refuse, at base.** Another cache commits a changed value (new minted `storedAt`):
Expect False (P4). Another cache reading under v3 (`DriftReused`) flushes the file, rewriting the record under
v3 with the **same** `storedAt`: Expect False (P5).
- **E3. A whole-file write rewrites every record under its local schema and keeps `storedAt`.**
`ToPersistedRecord` (`:3194-3204`) writes `entry.Descriptor.SchemaId` with `entry.StoredAt`, and both the
single-file `PushAll` (`:3361-3375`) and an unconditional or dirty `CommitBatchCore` (`:3393-3399`) write the
whole view. So the contract sentence "every write to a key mints a later storedAt"
(`Contracts/cultcache-store-composition.md:229-231`) is false for a flush: the schema id is the half of the
identity that sees it. The directory store writes only staged keys, each freshly stamped
(`DirectoryMessagePackBackingStore.cs:240-293`), so it never normalises a clean record.
- **E4. Lifetime at base.** After the cache's own `PushAll`, Expect commits True (P2): disk and descriptor now
agree. After *another* cache's normalising flush to the observer's own local schema, Expect commits True (P3);
after a re-pull, True (P6).
- **E5. Re-reads.** Both pull filters skip a record whose `storedAt` and local descriptor id are unchanged
(`CultCache.cs:3323-3327`, `DirectoryMessagePackBackingStore.cs:110-114`). A record normalised P to L by
another writer keeps `storedAt` and the local id, so with the persisted id on the entry the filter must
compare the persisted id, or a re-read never refreshes it and Expect stays a Mismatch forever.
- **E6. Other runtimes.** cultcache-rs `SingleFileMessagePackBackingStore::compare_exchange` and the snapshot
form (`packages/cultcache-rs/src/lib.rs:1352-1393`, `:1427-1450`) compare whole raw envelopes as read from
disk (`schema_id: Some(record.1)`, `:3189`); the expectation *is* the persisted envelope, and the crate has
no compatible-drift resolution, so the split cannot occur. Unchanged. cultcache-ts and cultcache-py have no
conditional commit (`cultcache-persistence-format.md:67-69`; grep finds none); nor do the cultnet and
cultmesh packages or cultmesh-kotlin. Not added here.
- **E7. Release path.** Aetheria pins `org.gamecult.cultlib#cultlib-unity-v1.0.60` and
`org.gamecult.caching.unity#caching-unity-v1.4.0`, both locked at `45c2f400` (2026-09-17). The CultCache
runtime ships as DLLs in `unity/org.gamecult.cultlib` (built by `scripts/build-unity-package.ps1`). Main is
1363 commits past the pin with a `### Breaking` section under `[Unreleased]`
(`unity/org.gamecult.cultlib/CHANGELOG.md`). `release/cultlib-unity-1.0.x` is the pin plus one commit
(1.0.61, Win64 QUIC plugins).
### Authority, and why
The backing store that read or wrote a record's durable bytes owns the schema id it is persisted under. It is set
when the store builds the entry from a persisted record (`ToStoredDocument`), defaults to the descriptor's id
for an entry this cache stamps (it is about to be written under that id), and is reset to the descriptor's id,
under the hold, after a whole-file write the store completed. The commit check, `Expect` and the pull filters
only read it. `Descriptor.SchemaId` stays what the entry decodes as; it is no longer the record's durable identity.
Consequence, recorded rather than ruled: after another writer's whole-file flush rewrites a drifted record
under a schema id this cache did not read it under, an `Expect` captured before is a Mismatch until the cache
re-reads (E4's P3 flips). That is the ruling's "compares that persisted id" applied to a rewrite that kept
`storedAt`; it is fail-closed, the standard retry (re-read, retry) recovers, and it is what refuses E2's P5.
## Observed, not owned by these cuts
- At the default tolerance the solver's answer on allocator-shaped problems depends on the start by up to the whole

707
docs/cultmath-interval-ground-cut.md
View file

@ -1,6 +1,7 @@
# CultMath intervals and the gamecult.org ground: cut map
Status, 2026-10-04: interval-ops, glsl-lowering, snoise-tie (48aadda0) and affine-forms with
Status, 2026-10-14: the target is r2 (section "Target r2"); snoise-hessian, expr-ir and
cycles-converters are on main. Earlier status, 2026-10-04: interval-ops, glsl-lowering, snoise-tie (48aadda0) and affine-forms with
affine-shaders (156782da) are on main; the as-built record is the last section. Figures quoted in
Passes 1-5 at SNOISE_LIPSCHITZ 10.099261 predate the tie fix (now 7.9640074) and are conservative;
design.md carries the current measurements. Map history: Imagination pass 4, 2026-10-03. Pass 1's Q1–Q3 are answered (below);
@ -8,7 +9,7 @@ the Self opened the `cultmath-tapes` campaign from pass 2; pass 3 revises the
two CultLib cuts after their first Hands reports and the operator's challenge
to the march, and changes the site field to envelope-then-noise; pass 4
(section "Pass 4") revises `interval-ops` to r3 after its second Hands report
and Soul's verdict, and names the one shipped void (Q9). Pass 5 (section "Pass 5") maps
and Soul's verdict, and names the one shipped void (Q9, ruled `operator-shipped-void`). Pass 5 (section "Pass 5") maps
affine forms against CultLib `main` at `a7966142`. Where a later pass and
an earlier section disagree, the later pass wins. Where this map and the Body
disagree, the Body wins and this map is stale.
@ -86,25 +87,181 @@ pane and read back. CultLib has no GPU harness (follow-up
### Campaign
`cultmath-tapes` (Q1), repos `GameCult/CultLib` and `GameCult/gamecult-site`,
target doc `docs/cultmath-tape-target.md` unparked for step 1 only. The site
cuts cite `site-masthead:ruling:operator-chromeless-column` and depend on
`wash-home-only` landing first, because they rewrite the same lines.
`cultmath-tapes`, repos `GameCult/CultLib` and `GameCult/gamecult-site`, in the
Eureka Self area (ruling `adopted-by-eureka-self-area`). The tape target doc
`docs/cultmath-tape-target.md` has steps 1 (intervals), 2 (graph and C#
evaluator) and 4 (HLSL interpreter) unparked, with affine forms beside step 1
(rulings `affine-forms-unparked`, `material-ir-cultmath-now`,
`tape-unparked-for-trophic-graphs`). Steps 3 (pruning) and 5 (meshing) stay
parked. The volumetric line (the culled march's proofs, the integration-native
noise domain, the site ground) leads the campaign (ruling
`volumetric-work-prioritized`). Site cuts cite
`site-masthead:ruling:operator-chromeless-column`.
### Owner, consumers, invariant
### Target r2: owners, consumers, invariants
- Owner: CultMath owns interval arithmetic and the noise bounds in C# and HLSL,
GLSL derived. The site owns its field, flow, march, TAA, stardust, DoF and
budget controller. `custom.scss` owns the brand palette the shader reads
(four nebula tokens) and the seed values of the ground tokens. `ground.js`
owns the sampled ground tokens while it runs.
- Consumers: the site ground now; Aetheria's nebula raymarch at its CultMath
pin bump; the tapes campaign's HLSL interpreter later.
- Protected invariant, **`intervals-enclose`**: every interval function returns
a `[lo, hi]` containing `f(x)` for every `x` in its input region, in C#, HLSL
and GLSL alike. A consumer that skips a region because `hi < cutoff` has
skipped nothing that was there. The domain warp does not weaken it: a warped
point lies in the ball enlarged by the warp bound, and a test says so.
**Owners.** CultMath owns every function a consumer evaluates and every bound on
it: the snoise kernel and its derivatives, the noise pins, intervals, affine
forms, the warp-chain stage functions and their span bounds, the closed-form span
integral, the Cycles converters, the trophic node primitives, and the expression
graph with its evaluator and interpreter. C# is the reference; the HLSL mirrors it;
the GLSL is generated. `GameCult.Caching.MessagePack` owns the wire shape of CultMath
values and graphs. A consumer owns its composition and nothing below it: the site
owns its field's brush, ramp, chain order and parameters, its motion, TAA,
stardust, DoF, budget controller and sampled tokens; Aetheria owns its species
graphs, their settings and when they are evaluated. `custom.scss` owns the four
nebula input tokens and the seed values of the ground tokens.
**Consumers.**
- The gamecult.org ground: the Span Lab framework with the void as its brush
(ruling `site-ground-field-ruled`), in WebGL2 GLSL, vendoring CultMath's
generated GLSL by CultLib commit.
- Aetheria fauna (`aetheria-fauna:target:r4`): trophic species graphs as
`CultMath.Expr` documents, evaluated by the C# evaluator in ServerShared and by
the HLSL interpreter in its presenters, over CultMath's node primitives.
- Aetheria's Cycles-matched bakes (`aetheria-release`): reference bakes compared
with CultMath's Cycles converters within the measured tolerance class.
**Not consumers.** Asura (its field is noise-dominated and needs no tape).
AetheriaEve and AetheriaThing (archived). The material graph line until it returns
after the demo (`aetheria-release:ruling:demo-material-scope`). The brush-grammar
station, until pruning and meshing unpark. Aetheria's nebula on the
span framework is an option to try (ruling `aetheria-nebula-spans-as-option`), not
yet a consumer; until a cut makes it one, the nebula consumes CultMath noise at its
pin bump and nothing else from this line.
**Invariants.**
- **`intervals-enclose`**: every interval function returns a `[lo, hi]`, and every
affine form's range is an interval, containing `f(x)` for every `x` in its input
region, in C#, HLSL and GLSL alike. A domain warp bounded by `D` is enclosed by
enlarging the ball radius by `D`. A consumer that skips a region because
`hi < cutoff` has skipped nothing that was there.
- **`affine-beats-interval-on-wells`**: on the deep-well scenario (wells about an
order of magnitude larger than the fog gradient), the affine bound's combined cost
ratio exceeds the centred interval bound's at warp 0 and at warp `D`, with the
cull floor at 0.5 (rulings `wells-scale-order-of-magnitude`,
`affine-wells-contract-cheaper-than-interval`).
- **`noise-pins-measured`**: there is one snoise kernel, and its value, gradient
and Hessian come from the same tie order. `SNOISE_LIPSCHITZ`, `SNOISE_HESSIAN`
and `SNOISE_THIRD` are each measured on that kernel by a committed test (the
refined maximum times 1.10), and every runtime's copy is pinned equal to the C#
constant by a value test. A kernel change re-measures every pin in the same
merge. Consumers read the pins and the derivative functions from CultMath,
never from page-local copies.
- **`span-bounds-enclose`**: each warp-chain stage (per-component offsets,
anisotropic stretch, plain or ridged `1 − 2|n|`, curl if ruled) is a CultMath
function in C#, HLSL and GLSL returning value, Jacobian and span bounds (speed,
acceleration, curvature, fold deviation) derived only from the noise pins. Each
bound holds at every point of its span. The composition helper composes them by
the chain rule; a consumer writes its chain and never derives a bound (ruling
`span-chain-shape-ruled`).
- **`span-integration-closed-form`**: along a span, optical depth is integrated in
closed form, the clamped linear ramp by `integral_clamp_linear` (split at the
exact clamp crossings, finite for infinite clamp bounds) and a proven
constant-density body by Beer–Lambert in one step (ruling
`operator-analytic-wall-body`). The linearisation error is bounded by the
composed stage bounds, and a span proved empty or saturated by them is so at
every point.
- **`glsl-derived`**: `CultMath.glsl`, `CultMath.Phacelle.glsl`,
`CultMath.Cycles.glsl` and the golden fixture are generated by one enumerated
lowering from the HLSL and C# and pinned by tests; the Unity package's shaders
are byte-identical copies. Nobody edits them by hand, and a dialect gap is fixed
in the HLSL common subset. The site vendors the GLSL by CultLib commit, SHA in
the header, with a drift check.
- **`three-runtimes-agree`**: HLSL is a bit-for-bit mirror of C#, and the GLSL
agrees with C# on the golden fixture within each family's stated tolerance,
evaluated on a WebGL2 device.
- **`cycles-exact-to-unfused`**: CultMath's `cycles_*` functions are Cycles'
formulas with one IEEE single rounding per operation, bit-exact to the Blender
5.2.2 Linux SSE4.2 oracle fixtures. Every other Cycles build and CultMath's own
shader lowerings are held to a per-function tolerance class measured on that
build and recorded in the oracle README; cancellation-prone functions (floored
modulo, wrap) carry an error bound relative to the operand scale and a divergence
note (ruling `cycles-kernel-reference-ruled`).
- **`one-expression-graph`**: `CultMath.Expr`'s `ExprGraph` is the one IR. One
validator admits a graph whole at build and at decode; one op table, append-only,
names every op; the graph has no loop construct, and stencils are build-time
substitution. Bytecode is derived from the graph, cache-only, and is never stored
as truth. The MessagePack resolver is the wire shape; JSON is never load-bearing.
- **`evaluator-composes-cultmath`**: the C# evaluator's result equals the direct
composition of the same CultMath calls bit for bit, one IEEE single per op; an
op's meaning is its CultMath function's, and a function's value change is a
Breaking release note.
- **`tape-interpreter-compute`**: the HLSL interpreter is one included source,
`CultMath.Tape.hlsl`, never a per-graph generated kernel. Its registers are
linear-scan allocated at lowering and live in groupshared memory, a slice per
thread, the group size shrinking as the register cap grows (64 registers: 128
threads; 128: 64; 255: 32). It and every presenter that uses it run as compute
passes writing to a texture (ruling `tape-register-model-ruled`).
- **`interpreter-matches-evaluator`**: the HLSL interpreter agrees with the C#
evaluator on a graph corpus and on each consumer's own graph within tolerances
measured per op, compiled by dxc and run on a device.
- **`lattice-node-reads-bodies`**: the weighted lattice cellular node takes the
zone body span and a falloff and weights each candidate cell at its cell centre
inside the node. It states a supported weight-ratio range, its search window
holds every cell that can win within that range, and graph validation refuses
parameters outside it (ruling `lattice-weight-from-bodies-ruled`).
- **`site-field-span-framework`**: the gamecult.org ground's density is a
site-owned composition over CultMath stage functions, integrated span by span:
a ramp, a warp chain, and one authored brush, the void, a true 3D spherical
cavity of hollow radius 198 and ramp 50 inside a full cloud volume, a sun in its
floor and the camera orbiting low (rulings `operator-ground-void-brush`,
`operator-void-true-cavity`, `operator-shipped-void`). The site defines no noise,
derivative or bound.
- **`one-motion-owner`**: the ground has one motion law, owned by the site's field.
What moves the density is what TAA reprojection and the stardust read; nothing
evaluates motion a second way.
- **`shader-owns-ground`**: the ground is the shader's moving render. The CSS wash
tokens are sampled from its output by a non-stalling readback, carried across
page loads in `sessionStorage`, and are seed or fallback only. The shader reads
the four nebula input tokens, never an output token.
- **`dof-from-transmittance`**: the stardust's blur is driven by the cloud layer's
accumulated opacity at that pixel, so the clouds blur what lies behind them.
- **`budget-measured`**: quality steps down by measured GPU or frame time only
(samples, then resolution, then the CSS wash), never by user agent.
**Not in scope.**
- Pruning (tape step 3) and meshing (step 5). Interval, affine and gradient
evaluation over `ExprGraph` belong to pruning and stay parked.
- Span evaluation over the graph (`tape-span-eval`): the later generalisation of
the stage functions once the interpreter exists.
- A GLSL or WebGL2 tape interpreter: groupshared memory exists only in compute
shaders, and WebGL2 has none. The site's field is written in GLSL against the
stage functions.
- The material line (cones, frontiers, a per-graph HLSL emitter), until the
material graph returns after the demo.
- Flow laws, envelopes, brushes and chain order in CultMath: the consumer's. A
curl noise primitive waits on the operator's answer to follow-up
`curl-warp-primitives`.
- The kernel cascade, unless the operator's nebula work picks it.
- Per-depth transmittance for the stardust DoF.
- A CultMath release for the site; the site consumes a CultLib commit.
**Canonical implementations.** Paths are under `packages/cultmath/` unless named.
- Kernel, pins and span primitives: `src/CultMath/math.cs`, `math.Interval.cs`,
`math.Affine.cs`, `math.Span.cs`; `shaders/CultMath.hlsl`,
`CultMath.Interval.hlsl`, `CultMath.Affine.hlsl`. The stage functions land
beside them.
- Lowering and parity: `tests/CultMath.Tests/GlslLowering.cs`,
`GlslMirrorTests.cs`, `HlslMirrorTests.cs`, `fixtures/glsl-parity.json`;
generated `shaders/*.glsl`; Unity copies under
`unity/org.gamecult.cultmath/Shaders/`.
- Cycles: `src/CultMath/math.Cycles.cs`, `shaders/CultMath.Cycles.hlsl`,
`tools/cycles-oracle/` (oracle and the tolerance-class table),
`tests/CultMath.Tests/fixtures/cycles-oracle/`.
- Graph: `src/CultMath/Expr/`; wire shape
`src/GameCult.Caching.MessagePack/CultMathExprResolver.cs`; interpreter
`shaders/CultMath.Tape.hlsl` (cut `tape-interpreter`).
- Ground: gamecult-site, created by `cut-site-ground.r1` (no ground code on
`origin/main` `af6e206`); palette input the four `--gamecult-nebula-*` tokens in
`custom.scss`.
- Research behind the rulings: `docs/research/integration-native-noise-domain.md`,
`span-chain-and-site-field.md`, `tape-register-model.md`,
`lattice-site-weight.md`.
## The authority map (all four cuts)
@ -390,7 +547,9 @@ from the mobile preset, the phone's GPU time from the operator's phone.
tolerance class and device named in the report.
Not proved here, said plainly: tape evaluation, automatic pruning, `min`/`max`
choice tracking, affine arithmetic. Those unpark with steps 2–4.
choice tracking, affine arithmetic. Pruning and choice tracking stay parked;
the graph, its evaluator and interpreter, and affine forms have since unparked
(Target r2).
## Cut order
@ -444,8 +603,8 @@ Results against the SSE4.2 fixture:
## Standing design decisions (means; Self may overrule)
- D1. Intervals are `float2(lo, hi)`. Affine arithmetic is not taken; the
tightness number says whether it is ever worth it.
- D1. Intervals are `float2(lo, hi)`. Affine forms sit beside them (ruling
`affine-forms-unparked`); the wells contract says where they pay.
- D2. No `iv_*` function has a value-only or gradient twin; the bounds take
the point value only.
- D3. The site vendors `CultMath.glsl` by CultLib commit, SHA in the header, a
@ -592,7 +751,7 @@ and `S` from the exponent (`e = 4` gives `S ≈ 1.3 Rh`, `e = 0.25` gives
`S ≈ 0.12 Rh`), so `(Rh, S)` are the knobs and `(Rc, e)` derived. Shipped:
`Rh = 198`, `S = 50`, so `Rc = Rh + S = 248` and `e = −ln CARVE / ln(1 −
(Rh/Rc)²) = 0.40`, `K = 1/30` (pass 4 corrects this line's `e ≈ 0.6, Rc ≈
220`, which did not satisfy the formula; Q9 holds the look question).
220`, which did not satisfy the formula; Q9 asked the look question and is ruled `operator-shipped-void`).
Imagination's model (scratchpad `cavity_model.py`; the test confirms or
refutes), 1080p, `N = 8`, per pixel for a converged frame; "ref" is the
@ -667,7 +826,7 @@ above. Everything else recovered carries over.
**The site scene** (`site-ground` r3; the brush parameters in this paragraph,
`Rc = 640`, `e = 4`, are superseded by pass 4's one shipped void, `Rh = 198`,
`S = 50`, `Rc = 248`, `e = 0.40`, pending Q9): the original's units. Outer volume: an
`S = 50`, `Rc = 248`, `e = 0.40`, as Q9 ruled): the original's units. Outer volume: an
unbounded uniform fill (chosen over a shell: every ray then ends in a wall
and the march terminates by opacity, never by a far plane; `Z1 = 2 Rc` bounds
the pre-pass range only). The carve: `d = |p − c_v|`, `g(d) = (1 −
@ -806,7 +965,7 @@ answers the first; the second is content and stays.
| --- | --- | --- |
| `frustum-ball-float-crossing` | Medium | the radius is widened by a derived float32 bound (below); seven extreme families join the enclosure test; the strictness sentence leaves the C#, the HLSL and `design.md` |
| `aetheria-far-plane-miscited` | Low | `far = 2048`, the Main Camera's (`ARPG.unity:31777`, `:31830`); the Tint camera's 1000 was the wrong camera |
| `shipped-void-two-scenes` | Medium | one shipped void, `(Rh, S) = (198, 50)`, `(Rc, e) = (248, 0.40)` derived; the map, the site-ground spec and the test agree; the look question is Q9 |
| `shipped-void-two-scenes` | Medium | one shipped void, `(Rh, S) = (198, 50)`, `(Rc, e) = (248, 0.40)` derived; the map, the site-ground spec and the test agree; the look question was Q9, ruled `operator-shipped-void` |
| `lod-weights-never-exercised` | Low | configuration "LOD far" (`Rh = 4000`, `S = 50`, low camera, `f = 467`) reaches weights in `(0, 1)` and at 0; the `[w(z1), 1]` bound is recorded as a deviation with its soundness argument |
| `stryker-timeout-survivor` | Low | 2525 named equivalent; ball radii to `1e3` so `L/r` dies to an enclosure test; every timeout hand-run; the report claims only demonstrated kills |
@ -853,7 +1012,7 @@ analytic body the second gains nothing: across a 442-wide ramp the optical
depth exceeds 4 before the density is provably constant, so rays end inside the
ramp and the body is never proven. The map's own rule says `(Rh, S)` are the
knobs, so the honest choice is the first; it changes the look relative to the
site section, hence Q9 (below). For `CARVE = 1.5` the ramp width is tied to the
site section, hence Q9 (below; ruled (a), `operator-shipped-void`). For `CARVE = 1.5` the ramp width is tied to the
exponent (`S/Rh = 1/√(1 − 1.5^(−1/e)) − 1`: `e = 0.4 → 0.25`, `e = 0.7 →
0.5`, `e = 1 → 0.73`, `e = 4 → 2.2`), so "pseudo-Gaussian" and "narrow" cannot
both be had without a third knob, and a narrow wall at `e = 4` needs
@ -947,7 +1106,7 @@ assumption failed: the pre-band's 0-snoise steps, or a body start later than
one more probe; the main pass reads it and never decides it. The Beer-Lambert
step is the consumer's integration, not a CultMath function.
- `(Rc, e)` are no longer authored; they are derived from `(Rh, S)`. The site
section's `Rc = 640, e = 4` is no longer an owner of the look (Q9 pending).
section's `Rc = 640, e = 4` is no longer an owner of the look (Q9 ruled (a), `operator-shipped-void`).
- (c)'s contract is derived from its own run (0.90 of the ceiling); no absolute
target is written for (c).
- Deletion line: the false strictness sentence, the Tint camera citation, the
@ -967,6 +1126,10 @@ mind's copies need re-admission by Self.
## Questions
Q4-Q9 are all ruled; the options below are kept as the record of what was
asked. Q9: (a), ruling `operator-shipped-void` (`rh198-s50`). Q4: (a), ruling
`operator-palette-tokens`. Q5: (a), ruling `operator-wash-carry`.
**Q9. The one shipped void** (pass 4; finding `shipped-void-two-scenes`).
Authored as `(Rh, S)` with `(Rc, e)` derived; `CARVE = 1.5`, `A = 20`,
`F0 = 0.01`, `K = 1/30`, the camera as site-ground r3.
@ -1720,10 +1883,14 @@ Aetheria twin.
### Questions (admitted in the mind)
- `tape-register-model`: the HLSL interpreter's register model. Gates the
interpreter cut only.
- `tape-register-model`: the HLSL interpreter's register model. Ruled
`tape-register-model-ruled` (groupshared registers).
- `lattice-site-weight-source`: how a weighted lattice node gets a per-site
weight (T7c). Gates fauna gap (1) only.
weight (T7c). Re-asked as `lattice-weight-from-bodies` and ruled
`lattice-weight-from-bodies-ruled` (node-reads-bodies).
- `lattice-window-sound-or-measured` (from section "lattice-weighted-cellular"):
a proven ring search or a fixed measured 7x7 window. Ruled
`lattice-ring-exit-sound` (ring by ring until no farther cell can win).
Prior art in hand: `F:\Projects\aetheria-material-path-prior-art.md` section 4
(GPU interpreters: Fidget's crates, Dolphin ubershaders, Laine et al. on
@ -1733,17 +1900,18 @@ found) and section 6 (IR placement); Aetheria
filaments). Neither covers a GPU tape interpreter's register storage (dynamic
indexing of local arrays and scratch spill, groupshared memory, Fidget-wgpu's
`tape_interpreter.wgsl`, Keeter's MPR 2020) or per-site weight fields in
lattice Voronoi. Both questions lack that prior art; Self runs Eyes before
putting them.
lattice Voronoi. Both questions have since been ruled.
### Cut sequence (Pass 7; one line each, mapped when about to run)
Fauna's first cut, `fauna-trophic-habitat`, evaluates the habitat graph in C# at
candidates. It needs `expr-ir` r2, gaps (2) and (4) with (3) under the strands,
gap (1), and their ops registered (line 5). It does not need the interpreter or
gap (5); its presenter does. The critical path is `cycles-converters` merge,
then line 3 beside the ruling on `lattice-site-weight-source` and line 4, then
line 5; line 1 runs now, off that path.
gap (5); its presenter does. Lines 1 and 3 and `cycles-converters` are on
main (`911fc755`). The critical path is line 4a, after the ruling on
`lattice-window-sound-or-measured`, then line 5. Line 4b, the shader twins,
is not on the path to fauna's first cut; it is on the path to fauna's presenter,
with line 6.
1. `expr-ir` r2 (spec admitted): the graph, builder with substitution, decode
validation, wire shape, C# evaluator. Touches no file `cycles-converters`
@ -1756,20 +1924,23 @@ line 5; line 1 runs now, off that path.
`CultMath.hlsl`). Families append after the previous cut's, so these cuts
map on the merged head, not beside the branch. Gap (3) folds into line 3,
whose strand kernel is its only consumer.
3. `trophic-nodes` (CultLib): gap (3), an `out`-free
`distance_to_segment(float2, float2, float2)` in C# with its HLSL twin (T6;
`point` is reserved in HLSL, so the parameter is `point_value`); gap (2), a
weighted diagram over a site span (F1, F2, first-order border distance;
multiplicative and power weights); and gap (4), the soft Gabriel gate
(computed per tick on the CPU from a site span; the per-point strand kernel
over the gated edge span has the HLSL twin), with the span convention of
the model page. Waits on the `cycles-converters` merge and its map pass.
4. `lattice-weighted-cellular` (CultLib): gap (1), 2D, `pcg3d` cells, zone
time, smooth-min blend, a stated weight range sizing its window. Waits on
question `lattice-site-weight-source`.
3. `trophic-nodes` (CultLib): spec `cut-trophic-nodes.r1` admitted; see
"trophic-nodes" below for its body facts and decisions.
4. Gap (1), mapped in section "lattice-weighted-cellular" on 3's merged head
(`911fc755`), in two cuts. 4a, `lattice-weighted-cellular`
(`cut-lattice-weighted-cellular.r1`), is C#: the node reads the body span
and weights each cell at its centre through line 3's exact `power_border`.
It waits on question `lattice-window-sound-or-measured`. 4b,
`lattice-weighted-shaders` (`cut-lattice-weighted-shaders.r1`), adds the
twins of its four steps, the GLSL, the families and the lavapipe bounds.
It waits on 4a.
5. `expr-trophic-ops` (CultLib): the span input kind and ops for 2-4 in the op
table; a CultMath Unity tag for fauna. Waits on 1, 3 and 4. Fauna's first cut
waits on this.
table; the strand reduction over the gated edge span (the graph has no loop,
so a span op is a CultMath function: a C# span overload over a step with its
shader twin, the convention line 3 sets); follow-up
`expr-create-copy-then-validate` (the first cut that touches `ExprGraph`;
line 3 does not); a CultMath Unity tag for fauna. Waits on 1, 3 and 4.
Fauna's first cut waits on this.
6. `tape-interpreter` (CultLib): bytecode lowering, `CultMath.Tape.hlsl`, and
the graph-parity harness (gap 5): C# evaluator against the interpreter on a
graph corpus and on a consumer's graph, dxc compile plus a device run, held
@ -1783,6 +1954,110 @@ line 5; line 1 runs now, off that path.
10. `cultmath-unity-math-parity` (follow-up): square, exp2, exp10, log2, log10.
11. Pruning (tape step 3) and meshing (step 5): parked as the target says.
### trophic-nodes
Written by Imagination (agent `imagination-ag4-trophic`, session
`self-2026-10-16-ag4`) against CultLib `origin/main` `f697e03f`, where
`cycles-converters` (`b136ad9b`), `expr-ir` and `stage-functions`/`stage-shaders`
(`d7d1bcf3`) are merged. One CultLib cut owns what fauna's map line
`cultmath-trophic-nodes` and this map's line 3 both name; fauna's consumer cut
(`fauna-trophic-habitat`) waits on line 5 and the Unity tag, not on this cut
alone. Spec: `cultmath-tapes:cut_spec:cut-trophic-nodes.r1`.
#### Body facts (trophic-nodes)
| # | Question | Result |
|---|---|---|
| TN1 | What does the probe compute, exactly? (Aetheria `docs/research/2026-10-14-trophic-dag-probe/trophic_probe.py:49-71`, `2026-10-14-body-voronoi-probe/body_voronoi_probe.py:36-67`, master `6b6e7c7c`) | Body diagram: every body visited, scores sorted, the two winners' `(F2 - F1) / |grad(F2 - F1)|` and its unit normal (the lattice web reads both). Power score `d^2 - sigma^2`, multiplicative `d / sigma` with `sigma` the well's Gaussian width. Gate: per pair `(i, j)`, margin = min over the other sites `k` of `(|k - mid| - r) / r`, `r = |ab| / 2`, unweighted positions; gate `smoothstep(-.15, .15, margin)`; no other site gives margin 9 (gate 1). Strands: max over gated edges of `gate * exp(-(segdist / width)^2)`, `width = 28 + .03 L`. |
| TN2 | Is the first-order power border exact? (scratch `border_probe.py`, double, 3,000 spans of 2-8 sites) | Yes: `F2 - F1` is affine in the query for power scores, so the first-order distance is the distance to the power bisector plane; worst relative error against that plane 2.2e-13. A one-pass fold (strict `<`, F1 then F2) gives the sorted order on every draw; an exact tie keeps the lower index first. Gabriel margin: a third site on the diameter circle gives 0, inside at half radius -0.5. |
| TN3 | Can the shader side take a span? | No resource in the mirror: `HlslSourceCompatibilityTests` compiles the HLSL text as C#, and `CultMath.hlsl` has no `StructuredBuffer` (0 in every `shaders/*.hlsl`). The struct-aware twin rule (`:153-161` at stage-shaders, now `:162-181`) compares struct and `int` arguments and private twins; `compared.Count` is 127 at `:181`. `GlslMirrorTests.FromScalars` (`:488-500`) decodes `int` and structs; `GlslTypes` (`:874-879`) maps names; `Resolve` (`:882-888`) binds public `math` methods only and cannot parse an empty parameter list. |
| TN4 | Wire coverage of a new public struct (`CultMathSerializationTests`, `CultMathJsonTests`, T5) | Every exported struct needs a `CultMathResolver` shape and a `CultMathJson` converter; JSON `Fields<S>` (`CultMathJson.cs:269`) holds one scalar type per struct, so a struct mixing `float` and `int` fields needs new JSON machinery. Pattern: `52d25740` (SpanPath, SpanStage: resolver, JSON, a `Json`, a `RoundTrip` and a `Hex` pin). |
| TN5 | Who calls `distance_to_segment(.., out closest)`? | `MathTests.cs` and Aetheria `Assets/Scripts/ServerShared/Galaxy.cs`; it stays, unchanged in value. |
| TN6 | Is `CultMath` free to take `ReadOnlySpan`? | Yes: `net8.0;netstandard2.1` (`CultMath.csproj:4`); `BatchMath` already takes `ReadOnlySpan<float2>`. |
#### Decisions (defaults, not forks)
- **A span function is a C# loop over a step that has a shader twin.** The
shader cannot take a span in the mirror (TN3), and a buffer-taking HLSL
function would be a second, untested path. So each span function is one fold:
`site_cells_empty()`, then `site_cells_insert(cells, score, index)` per site,
read by `site_cells_border(cells)`; the per-site score is `power_score` or
`multiplicative_score`. C#'s `power_cells(p, sites)` and
`multiplicative_cells(p, sites)` are that loop and nothing else. A shader
caller (and later the tape interpreter) writes the same loop over its own
buffer. One rule, one path; the per-site functions are mirrored bit for bit.
- **Sites are `float4(position.xyz, weight)`**, the model page's span element; a
2D consumer uses z = 0. For an explicit span every site is visited, so no
window and no weight-ratio range apply (the lattice node's range is line 4's).
- **Indices are `float` fields** (`nearestSite`, `secondSite`, -1 for none), as
`CultCellular.id` is: one scalar type keeps the MessagePack and JSON shapes on
the existing helpers (TN4), and an index is exact below 2^24, far past any
span a per-point O(N) search serves.
- **The gate is pointwise per pair**: `gabriel_margin(a, b, other)` with a
shader twin, and C#'s `soft_gabriel_gate(sites, first, second, softness)`
folding it over the span (TN1's unweighted positions; a power-weighted
Gabriel graph is not requested by any ruling).
- **The strand reduction is not in this cut.** Its Gaussian profile and width
law are fauna's composition until the graph needs it as an op; line 5 owns
that op, with the same step-and-twin convention.
- **Shader part separable.** The C# functions, struct and wire shapes land
first; the HLSL twins, the regenerated GLSL and the fixture families land in
the last commits under today's `GlslLowering`. Pass `imagination-ag4-shader-runner`
is re-posing the lowering and shader testing (question `device-run-driver`);
if that lands first, only the shader commits are re-mapped.
### lattice-weighted-cellular
Written by Imagination (agent `imagination-ag4-lattice-weighted`, session
`self-2026-10-16-ag4`) against CultLib `origin/main` `911fc755`, where
`trophic-nodes` is merged (`7e6881af`). Ruling `lattice-weight-from-bodies-ruled`
(node-reads-bodies). Line 4 splits into two cuts because of Hands' budget:
`cut-lattice-weighted-cellular.r1` (C#) and then `cut-lattice-weighted-shaders.r1`
(twins, GLSL, families, lavapipe bounds). One question,
`lattice-window-sound-or-measured`, gates the first cut's search loop.
#### Body facts (lattice-weighted-cellular)
The probes are numpy float64 scripts in
`docs/research/lattice-site-weight/weighted-node/` (`probe.py` is the node, and
each other script is one measure). They use 6 bodies orbiting with period equal to
distance and a sigma ratio of 5:1, `cellSize 260`, `falloff 120` (the body probe's
`3 * EW_U`), `KW .35` and boost 1.6, the ruled look.
| # | Question | Result |
|---|---|---|
| LW1 | What weight does the ruled look use? (Aetheria `body_voronoi_probe.py` `lattice_web_edges`, master `4e620671`) | `base(.5..1) * (1 + 1.6 * exp(-(border(cell centre) / 120)^2))`. The border is the body diagram's first-order distance under the ruled power metric (`body-diagram-metric-ruled`). Distance is stretched by `1 + near` (up to 2x) across the body border's normal at the query point. Output `max(soft^2, exp(-((F2 - F1)/.1)^2))`, with soft from her smooth-min fold. |
| LW2 | Window sufficiency (`a.py`, 67,600 points against a 19x19 box) | 7x7 has no F1/F2 misses at ratio 5.2, 10 and 20 with stretch 1 or 2. 5x5 misses .1-.3% of points with stretch 2 (output error up to .92) and .04% at 20:1. A worst-case proof covers 7x7 only for ratio x stretch up to about 1.5. |
| LW3 | Are the weights continuous in time? (`c.py`, 576 cell centres, 300 s at 1/60 s) | First-order border: max step .68, and 113 steps over .01, where the second-nearest body changes. Exact `power_border`: max .0032, none over .01. |
| LW4 | Does her fold (from 8, row-major) depend on the window? (`d3.py`, 40,000 straddles of lattice cell boundaries) | Yes. With F1 and F2 exact, soft jumps .114 at 5x5 and .056 at 7x7 with stretch 2. It vanishes at 13x13 on this corpus, but nothing proves that. |
| LW5 | Which fold start is window-independent and continuous? (`d3.py`, `g.py`) | Start at F1 (F1 cell skipped): no window jump, but at F1 identity swaps m jumps .011 and soft .25, because the polynomial smooth min is not associative. Start at F1 + k and fold every cell: swaps jump 4e-15 and window jumps are 0 once the box holds every `d < F1 + 2k`. Its overshoot above m is at most .017k per step (computed), hence the clamp to F1 + k. |
| LW6 | Look of start F1 + k against hers (`h.py`, 40,000 points, 13x13) | Mean output difference 0.0000, max .032 (stretch 1) and .030 (stretch 2). |
| LW7 | Ring-exit cost (`e.py`) | Cells visited: at the ruled look median 25, max 49 (stretch 1) and 81 (stretch 2); at boost 4 with stretch 2, max 121; at boost 9, max 81. The worst-case sound fixed box is 19x19 to 67x67. |
#### Decisions (defaults, not forks)
- **The weight reads the exact power border** (`power_border`, trophic-nodes),
not the first-order one the ruled look's probe used: LW3, under
`aetheria-fauna:ruling:habitat-no-pop-ruled`. The look moves where the
first-order distance was wrong (LW3's jumps are the same places).
- **Her fold starts at F1 + k and is clamped to F1 + k**, in absolute row-major
order (LW4-LW6). The look is unchanged to within .03, and the node is
window-independent by construction, which the search's exactness needs.
- **The alignment stretch is in the node.** It is a query-point anisotropy, and
it does change winners (B41). Only the node sees the candidate sites, so a
caller cannot apply it. Stretch 1 is isotropic.
- **float4(F1, F2, smooth, id), no gradients, no struct.** fauna's composition
reads values only, and a struct would owe a wire shape (TN4). Gradients can be
added later as their own function, if a consumer asks.
- **The range is public constants**, so line 5's graph validator reads the range
from CultMath rather than restating it: boost up to 4 (10:1), stretch up to 2,
smoothing up to 1.
- **The base weight takes its own hash lane** (`pcg3d(cell).z`), not the jitter
phase the probe reused.
- **Steps for shaders, the span function in C# only**, by trophic-nodes'
convention: `lattice_site`, `lattice_weight`, `lattice_distance`,
`lattice_smooth_step`.
## Pass 8: volumetric prioritized (ruling `volumetric-work-prioritized`)
Written by Imagination (agent `imagination-volumetric-frontier`, session
@ -1832,10 +2107,11 @@ authoring framework" she will build on. The kernel cascade (the probe and the Ke
Lab) is the earlier exploration, and it stays a line below.
**Correction to the brief's expectation.** The next cut is not site-ground. The
site ground's field is open: the Span Lab framework, the kernel cascade, or r3's
marched void. The r3 machinery (the interval pre-pass, the centred warp, the
debug-versus-dense readback) is shaped for a marched field. Question
`site-ground-field` asks this.
site ground's field was then open between the Span Lab framework, the kernel
cascade and r3's marched void. The r3 machinery (the interval pre-pass, the
centred warp, the debug-versus-dense readback) is shaped for a marched field.
Question `site-ground-field` asked this. It is ruled `site-ground-field-ruled`:
the Span Lab framework with the void as its brush.
### `SNOISE_LIPSCHITZ`: 10.099 in the lab, 7.964 in CultMath
@ -1917,8 +2193,8 @@ is traversal, which is unmeasured. Optical depth through it is a sum of per-kern
segment integrals, so it is order-independent. Additive blending of each kernel's
screen footprint would give exact transmittance in WebGL2, with no compute and no
sorting. Emission is not order-independent. This is a hypothesis with no probe
behind it. The cascade becomes a cut only if `site-ground-field` or the operator's
nebula work picks it.
behind it. `site-ground-field-ruled` picked the Span Lab framework, so the cascade
becomes a cut only if the operator's nebula work picks it.
### Cut sequence (volumetric line)
@ -1926,9 +2202,324 @@ nebula work picks it.
on the tied kernel, `integral_clamp_linear`, in C#, HLSL and GLSL with fixture
families. It touches no `expr-ir` file. Its provenance prose goes in code
comments and the README, not `design.md`, which `expr-ir` edits.
2. Question `span-chain-shape`, then the warp-stage and span-bound cut it picks.
2. Ruled `stage-functions` (`span-chain-shape-ruled`), split in three for size, on one
branch `hands/cultmath-stage-functions`:
- `stage-functions` (specs `cut-stage-functions.r1`-`r3`) and `stage-shaders` (spec
`cut-stage-shaders.r1`): merged to main 2026-10-16 (verdicts cut-stage-functions.s2-s4,
cut-stage-shaders.s1). SpanPath/SpanStage inputs, float-rounding allowances, HLSL and GLSL
twins with SpanPath/SpanStage/SpanNoise as shader structs, a struct-aware mirror with no
`MirrorOnly` list (125 functions compared), `tools/compile-hlsl.ps1` (x64 dxc, `-WX`).
SpanStage.Identity has no shader form (decided in the spec: callers build the struct; a
mirror test proves the shader at a C#-built identity equals fbm_grad/ridged_grad).
- `stage-shaders-device`: a committed WebGL2 runner on Starfire (ANGLE to D3D11, Unity's
driver path; also runnable on a phone) from follow-ups `glsl-browser-parity-run` and
`affine-device-runner-seed`; fixture families across coordinate bands 1e2..1e6; a C# test
reading the device readback against the double kernel; settles follow-up
`span-allowance-gpu-margin`'s four CPU survivors (eg-abs, eh-abs, So-reach, two-en-to-one)
and the size of the 2^-14 absolute term (en-abs-halved survives every CPU test). `fold`
and the ridged sign jump at zero, so their device check is one-sided enclosure, not ulps.
- `cultmath-unity` span release: after `stage-shaders-device`; the smoke shader calls both
span twins; its own Soul pass; then Aetheria's pin bump and its spans-option lowering
(ruling aetheria-nebula-spans-as-option; a moving nebula also waits on
`span-bounds-under-flow`).
- `span-compose`: the composition helper (`span_compose`) and `span_range` for
the empty and saturated proofs, in all three runtimes, with a two-stage chain
enclosure test against dense sampling.
`trophic-nodes` (spec admitted on main after this branch merged) and then
`lattice-weighted-cellular` (waits on it): Pass 7 lines 3-4.
Body fact (probe on Yggdrasil, scratch `edb3c1c3` over `2fd9431a`; log
`F:\Projects\eureka-handoff-aux-2026-10-14\imagination-stage-functions\probe2.log`):
the HLSL mirror that `HlslSourceCompatibilityTests` compiles exposes every
`static const` as a public static literal field of `CultMathHlsl.HlslShader`
(`CULTMATH_PI` through `CULTMATH_SNOISE_THIRD` and `CULTMATH_CYCLES_*`). So one
reflection test can pin every shader constant to its C# twin.
3. The curl primitive, once the operator rules on curl (the Aetheria flow test).
4. Question `site-ground-field`, then `cut-site-ground.r1` on the field it picks.
The marched option carries all three site follow-ups; the other options
withdraw the two marched-only ones.
4. `cut-site-ground.r1` on the Span Lab framework (`site-ground-field-ruled`).
Under that option the two marched-only site follow-ups are withdrawn, and
only the third carries.
5. The kernel cascade, if picked.
## Pass 9: the shader pipeline (rulings `glsl-es-via-dxc-spirv-cross`, `conforming-run-lavapipe`, `real-device-native-angle`)
Imagination `imagination-ag4-shader-pipeline`, session self-2026-10-16-ag4, 2026-10-16, on
`origin/main` f65bde6f. The evidence the operator ruled on is `docs/research/native-shader-testing.md`
(f697e03f) and `docs/research/native-shader-probes.md` with its scripts (e598c443). This pass adds
the body facts below, then sequences the three rulings.
### Body facts (Pass 9)
Every probe ran through `ygg-verify.sh` on Yggdrasil at f65bde6f, 2026-10-16 between about 16:30
and 16:50 UTC (18:30 to 18:50 CEST). Logs are in the Imagination scratchpad
(`imagination-ag4-shader-pipeline/split-run1.log`, `img-run1.log`, `snap-run1.log`).
- **SPIRV-Cross pin.** The newest SDK tag is `vulkan-sdk-1.4.363.0` (commit f11ba9f0, 2026-09-16).
Built from that tag (`-DSPIRV_CROSS_CLI=ON`, Release, 4 cores), it takes 70 s and gives the same
result as the HEAD build the earlier probe used. `tools/get-spirv-cross.ps1` clones HEAD unpinned.
- **dxc pin (Linux).** The asset is
`.../v1.9.2607/linux_dxc_2026_07_29.x86_x64.tar.gz`, sha256
`55665c87824051ed4774ff3280a79ccbbb7d39243b9736ca5e98222134112d54`, and it reports libdxcompiler 1.9.0.1.
`get-dxc.ps1` pins the Windows zip of the same release.
- **Overloads collapse in dxc, not in SPIRV-Cross.** With `export` on the 173 non-texture definitions,
the library has 127 `OpFunction`s and 127 exports: one per name. Today's source has 22 overloaded
names, 68 definitions and 46 lost. Trophic-nodes (branch `hands/cultmath-trophic-nodes`, unmerged) adds
`cultmath_segment_closest` and `cultmath_distance_to_segment` in 2D and 3D, so 24 names and 72 definitions.
The loss comes from exporting overloaded names, and build-only wrappers avoid it (next bullet). The
source keeps its overloads (ruling `shader-overloads-stay-native`).
- **Build-only export wrappers keep every overload (Pass 9b).** Imagination `imagination-ag4-overload-wrappers`,
2026-10-16, on 09940e20, ubuntu:24.04 on Yggdrasil, 17:40 to 17:55 UTC (19:40 to 19:55 CEST), log
`imagination-ag4-overload-wrappers/wrap-run2.log` in the session scratchpad. The probe generated one
wrapper per non-texture definition (173) in a build-only file. Each wrapper looks like
`export float3 cmw036_cultmath_lerp(float3 start, float3 end, float3 amount) { return cultmath_lerp(start, end, amount); }`.
The source was unchanged, with no `export` keyword anywhere, and it compiled with dxc `-T lib_6_3 -spirv`.
- At default optimisation dxc inlines every wrapper. The 173 wrappers hold the bodies, and no
overload survives as its own function.
- At `-Od` it keeps the call. That gives 346 functions: 173 wrappers, each with exactly one
`OpFunctionCall` to its own distinct callee. Each callee's `OpName` is already the plain source
name (`cultmath_lerp` four times), so no rename is needed.
- An instruction-level edit of the SPIR-V binary then makes the library. It keeps only the Shader
and Linkage capabilities, drops every `OpExtension`, deletes the wrapper functions with their
names and decorations, and moves each wrapper's `LinkageAttributes ... Export` onto its callee.
`spirv-val --target-env spv1.5` accepts the result. Deleting the export instead of moving it gives
a module SPIRV-Cross refuses ("no entry point").
- SPIRV-Cross `vulkan-sdk-1.4.363.0 --es --version 310` emits 173 `cultmath_` definitions, as many
per name as the source has. There are no `_N` suffixes and no wrapper names: its own overload
handling renames only on a signature collision. The emitted text is 2621 lines, against 1497 for
today's three files, because `-Od` leaves temporaries.
- Call sites resolve. glslang compiles as a `#version 300 es` fragment shader that calls
`cultmath_lerp(float..)`, `cultmath_lerp(vec3..)`, `cultmath_saturate(vec4)` and
`cultmath_saturate(float)`. A call with `ivec2`, which has no overload, is refused ("no matching
overloaded function").
- Same computation. All 105 families the probe generator covers ran on lavapipe through the
restored library, and every one is bit-identical to the committed GLSL route. 85 of the 105 are
also bit-identical to the HLSL route, the same 85 where the committed GLSL matches it.
- The rest of the pipeline carries over. Partitioned by origin file, the output gives 178 chunks
(125 core, 51 Cycles, 2 Phacelle). No origin is unknown and nothing calls across licences.
Every overload set lives in one file, and no signature repeats. glslang compiles all four
combinations as ES 3.00. The source has 13 `static const` lines, all of the form
`static const float CULTMATH_X = <literal>;`. None of them reaches the output, and today's GLSL
holds 13.
- **What `-Od` leaves, and what cleans it (Pass 9c).** Imagination `imagination-ag4-od-temps`,
2026-10-17, on e626bf26 (188 wrappers, 140 names, 117 families), ubuntu:24.04 on Yggdrasil, 10:05 to
10:35 UTC (12:05 to 12:35 CEST), dxc v1.9.2607, SPIRV-Cross and SPIRV-Tools both built from
`vulkan-sdk-1.4.363.0` (spirv-opt v2026.4), log `od-temps/imag-od7.log` in the session scratchpad,
probe in `docs/research/native-shader-probes/od-temporaries/`. Pass 9b's route as ruled, with no
further step, emits:
- 2824 lines, 86129 bytes, 188 `cultmath_` definitions, 140 names, no suffixed names. The
committed files are 1663 lines of one-line bodies, so line counts do not compare; SPIRV-Cross
prints every body in Allman style.
- Named locals survive, so most functions read like the source. Two things do not. First, 472
`param_var_*` copies: dxc passes every parameter as a pointer and the caller copies in. Second,
11 functions assign to a parameter in their body (`smootherstep`, `integral_clamp_linear`,
`fbm_grad`, `ridged_grad`, both Bezier pairs, `span_stage`, `iv_fbm_ball`, `af_fbm`) and come out
`inout`. The HLSL has no `out` or `inout` anywhere.
- That `inout` is a defect in the route, not a blemish. glslang refuses a consumer call with a
value, `cultmath_smootherstep(x + 1.0)` ("l-value required"), and a call with a variable
rewrites the caller's variable. Pass 9b's bit-identical result held because the families call
through their own copies. Overloads that look native must also take values like native ones.
- NaN-safe `min` and `max` expansions (`mix(mix(max(a, b), b, isnan(a)), a, isnan(b))`) from
`NMin` and `NMax`.
- Six `spirv-opt` pass lists on the edited module alone (`--convert-local-access-chains`,
`--ssa-rewrite`, `--eliminate-local-single-block`, `--eliminate-local-single-store`,
`--eliminate-local-multi-store`, `--simplify-instructions`, `--eliminate-dead-branches`,
`--merge-blocks`, `--vector-dce`, `--redundancy-elimination`, `--scalar-replacement`,
`--copy-propagate-arrays`, each with `--eliminate-dead-code-aggressive`) are valid and keep 188
functions, but make the text bigger (2870 to 2930 lines) and swap named locals for `_N` ids (218
tokens become 2155 to 2517). `param_var_` and `inout` stay. They are not a cleanup.
- A by-value lowering of the SPIR-V (`od-temporaries/byvalue.py`, 50 lines) makes each Function
pointer parameter a value parameter, with the old id kept as an entry-block local stored from
it, and each call loads what it passed. It changed 187 functions and 456 parameters, validates,
and leaves `inout` at 0. On lavapipe all 117 families stay bit-identical to the committed GLSL.
- Then `spirv-opt --eliminate-local-single-store --eliminate-dead-code-aggressive` removes every
`param_var_` copy (472 to 0): 2604 lines, 79782 bytes, 188 functions, 316 distinct `_N` ids, valid.
Adding SPIRV-Cross `--relax-nan-checks` drops the `isnan` expansions: 2556 lines, 74263 bytes,
268 `_N` ids. glslang accepts both as ES 3.00, `cultmath_smootherstep(p.x + 1.0)` included.
- That cleaned module is bit-identical to the lavapipe HLSL result on 117 of 117 families, where
the committed GLSL is on 97. It differs from the committed GLSL in 20 families. Against the C#
fixture the exact, at most 4 ulp, at most 64 ulp and over 64 buckets are 80, 11, 9, 17, where
the committed GLSL is 80, 9, 10, 18. `--relax-nan-checks` changes no lavapipe bit. So the
cleaned route is closer to the HLSL than today's GLSL is, and "bit-identical to today's
lowering" cannot be its gate. With apt's spirv-tools 2025.1~rc1 the figures for this route are the same.
- **Why `span_fbm` and `span_ridged` differ by 1 ulp (Pass 9c).** Imagination `imagination-ag4-span-ulp`,
2026-10-17, on 23ac4e89, same image and tools as `cut-glsl-build-dxc.h1`, probes in the session
scratchpad `logs/ag4-ulp/` (`probe1.sh` to `probe4.sh`). Only the curvature word (word 4 of 6) differs, in
41 and 42 of 1536 words. Two causes, both measured:
- **dxc's entry-point optimiser reassociates.** `cultmath_snoise_hessian` ends every entry in
`105.0 * (...)`. In the `-O3` entry module the 105 is hoisted out: nine squares are summed bare and
the sum is multiplied once (`OpFMul %11025 %sum`, then `Sqrt`), where the source squares `105 * a`
nine times. That is real arithmetic, not float arithmetic. Neither `NoContraction` nor
`--relax-nan-checks` touches it. `dxc -Gis` (IEEE strictness) stops it.
- **The driver contracts mul+add by use count.** lavapipe (NIR) fuses an `OpFMul` into the `OpFAdd`
that consumes it only when the product has one use. dxc's CSE shares the symmetric Hessian's
off-diagonal squares (two uses each), so the entry module fuses some of the nine pairs; the `-Od`
library module has nine single-use products and fuses all nine. The `OpFAdd` order is the same in
both (left fold, as the C# `span_frobenius`).
- Decorating every `OpFMul`, `OpFAdd`, `OpFSub`, `OpVectorTimesScalar` and `OpDot` `NoContraction` in
both final modules, f103 and f104: `-Od` HLSL equals the built GLSL (0 words differ); `-O3`
HLSL still differs in 31 and 44 (the hoist). `-Gis` (with `-O3`) plus `NoContraction` equals the built GLSL on
127 of 127 families, and so does `-Od` plus `NoContraction`. Plain `-Od` HLSL differs from the built GLSL in 664 words
(contraction depends on use structure), so `-Od` alone is worse than `-O3`: your suspicion of `-Od`
on the HLSL route is not a fix.
- **Which is right.** Against the C# fixture, exact cases of 256 for f103 are 3 (HLSL) and 4 (GLSL) plain,
and 151 for both under `NoContraction`; for f104, 7 and 10 plain, 141 under it. Of the first 60 families whose
count moved, the strict form is nearer the C# in all but one, which moved by 1. Strict IEEE, no contraction, no
reassociation is what C# computes, so it is "right"; the plain routes are each one driver's
fusion pattern. The built GLSL is source-faithful (nothing in the `-Od` route reassociates).
- **What makes them agree.** The equality gate (`GLSL_DIR` mode) compares both routes in the strict
form: HLSL entry built with `-Gis`, and a harness pass that adds `NoContraction` to every float op of
both modules (about 12 lines of `spirv-dis` and `spirv-as`; `tools/device/nc.py` in the probe). That is
127 of 127. The shipped GLSL and the C# bounds judge are unchanged. See question `glsl-gate-strict-form`.
- **The build at main, with the rename and the strict form (Pass 9d).** Imagination `imagination-ag5-glsl`,
2026-10-17, scratch worktree `C:\cl-wt\imag-ag5-probe` (not pushed): `23ac4e89` merged with origin/main
`86cd84a3` as `69987d91` (no conflict; README.md and tools/device/gen.py auto-merged), probe commits
`575badb6` to `d14bfbbe` (`probe/p1.sh`, `p2.sh`, `spec.py`). Yggdrasil, `mcr.microsoft.com/dotnet/sdk:10.0` and
`ubuntu:24.04`; logs in the session scratchpad `imagination-ag5-glsl/p2.log` to `p4.log`.
- **Site arrays break the wrappers.** `wrappers.py` passes `sites[CULTMATH_MAX_SITES]` as the argument (its
last-token rule), and dxc refuses the three wrappers of `cultmath_power_border`,
`cultmath_lattice_cell_distance` and `cultmath_weighted_lattice_cells` ('no known conversion from float4 to
float4 [128]'). Stripping the `[..]` suffix from the argument fixes it; then the build is green: core 139
functions over 91 names (1554 lines, 54775 bytes), cycles 51/51 (867, 21250), phacelle 1/1 (82, 3199); zero
suffixed ids and zero inout.
- **The renamed fields survive SPIRV-Cross.** With `float4 cosine; float4 sine;` in the HLSL (HLSL only, in
the job), the built phacelle.glsl declares `vec4 cosine; vec4 sine;`.
- **The strict-form gate holds at main.** With `-Gis` on the HLSL entry and NoContraction on both modules
(the text `nc.py` of Pass 9c, probe only): lavapipe-glsl equals lavapipe-hlsl on 129 of 129 work items.
The work list is 125 fixture families (123 at `774bebb0`; the lattice loop added two) plus one
reference item for each of the 4 sampled affine families, so h1's 127 was 123 + 4.
- **The built GLSL loses the capacity override.** dxc preprocesses `CULTMATH_MAX_SITES`, so built core.glsl
has no mention of it and four literal `[128]` arrays. A consumer with `#define CULTMATH_MAX_SITES 4`
passing `vec4 s[4]` fails glslang ('no matching overloaded function'); the same consumer compiles
against the committed (lowered) GLSL and through dxc against the HLSL.
- **dxc has no spec-constant array size.** `[[vk::constant_id(7)]] const int CAP = 128;` as an array
parameter's length is refused ('variable length arrays are not supported in HLSL').
- **A SPIR-V spec constant carries it.** In the built SPIR-V exactly one constant is an array length
equal to 128 (id 1297), used by nothing but OpTypeArray. Made an OpSpecConstant with SpecId 0, it passes
spirv-val, and SPIRV-Cross ES emits `#ifndef SPIRV_CROSS_CONSTANT_ID_0` / `#define ... 128u` /
`const uint _1297 = SPIRV_CROSS_CONSTANT_ID_0;` with `vec4 sites[_1297]` in every site-array
signature. The probe's consumer defined the macro as a plain int and was refused for the uint
initializer, so the `uint(CULTMATH_MAX_SITES)` mapping that `cut-glsl-build-dxc.r2` writes is
the route, but has not been compiled yet.
- **Capability strip.** `spirv-opt --trim-capabilities` (spirv-tools 2025.1~rc1) leaves all 14
capabilities and both extensions in place, and SPIRV-Cross still refuses ("RayQuery requires Vulkan
GLSL 460"). Stripping every capability except Shader and Linkage and every OpExtension in the
disassembly, then running `spirv-as`, gives a module that `spirv-val --target-env spv1.5` accepts.
SPIRV-Cross `--es --version 310` lowers it in 54 ms, and dxc takes 322 ms.
- **Named constants vanish.** The SPIR-V has no `OpName` for any `CULTMATH_` constant, so the output
has none. Today's GLSL has 13: 9 in CultMath.glsl and 4 in CultMath.Cycles.glsl. The HLSL source of
every one of them is a `static const float NAME = <literal>;` line, so a build can emit them
from the source lines and should refuse any `static const` line of another form.
- **Per-file split works.** The SPIRV-Cross output was partitioned by the HLSL file that defines each
function and struct (CultMath, Interval and Affine go to core, plus Cycles and Phacelle). That gives
132 chunks: 79 core, 51 Cycles and 2 Phacelle. No chunk's origin is unknown, no chunk is duplicated,
and no Cycles or Phacelle function calls a function outside core. The output has no `spv*` helper
functions. Its prologue is 3 lines: `#ifdef SPIRV_CROSS_LIBRARY_HEADER`, `#version 310 es` and
`#endif`. glslang compiles core, core+Cycles, core+Phacelle and core+Cycles+Phacelle as `#version 300 es`
fragment shaders. So the licence split survives the switch as a partition step of about 40 lines in the build.
- **One image runs the whole conforming job.** `mcr.microsoft.com/dotnet/sdk:10.0` is Ubuntu 24.04.5
(noble), dotnet 10.0.401. Its apt has mesa-vulkan-drivers 25.2.8-0ubuntu0.24.04.4 (lavapipe, the
probe's device), glslang-tools 15.1.0, spirv-tools 2025.1~rc1, libvulkan-dev 1.3.275 and gcc 13.2.
`apt-get update --snapshot 20261016T000000Z` (apt 2.8.3) then installs the same versions. That pins
the package set to a date and lets the C# judge run in the same container as the device.
- **Consumer audit.** No public name changes under the wrapper route, so it shows only who reads the
committed GLSL and HLSL. It covers `origin` of Aetheria (master a8586d08 and all of its
remote branches), Asura (60fbd19), Odin (8e1de66) and gamecult-site (af6e206), and HEAD of every
other git repo under F:\Projects except the CultLib worktrees, the archived AetheriaEve and the
retired CultMath history repo. The search was for `cultmath_[a-z]` outside docs. Only one shader
calls CultMath: `Muninn` (1d0ea0b, and its
worktree `Muninn-next`) `crates/muninn-move-tracker/shaders/MoveSphereCandidate.comp.hlsl:88`, one
call to `cultmath_saturate(float)`. Aetheria uses CultMath only from C#. The operator's Span Lab
artifact (be6a9e31) embeds its own copy of the GLSL (follow-up `span-lab-pre-tie-kernel`) rather
than including the file.
### Decisions (defaults, not forks)
- **The fixture owner owns the work list and the verdict.** `GlslMirrorTests` writes what a device
must compute: the fixture's families, plus reference families for the four sampled-form checks,
whose reference is the device's own `snoise` or fBm at 75 points of each case. It also judges
what the device returns. The generator only spells shaders, the host only dispatches, and neither
compares anything. The lavapipe job and the later ANGLE tool share this one work list and this one judge.
- **Each device has a recorded bound.** Under `cycles-kernel-reference-ruled`'s stance, a device's
ulp-bounded and platform families are held to a bound measured on that device and committed with
the device's identity. lavapipe re-runs bit-identically, so the bound is a regression gate, and an
identity change refuses until it is re-recorded. Exact and integer families must match bit for
bit, and `check` families must pass their `Verify`.
- **Unique names for dxc live only inside the build** (ruling `shader-overloads-stay-native`). The
build owns three things here.
- The wrapper list. It is generated from the source's own definitions, one wrapper per non-texture
`cultmath_` definition, and the build refuses a definition it cannot read, just as it refuses a
`static const` line of another form. The wrappers are text in the build's temp directory and are
never committed. The source gains no `export` keyword and no `CULTMATH_EXPORT` macro.
- The compile mode. The library compiles at `-Od`, because the default optimiser's inlining is
what erases the overloads. The text it leaves is cleaned by the next two items (Pass 9c), not
shipped as is: `-Od` alone emits `inout` parameters, which native overloads must not have.
- One SPIR-V edit. A single instruction-level pass over the binary, never a regex over text,
strips capabilities and extensions, deletes the wrappers and moves their exports onto the
callees. It replaces the disassembly grep that the capability-strip probe used. It refuses a
wrapper that is not exactly one call. SPIRV-Cross then emits the overloaded names as they are,
so the build has no rename step and no name map.
- By-value parameters. The same pass makes every Function-pointer parameter a value parameter
(`byvalue.py` in the probe), so no emitted function can be `inout`. The build then runs exactly
`spirv-opt --eliminate-local-single-store --eliminate-dead-code-aggressive`, never an inliner
and never `--simplify-instructions` or `--ssa-rewrite`, which turn named locals into `_N` ids,
and SPIRV-Cross runs with `--relax-nan-checks`, so `min` and `max` are the plain intrinsics.
- The gate is the HLSL, not the old text. A cleaned route's lavapipe-glsl result must equal the
lavapipe-hlsl result bit for bit in every family (117 of 117 measured), and the C# judge's
bounds are re-recorded only where a family moved and equals its HLSL result.
- **CultPhasor's JSON keys (model page row).** Named by `CultMathJson`; written as `cosine`/`sine` from
`phasor-field-rename` on, and read as either those or the legacy `cos`/`sin`, forever unless a later
Breaking change removes the legacy names. Decided by ruling `phasor-fields-renamed-at-source`. The
MessagePack wire is positional and never named the fields, so `.cc` documents are untouched both ways.
A reader from before the rename skips the new keys and decodes zeros (its unknown-property rule), which
is why the change is Breaking in both changelogs: upgrade readers before writers.
- **The capacity through the build.** dxc cannot keep a macro and refuses spec-constant array sizes, so
the build turns the one array-length constant into a SPIR-V spec constant and `partition.py` maps
`CULTMATH_MAX_SITES` onto SPIRV-Cross's `SPIRV_CROSS_CONSTANT_ID_0` with source-copied header lines.
Rejected: building the GLSL per capacity (a consumer would run the build to change one number), and
rewriting `[128]` in the emitted text (the forbidden text post-pass). Pass 9d.
### Cut sequence (Pass 9; one line each, mapped when about to run)
1. `shader-conforming-run`: spec admitted. The lavapipe job with the C# device judge, run against
today's committed HLSL and GLSL, so the next cut is verified against a running check.
2. `glsl-build-dxc` (spec `cut-glsl-build-dxc.r2` admitted; r1's h1 left the gate red, Pass 9d): the build, not yet the switch. r2 merges main, carries site arrays and their capacity (a spec constant mapped to `CULTMATH_MAX_SITES`), the strict-form gate and the renamed phasor fields, and waits on `phasor-field-rename` (spec admitted: `cosine`/`sine` in C#, HLSL, GLSL and CultMathJson, legacy keys read; CultMath's next release is 0.5.0, a release cut of its own). `packages/cultmath/tools/glsl/`
holds `build-glsl.sh`, which pins dxc v1.9.2607 (the sha above), SPIRV-Cross and SPIRV-Tools
`vulkan-sdk-1.4.363.0`, plus one SPIR-V tool in Python. The build generates the export wrappers into
its temp directory (never committed), compiles source plus wrappers with `-T lib_6_3 -spirv -Od`,
and runs the one SPIR-V edit (strip, wrapper delete, export move, by-value parameters; Pass 9c).
It runs the pinned `spirv-opt` pair, `spirv-val`, and SPIRV-Cross ES 3.10 with `--relax-nan-checks`,
then partitions by origin file and emits the 13 constants. Output goes to a directory the caller
names. The committed GLSL, `GlslLowering.cs` and its tests are untouched, and the lavapipe job gains
one switch to run a built directory in place of `shaders/*.glsl`. Waits on 1, which is merged.
2b. `glsl-switch` (mapped when 2 is verified): commits the build's output with the licence lines and
guards, deletes `GlslLowering.cs` and the lowering contract tests, makes the lavapipe job check
the committed files against the build, re-records the lavapipe bounds for moved families
(Pass 9c's gate), and pins `get-spirv-cross.ps1`. If `trophic-nodes` merges first, the build
covers its four new overloads with nothing extra.
3. `angle-capture` (stage-shaders-device part 2): the Starfire native-ANGLE tool on the shared work
list (ES 3.00 fragment flavour, RGBA32UI readback), loading a pinned ANGLE build rather than Docker
Desktop's copy. Captures carry the renderer, ANGLE version and revision, the shader sha and per-family
argument hashes and bits, and the ordinary test run judges them. A shader-sha change marks them stale.
It settles `span-allowance-gpu-margin`'s six CPU survivors, `iv-exp-gpu-widening`'s measurement and
`device-p2-pin-notes`, and closes `glsl-browser-parity-run` and `affine-device-runner-seed`. It waits
on 2b, because a capture of the old lowering goes stale at the switch.
### shader-lattice-loop (follow-ups `shader-weighted-lattice-loop`, `lattice-shader-pins`)
Written by Imagination (agent `imagination-ag4-shader-loop`, session `self-2026-10-16-ag4`) against `origin/main`
`e2051c5a` and the build branch at `23ac4e89`. Research and probes: `docs/research/shader-variable-inputs.md`.
Spec: `cultmath-tapes:cut_spec:cut-shader-lattice-loop.r1`. Question: `shader-site-capacity` (gates the spec).
- **Bodies reach the shader as `float4 sites[CULTMATH_MAX_SITES]` plus `int count`, by value.** A caller accessor
macro cannot survive the library build (the built GLSL has no consumer macros), and library-owned uniforms would
take a binding the consumer owns. dxc, the SPIR-V edit, SPIRV-Cross and a WebGL2-style uniform-array consumer all
accept the array form (probes P1-P3).
- **The shader owns the whole search.** `cultmath_weighted_lattice_cells` and `cultmath_power_border` replace the two
comment recipes; the four steps stay public. The box pass recomputes distances (no shader cache), the ring loop is
bounded by 37 with C#'s exit.
- **The lavapipe run kills the exit and the normaliser** (probe table, research section 3): M1-M6 fail the C# device
judge; M7, a late exit, is equivalent by the window-independence proof. M1 needs several far-ring points.
- **Smooth-step cap point** joins its family (m = f1 + smoothing, d past it, smoothing >= 0.5).
- **Yggdrasil's sdk:10.0 Ubuntu image owns `glsl-parity.json`'s platform values** with the lavapipe bounds; a write
on another OS is refused.
- Left: `shader-span-functions-array-form` (power_cells, multiplicative_cells, soft_gabriel_gate array forms).

25
docs/cultmath-tape-target.md
View file

@ -108,15 +108,18 @@ source.
- Asura does not need this campaign. Its field is noise-dominated, and its
`A_max` shell and band-limiting already give the cheap culling.
## Open before unparking
## Decided since the unpark, and what stays open
- Run the (a) spike first, or go straight to (c)?
- Pick the interval representation (plain intervals or affine arithmetic) and
the noise bound strategy.
Affine forms for noise and warp were unparked on 2026-10-03 (ruling
`cultmath-tapes:ruling:affine-forms-unparked`): one shared symbol, mapped in
`docs/cultmath-interval-ground-cut.md`, "Pass 5".
- Choose the bytecode register model for the HLSL interpreter, given GPU
register pressure.
- Decide whether the brush-grammar station project lives in Aetheria or in
its own repo.
- The spike-or-(c) choice was settled by the operator's unpark: the tape
unparked for Aetheria's trophic species graphs (ruling
`cultmath-tapes:ruling:tape-unparked-for-trophic-graphs`).
- Interval representation: affine forms for noise and warp, unparked on
2026-10-03 (ruling `cultmath-tapes:ruling:affine-forms-unparked`): one
shared symbol, mapped in `docs/cultmath-interval-ground-cut.md`, "Pass 5".
- Register model: the HLSL interpreter keeps its registers in groupshared
memory, a slice per thread, linear-scan allocated at lowering (ruling
`cultmath-tapes:ruling:tape-register-model-ruled`; research in
`docs/research/tape-register-model.md`).
- Open: whether the brush-grammar station project lives in Aetheria or in
its own repo (follow-up `cultmath-tapes:follow_up:brush-grammar-station-repo`,
raised as an operator question when a station cut reaches the frontier).

286
docs/cultmesh-ts-content-map.md Normal file
View file

@ -0,0 +1,286 @@
# TypeScript CultMesh Content Plane: Map
Campaign `cultmesh-ts-content`. Imagination wrote this on 2026-10-19 at CultLib
`8990bd46` (origin/main `a88c629e` plus the target doc), and refreshed it the
same day at `2de07065` for C2 and C3, reading C1 from `b8b825e1`. The target
is `docs/cultmesh-ts-content-target.md`. The HTTP prior art is
`docs/research/2026-10-19-content-addressed-http-answers.md` (cited below as
**R**, with its section number). This page holds the body facts, the model
page, the rationale and the cut sequence. Specs and questions live in the
typed mind, not here.
## 1. Body facts
F1-F18 were probed by running code or read from source at `8990bd46`. F19-F27
were probed or read at `b8b825e1` (C1's head) and `2de07065`. F28-F33 were
probed on 2026-10-19 over the C2+C3 merge tree (`7adbf3cb` merged with C3's
head, extracted by `git archive`), on Windows and in a Linux container. Scratch probes
ran under the session scratchpad (`imagination/`), not in the repo.
| # | Fact | Evidence |
|---|---|---|
| F1 | A C#-packed manifest is a MessagePack **fixarray of 10** (`0x9a`): `[artifactId, kind, version, contentHash, sizeBytes, mimeType, createdAtUtc, chunks[], tags[], metadata{}]`. Each chunk ref is a 4-array `[chunkHash, offset, sizeBytes, recordKey]`. The fixture manifest is 1,111 bytes: 6 chunks (5 x 262,144 + 4,831), content hash `f4ae2a7c…2af1`. The empty body packs to 315 bytes with one zero-size chunk whose hash is `e3b0c442…b855` (the SHA-256 of nothing). | probe: `@msgpack/msgpack` 3.1.3 decode of `contracts/cultmesh/content-vectors.cs-written.json`, kind `manifest` |
| F2 | `@msgpack/msgpack` 3.1.3 re-encodes all four C# manifest vectors **byte-identically**, metadata included (`zeta, alpha, mid` keep insertion order). Integers come out minimal (`4831` is `cd12df`, `262144` is `ce00040000`), as MessagePack-CSharp writes them. | same probe, `encode(decode(bytes))` equals the input for all four |
| F3 | Three decoder hazards for a TS manifest reader. (a) JS objects put integer-like keys first (`{zeta,"2",alpha,"1"}` iterates `1,2,zeta,alpha`), so metadata must be ordered entries, not an object, or a `{"10":…,"2":…}` manifest re-encodes in another order. (b) Plain `decode` takes the last of duplicate map keys silently (`{"a":"2"}`); C# `Dictionary` and Rust `ordered_map` refuse duplicates. `new Decoder({ mapKeyConverter })` can refuse them: probed. (c) Plain `decode` rounds int64 above 2^53 (`…993` reads as `…992`); `useBigInt64: true` keeps it exact. | probe (node) |
| F4 | `@msgpack/msgpack`'s decoder is iterative: a 1,000,000-deep `0x91` input decodes in 160 ms but costs about 158 MB of heap. So nesting does not crash it but amplifies memory about 150 times. `checkPayloadDepth` (the owner of ruling `payload-depth-limit-256`) is not exported from any `@gamecult/cultcache-ts` entry: `document.ts:108` exports only the error and the constant. | probe (node); source `packages/cultcache-ts/src/document.ts:2,108,123` |
| F5 | The C# HTTPS connector sends `GET <endpoint>?chunkHash=<hex>&recordKey=<escaped>`, **without** a `sha256:` prefix, because `BuildChunkUri` runs `NormalizeHash` first. Probed URIs: `https://ygg.example/content?chunkHash=5993ac…&recordKey=mesh%3Acdn%3Achunk%3A5993ac…`. An existing endpoint query is kept and joined with `&` (`?tenant=sp&chunkHash=…`). A null record key sends `recordKey=` (empty). Other characters use `Uri.EscapeDataString`, so on the wire `k with space/ü&=?#` becomes `k%20with%20space%2F%C3%BC%26%3D%3F%23`. | probe: `BuildChunkUri` lines copied into a net10.0 console (`CultMeshHttpsContentTransport.cs:114-122`) |
| F6 | The connector calls `EnsureSuccessStatusCode` and fails on any status outside 2xx. When `Content-Length` is present and is not the chunk's `SizeBytes`, it throws before reading. With no `Content-Length` it copies the stream without checking the length. Verification belongs to the caller's transfer owner. `CanConnect` admits only `https`. The constructor takes a `Func<HttpMessageHandler>`, so a test can pin a self-signed certificate without modifying the connector. | source `CultMeshHttpsContentTransport.cs:18-29,92-104` |
| F7 | `NormalizeHash` (C#) strips a case-insensitive `sha256:` prefix *before* trimming, then `Trim().ToLowerInvariant()`. It never checks for hex or a length of 64. So the record-key grammar accepts any non-blank string, and a provider must add its own hex check before a hash can become a file name. | source `CultMeshCdn.cs:549-560` |
| F8 | `StableToken` (the token in a manifest record key) is a C# rule with no Rust mirror: `cultnet-rs/src/content.rs` builds no artifact record key. Its Unicode semantics differ from the naive JS spelling in four probed ways. .NET `ToLowerInvariant` leaves `U+0130` unchanged, while JS gives `i` plus `U+0307`. .NET `Trim` keeps `U+FEFF` (and trims `U+0085`, per `char.IsWhiteSpace`), while JS `trim` drops `U+FEFF` and keeps `U+0085`. JS whole-string lowercasing applies final-sigma (`ΑΣ` becomes `ας`), while per-unit lowercasing gives `ασ`. `char.IsLetterOrDigit` is false on a lone surrogate and on `U+00B2`, which JS `\p{L}\p{Nd}` agrees with only when it is tested per UTF-16 unit. | probe (net10.0 console and node); source `CultMeshCdn.cs:562-580` |
| F9 | `CreatedAtUtc` defaults to `DateTimeOffset.UtcNow.ToString("O")`, which looks like `2026-10-19T12:07:25.5235579+00:00`: 7 fractional digits and `+00:00`, never `Z`. | probe (net10.0) |
| F10 | `contracts/cultmesh/content-vectors.cs-written.json` and `.rs-written.json` are written by `tests/GameCult.Mesh.Tests/CultMeshContentVectorTests.cs` and `packages/cultnet-rs/tests/content.rs`, under `CULTNET_WRITE_VECTORS=1`. Each side judges both files. Kinds are `request`, `response` and `manifest`. Rust panics on an unknown kind (`tests/content.rs:302`), and C# requires the Rust labels to equal its own (`CultMeshContentVectorTests.cs:54`). **So a new kind or label in the cs-written file breaks both siblings.** A TS file must be a separate file. | source |
| F11 | **Neither content-vector suite runs in CI.** `cultmesh-portability.yml` filters `dotnet test` to `AuthorityProof`, `RealtimeWireProtocol` and a lease churn test. `cultnet-interop.yml` runs `cargo test … rudp` (a name filter) and the Networking `Rudp` filter. The C#/Rust content parity has only ever been run by hand. | grep of `.github/workflows/*.yml` |
| F12 | TS→C# vector precedent: `scripts/write-cultmesh-realtime-frame-vectors.mjs` writes `realtime-frame-vectors.ts-written.json` from built `dist`, deliberately from **different inputs** than the C# file. `CultMeshRealtimeWireProtocolTests` judges it, and `cultmesh-portability.yml:145-149` runs it. | source |
| F13 | Cross-process precedent for C# against a Node provider exists. `packages/cultnet-ts/test/interop/cultnet-interop.test.ts:536-560` spawns the TS QUIC provider, then runs `dotnet test --filter NativeConnectorReceivesFromExternalProvider` with the endpoint in `CULTMESH_NATIVE_EXTERNAL_ENDPOINT`. The C# test skips without the variable (`tests/GameCult.Mesh.Quic.Native.Tests`, line 21). | source |
| F14 | `cultnet-ts` is CommonJS. Its `./wire`, `./contracts` and `./authority` subpaths import only `@msgpack/msgpack`, `ajv` and `cultcache-ts/document`, so they are browser-safe. Its root `index` pulls `node:` modules (`peer`, `rudp`, `transport`, `security` and others). `cultmesh-browser` (ESM) already imports `cultnet-ts/wire`, `/contracts` and `/authority`. `cultmesh-ts` loads `koffi` lazily, inside a function (`realtime-quic-native.ts:139`), so a new subpath of it pays no native load. The root has `esbuild` ^0.28.1 as a devDependency. | source, grep |
| F15 | TS packages build with `tsc` and test with `node --test dist-test/test/*.test.js` per workspace. `scripts/run-typescript-workspaces.mjs` runs them in order `cultcache-ts, cultnet-ts, cultmesh-ts, cultmesh-browser`, after `check-typescript-tests-wired.mjs`. No StrykerJS config exists in CultLib (`docs/mutation-testing.md:156-159`). | source |
| F16 | **No CultLib TS package except `cultcache-ts` has a publish job.** `publish-packages.yml` triggers on `cultcache-ts-v*`, `cultcache-py-v*`, `cultnet-py-v*` and `cultmesh-py-v*` only. Tags `cultnet-ts-v0.1.1` and `cultmesh-ts-v0.1.1` exist, but no workflow consumes them. `npm view` returns 404 anonymously for `cultnet-ts`, `cultmesh-ts`, `cultmesh-browser` and even `@gamecult/cultcache-ts` (`@gamecult/cultcache-ts` may be private; unverified). The doctrine's rule of at most three tags per push stands (SKILL.md:674). | source; `npm view` |
| F17 | StreamPixels consumes CultLib as the git submodule `vendor/CultLib` (gitlink `30ee8b97`) with `file:` deps and `pnpm.overrides`. Its CI and Idunn recipes build the vendored workspaces (`npm ci --ignore-scripts`, then build `cultcache-ts`, `cultnet-ts` and `cultmesh-ts`). `apps/web` depends only on `cultnet-ts` today, not on `cultmesh-browser`. | StreamPixels `.gitmodules`, `apps/*/package.json`, CI workflow lines 62-95 |
| F18 | The Rust reader sets two guards that the TS reader should match. A caller cap `max_bytes` is checked before any request. And a zero-size chunk is refused unless it is the only one, because a manifest of zero-size chunks would be a request flood. | source `cultnet-rs/src/content.rs:364-385` |
| F19 | `BuildChunkUri` on more endpoint shapes, as `AbsoluteUri` (what `HttpClient` sends). `https://h` and `https://h/` both give `https://h/?chunkHash=…&recordKey=k`. `https://h:443/x` drops the default port. `https://H.Example/X/` lowercases the host and keeps the path's case. `https://h/x#frag` keeps the fragment after the query. `https://h/x?a=%20b+c` keeps the existing query verbatim and joins with `&`. **`https://h/x?` (an empty query) gives `https://h/x?&chunkHash=…`**, because `UriBuilder.Query` is `?`, which is not blank. `https://[::1]:8443/c` keeps the bracketed host. `Uri.EscapeDataString("!'()*~-._ +é𐐀")` is `%21%27%28%29%2A~-._%20%2B%C3%A9%F0%90%90%80`: unlike JS `encodeURIComponent`, it escapes `!'()*`. | probe: `dotnet fsi` on .NET 10, the three lines of `BuildChunkUri` (`CultMeshHttpsContentTransport.cs:114-122`) copied verbatim |
| F20 | Node 24.15 global `fetch` (undici) against a raw-socket server. A gzip body (`Content-Encoding: gzip`, `Content-Length: 29`) arrives as 1,000 decoded bytes, while `headers.get("content-length")` still says `29`. A body shorter than its `Content-Length` (400 of 1,000) rejects while reading: `TypeError: terminated`, cause code `UND_ERR_RES_CONTENT_LENGTH_MISMATCH`. A body longer than its `Content-Length` (1,000 against 400) is cut silently to 400 by HTTP framing. With no `Content-Length`, all 1,000 bytes arrive. So a reader that trusts the header refuses valid gzipped content, and it learns nothing the received bytes do not tell it (R §6: Fetch decodes content codings, and its own note calls `Content-Length` unreliable). | probe: `imagination/fetchprobe.mjs` |
| F21 | The C# TCP content server is the reference for what a provider does per request (`SendChunkAsync`, `CultMeshTcpContentTransport.cs:384-430`). It runs `NormalizeHash` on the requested hash, refuses a non-blank `RecordKey` that is not `CreateRecordKey(hash)` (ordinal compare, message "record key disagrees with its content hash"), resolves the chunk **by hash** (`_resolveChunk(hash)`), and runs `CultMeshCdn.ValidateChunkPayload` before it sends a byte. So **verify-before-serve on every request is the reference behaviour**. C# chunk storage is one record per chunk, keyed by the chunk record key (`CultMeshDatabaseContentProvider`, `CultMeshContentTransfer.cs:41-71`). Per-chunk storage is the reference layout. | source |
| F22 | C1's exports at `b8b825e1` (`packages/cultnet-ts/src/content.ts`): `normalizeHash`, `parseSha256Hex`, `sha256Hex`, `chunkRecordKey`, `artifactRecordKey` (async), `validateManifest`, `encodeManifest`, `decodeManifest`, `packContent`, `CULTMESH_CDN_MANIFEST_SCHEMA`, `DEFAULT_CHUNK_SIZE_BYTES` (4,194,304), `EMPTY_SHA256`, and the types `CultMeshCdnChunkRef`, `CultMeshCdnArtifactManifest`, `CultMeshCdnPackOptions` and `CultMeshCdnPackedChunk`. `sha256Hex` throws a plain `Error` ("WebCrypto SHA-256 is unavailable.") when `crypto.subtle` is missing. `validateManifest` takes the whole manifest (it reads `metadata`). `cultnet-ts/authority` already exports `isLoopbackEndpoint` (`cultmesh-authority.ts:377`: `localhost`, `loopback`, `::1`, `::ffff:7f00:1`, `127.x.x.x`), and `cultmesh-browser` already imports it (`index.ts:23`). | `git show b8b825e1:…`; cut report `cut-c1-shapes.h1` landed names |
| F23 | The consumer (StreamPixels `dfbe2a6`, `docs/web-character-customization-map.md`) stores one whole-artifact file per content hash at `/srv/streampixels/content/<hex>`, builds manifests at service boot, and hashes every referenced blob at boot (lines 149-159, 175, 317-325). Its artifacts are 10-125 KB (W10), well under the 4 MiB default chunk. It also builds manifests for `catalog.cc` and `character-artifacts.cc`, whose sizes the map does not give. It mounts the handler at `GET /api/content` on its existing server (line 320). Its sketch, `createHttpsContentHandler(store)` (line 293), takes `(chunkHash, recordKey)` and returns bytes or not-found. | StreamPixels `git show dfbe2a6:docs/web-character-customization-map.md` |
| F24 | Build shapes for the two packages. `cultmesh-ts` is CommonJS (`module` and `moduleResolution` `Node16`), and its `package.json` exports only `.` (lines 7-13). `cultmesh-browser` is ESM (`"type": "module"`, `moduleResolution` `Bundler`) and is one file, `src/index.ts`, with no relative imports. A new relative import must therefore be spelled `./content.js`, or Node's ESM loader cannot find it at test time. Both packages' `test` scripts run the glob `dist-test/test/*.test.js`, which the wiring check accepts (`scripts/check-typescript-tests-wired.mjs`). CI runs both in `cultmesh-portability.yml`: the browser at lines 119-120, and `cultmesh-ts` at lines 121-129 under the step name "Verify the realtime frame codec with the TypeScript codec". `esbuild` resolves from the root `devDependencies`, as C1's browser-bundle test shows. | source at `b8b825e1` |
| F25 | The existing TLS fixture `packages/cultmesh-ts/test/fixtures/quic-test.p12` is a self-signed P-256 certificate with subject `CN=cultmesh-quic-native-tests` and no subjectAltName (fixtures README). Node's TLS hostname check therefore refuses it for `127.0.0.1` even when it is trusted through `NODE_EXTRA_CA_CERTS`. A C# handler that pins by certificate hash does not care about the name. | source: fixtures README |
| F26 | `CultMeshCdn.ValidateChunkPayload`, `HashBytes`, `ValidateManifest` and `NormalizeHash` are `internal`, and `GameCult.Mesh` declares no `InternalsVisibleTo`. A test in `tests/GameCult.Mesh.Tests` can therefore verify chunk bytes only with `SHA256.HashData`, or through a public consumer of the connector. The public surface is `CreateRecordKey`, `PackArtifact`, `PublishAsync` and `ReadArtifact(Async)`. | grep of `CultMeshCdn.cs`, `src/GameCult.Mesh` |
| F27 | No CultLib workflow runs `cultmesh-ts` interop tests. `cultnet-interop.yml` (windows-latest, main only) runs `cultnet-ts`'s `test/interop`, and its QUIC native case spawns `dotnet test --filter NativeConnectorReceivesFromExternalProvider` through `runDotnetTestFilter` (`cultnet-interop.test.ts:550,2826`). `cultmesh-ts` has a `test/interop/` folder (a QUIC peer script) but no `test:interop` script. | grep of `.github/workflows`, source |
| F28 | **The real provider meets the real reader across processes, on Windows and Linux.** C2's handler, store and `toNodeListener` served on `node:https` with a self-signed P-256 certificate (subjectAltName `IP:127.0.0.1, DNS:localhost`, `CA:FALSE`, `serverAuth`, 100-year validity). They served a 4-chunk artifact (3.5 x 64 KiB) and an empty one. C3's `fetchVerifiedContent` ran in a child `node` with `NODE_EXTRA_CA_CERTS` set to that certificate, through the real global `fetch`, and returned both bodies with the right hashes. Without the variable it rejected `network` (cause `DEPTH_ZERO_SELF_SIGNED_CERT`). Behind a test wrapper that flips byte 0 of every non-empty 200 body, it rejected `chunk-hash-mismatch`. Platforms: Windows (Node 24.15) and a `node:24-trixie` container (Node 24.21). | probe: `imagination/probe/{serve,read,drive}.mjs` over the C2+C3 merge tree `8d1482c8` |
| F29 | **The C# consumer that verifies is `CultMeshContentTransferService`, and it runs against an HTTPS route without Odin.** The probed path: `CultMeshDiscoveryService` over a test `RouteSource(<https endpoint>)`, then `CultMeshSessionManager` with `LocalDevelopment` trust and the unmodified `CultMeshHttpsContentTransportConnector`, then `CultMeshSessionContentProvider`, then `FetchAsync(manifest)`. This is the shape of `CultMeshContentSessionTests.TcpContentSessionStreamsLargeBodyAndReusesCommittedBody` with the TCP connector swapped for the HTTPS one. It fetched both artifacts from the TS provider, decoding the **TS-encoded** manifest bytes with `CultDocumentMessagePackSerialization.DeserializeUntyped`. Trust was a `SocketsHttpHandler` whose `SslOptions.CertificateChainPolicy` is `CustomRootTrust` with only the fixture certificate in `CustomTrustStore` (`RevocationMode.NoCheck`). That is real chain and name validation, with no callback and no system roots. The default handler failed with `UntrustedRoot`. Against the tampering wrapper, `FetchAsync` threw `AggregateException` over `InvalidDataException` ("CDN artifact chunk hash does not match its manifest."), leaving only `.<hash>.partial`, never `<hash>.body`. The connector itself does not verify (F6). The C# verifier is the transfer service's `BoundedHashingWriteStream` per chunk and its whole-file hash. The results on the Linux container (self-contained linux-x64 publish) were identical. | probe: `imagination/csprobe/Program.cs` and `probe/drive-cs.mjs`; source `CultMeshContentTransfer.cs:115-330`, `CultMeshContentTransports.cs:138-160` |
| F30 | The two consumers request the same URLs. C# sent five requests: four chunks, plus the empty chunk `e3b0…`, which the provider answers intrinsically. Each was `/api/content?chunkHash=<hex>&recordKey=mesh%3Acdn%3Achunk%3A<hex>`. The reader sent the same four chunk URLs, and nothing for the empty chunk, which it verifies locally. | same probes, the provider's request log |
| F31 | **Build order decides where the cross-process test lives.** `scripts/run-typescript-workspaces.mjs` builds `cultcache-ts, cultnet-ts, cultmesh-ts, cultmesh-browser`, and each package's `test` compiles `test/**/*.ts`, `test/interop` included. A `cultmesh-ts` test that imports `cultmesh-browser` therefore fails to compile on a clean checkout, while a `cultmesh-browser` test that imports `cultmesh-ts/content` follows the order. Named ESM imports of the CommonJS `cultmesh-ts/content` work from an ESM file: all four provider names import as functions. At `7adbf3cb`, `cultmesh-portability.yml` runs the browser step (lines 119-120) **before** the `cultmesh-ts` step (129-130), which is the opposite of the script's order. The wiring check requires `test/interop/*.test.ts` to be run by a `test:interop` script. | probe (node, ESM import); source |
| F32 | Node's `execFile("dotnet")` resolves `dotnet.exe` on Windows without an extension. The C3 merge into `7adbf3cb` is clean at C3 head `67ff3f64` (`git merge-tree` gives `666162a3`). It touches the workflow only at comment lines 105-107, and the reader's exports keep their names. The probe's body pattern `(i*31+7)&255` repeats every 256 bytes, so all of its 64 KiB chunks shared one hash. A parity fixture needs non-periodic bytes, or chunk order is invisible. | probe (node); `git merge-tree` |
| F33 | `cultmesh-portability.yml` triggers on `push` only for `main` and `codex/**`. A `hands/**` branch runs it through `pull_request` or `workflow_dispatch`, and both triggers exist. The job `reactive-and-resource-lifetime` has .NET 10 and Node 24 on `windows-latest` and `ubuntu-latest`, and it already builds `tests/GameCult.Mesh.Tests` in earlier steps. | source at `7adbf3cb` |
## 2. Model page
There is one row per persistent or wire-visible kind. Identity is what names the
kind, lifecycle is what happens to it over time, and authority is who decides.
The HTTP answers are new design: no runtime has an HTTPS content server (target,
"Why now"). They rest on R, which fetched the specs and sources on 2026-10-19
and corrected this page's first draft (R §8). In short:
- **404 for absent** is universal (R §9). Nix also reads 410 and S3's 403 as
absent; we send 404 only.
- **400 for a malformed id** comes from the IPFS path gateway (generic client
error, with a malformed CID as its example) and bazel-remote (a hash that is
not 64 lowercase hex). The OCI blob-pull text defines no 400; its
`DIGEST_INVALID` is an upload code, so the first draft was wrong to cite it.
- **No system specifies a server answer for a stored object that fails its
digest** (R §9). Every one puts verification on the client. bazel-remote
streams without checking, so a failure after the headers have gone "cannot
change the status" (R §4). That is the scar `verify-before-serve` avoids by
buffering and hashing before the status line.
- **Range** may be ignored (RFC 9110 §14.2). OCI says registries SHOULD
support it, which the first draft called "optional". We ignore it.
- **`Cache-Control: public, max-age=29030400, immutable`** is the IPFS path
gateway's MUST for immutable content (R §1). The trustless gateway itself
does not state it.
- **The Fetch facts** decide the reader (R §6). Fetch decodes content codings
before the page sees the body, so `Content-Length` describes the encoded
body, which F20 reproduces in Node. `Content-Length` is CORS-safelisted, and
`ETag` is not. `Access-Control-Allow-Origin: *` passes the CORS check for any
request whose credentials mode is not `include`.
| Kind | Identity (what names it) | Lifecycle | Authority (who decides; forbidden writers) |
|---|---|---|---|
| **Manifest** `gamecult.mesh.cdn_artifact_manifest.v1` | The record key (row below), derived from `artifactId`, `version` and `contentHash`. On the wire it is the 10-array of F1. Its namespace is the schema id. Two manifests for one body under different ids or versions are distinct and legitimate. | Immutable once packed. A new body or version is a new manifest and a new key. Nothing revises one in place, and deletion is not in scope (no GC). `createdAtUtc` is set at pack, and a caller may pin it so that repacking is reproducible. | `packContent` in `cultnet-ts/content` (C1, `b8b825e1`) is the only TS writer. The encoder and decoder in the same module are the only TS byte owners. The reader takes a manifest value and never builds or edits one. **Forbidden:** a second TS manifest type or encoder in `cultmesh-ts`, `cultmesh-browser` or a consumer, and any URL or endpoint field. |
| **Chunk ref** `CultMeshCdnChunkRef` | Its `chunkHash` (SHA-256 of the chunk bytes) within one manifest, plus `offset`. Two refs may share a hash: this is legal, and the reader fetches each ref. | Immutable inside its manifest. `recordKey` is derived and may be blank on read. A blank key goes on the wire as `recordKey=` (F5), and the provider accepts it. | The packer writes. `validateManifest` (C1) judges contiguity, non-negative sizes and the size sum. The reader and provider never edit one. |
| **Chunk record key** `mesh:cdn:chunk:<hex>` | Fully derived from `normalizeHash(chunkHash)`. It is injective only over valid hex, because C# accepts any non-blank string (F7). | Derived on demand, never stored independently of a manifest. | `chunkRecordKey` in `cultnet-ts/content`. On the HTTP wire the provider **checks** a non-blank `recordKey` against it, as the C# TCP server does (F21), and never derives a file from it. |
| **Artifact record key** `mesh:cdn:artifact:<token>:<versionToken\|unversioned>:<hex>` | Derived by `StableToken` (F8). It is not injective: `A/b` and `a b` both give `a-b`, and the content hash at the end disambiguates. | Derived on demand. | `artifactRecordKey` in `cultnet-ts/content` (async; C1 deviation). Truth comes from the C#-written `contracts/cultmesh/cdn-record-key-vectors.json`. Neither C2 nor C3 uses it. |
| **Content hash / chunk hash** | Lowercase 64-hex SHA-256. On input, `normalizeHash` accepts `sha256:`, any case and surrounding whitespace, as C# does. The HTTP boundary and the reader also demand 64 hex through `parseSha256Hex`. | Immutable. It *is* the identity. | `sha256Hex` and `parseSha256Hex` (C1) are the only TS hasher and the only hex rule. The provider and reader call them and keep no copy. The negative greps pin this. |
| **Stored chunk** (provider disk) | File `<root>/<hex>`, flat, one file per distinct chunk hash. This is the reference layout (F21: C# stores one record per chunk). For a single-chunk artifact the chunk hash equals the content hash, so StreamPixels' whole-artifact file `/srv/streampixels/content/<hex>` (F23) is already a stored chunk. A multi-chunk artifact's publisher writes `packContent(...).chunks`, one file each. | Written by the publisher outside the provider, and never written by the provider. Immutable. Deletion has no owner (GC is out of scope). A corrupt or oversized file stays on disk and is never served. | The publisher writes; **the provider is read-only**. The provider verifies every read against its name before serving (`verify-before-serve`, the C# per-request behaviour of F21). A symlink or a replaced file cannot leak anything, because only bytes that hash to the requested name are ever served. **Forbidden:** any path derived from `recordKey`, the URL path or an unvalidated hash. |
| **Provider store index** | **None.** The file name is the index. The directory store maps hex to `<root>/<hex>` on each request, after its own `parseSha256Hex` equality check. No in-memory table exists, so nothing can go stale. | Not applicable: it is computed per request. | `ContentChunkStore` is the port: `readChunk(chunkHash, maxBytes) → Promise<Uint8Array \| undefined>`. `createDirectoryChunkStore(root)` stats before reading and rejects a file over `maxBytes` with `ContentChunkTooLargeError`, so an oversized file is never read into memory. A host may supply another store, for example one indexed from manifests, and the handler verifies whatever any store returns. |
| **The empty chunk** `e3b0c442…b855` | The SHA-256 of zero bytes. | Intrinsic: it needs no file. | The provider answers it as 200 with `Content-Length: 0` without consulting the store, so empty artifacts work against any store (bazel-remote does the same, R §4). The reader verifies it locally and sends no request. |
| **Wire request** | `GET` or `HEAD <mount>?chunkHash=<hash>&recordKey=<key>`. Extra query parameters are ignored, because an endpoint may carry its own (F5). The mount path belongs to the host, and the handler never reads it. | Stateless. | Built by the C# connector or the TS reader. The TS reader builds the URL **byte-identical** to `BuildChunkUri`, `Uri.EscapeDataString` escaping (F19) and the empty-query `?&` join included, so C# and browser clients share HTTP cache entries. |
| **Content coding** | Identity, always. | Not applicable. | Provider: no `Content-Encoding`, and `Cache-Control` carries `no-transform`, which forbids an intermediary to transform the content (RFC 9111 §5.2.2.6). That keeps the C# connector's `Content-Length` check meaningful (F6). The reader does not rely on it: it judges the bytes it received (F20). |
| **Answer 200** (GET) | The body is exactly the verified chunk bytes. | Immutable per URL. | Provider. Headers: `Content-Length: <size>`, `Content-Type: application/octet-stream`, `X-Content-Type-Options: nosniff`, `Cache-Control: public, max-age=31536000, immutable, no-transform` and `Access-Control-Allow-Origin: *`. Octet-stream with `nosniff` keeps a public chunk from rendering as HTML on the host's origin. The IPFS `Content-Disposition: attachment` adds nothing over that for a fetch client, so it is not sent. |
| **Answer 200** (HEAD) | The same headers as GET, with no body. | Same as GET. | Provider. HEAD verifies exactly as GET does, so it can never advertise a corrupt chunk. RFC 9110 §8.6 requires HEAD's `Content-Length` to equal GET's. |
| **No ETag, no 304** | Not applicable. | Not applicable. | Provider. `immutable` with a one-year max-age means browsers do not revalidate. `ETag` is not CORS-exposed (R §6), and the URL is already the content identity. An `ETag` without a 304 path is noise, and a 304 path is a second answer to test. The first draft had both, and both are cut. |
| **Answer 400** | `chunkHash` missing, repeated, or not 64 hex after `normalizeHash`. Or `recordKey` repeated, or non-blank and not exactly `mesh:cdn:chunk:<hex>` (the C# server's "record key disagrees" rule, F21). | Stateless. | Provider. Body is a fixed `text/plain; charset=utf-8` reason with no echo of input (bazel-remote echoes the URL; we do not). Headers: `nosniff`, `Cache-Control: no-store`, `Access-Control-Allow-Origin: *`. |
| **Answer 404** | A valid hash whose chunk the store does not hold. | Stateless; a later publish can turn it into 200. | Provider. Same error headers as 400. `no-store` keeps an absence from being cached as immutable. |
| **Answer 405** | Any method but GET and HEAD, OPTIONS included. | Stateless. | Provider, with `Allow: GET, HEAD`. The reader's GET is a CORS simple request (no custom headers, credentials omitted) and never preflights. A client that adds headers would preflight, and its preflight is refused, which is correct because the provider honours no request header. The first draft's 204 preflight answer is cut. |
| **Answer 500** | The store returned bytes whose SHA-256 is not the requested hash; the store holds a file over `maxChunkBytes`; or the store failed. | Persists until the operator replaces the file. | Provider. It sends a fixed reason and none of the chunk's bytes, then calls the host's `onServerFault({ chunkHash, kind, error })` with `kind` ∈ {`hash-mismatch`, `too-large`, `store-error`}, so the host can fail health. A throwing hook does not change the answer. It is 500 and not 404 because the IPFS trustless gateway reserves 500 for "genuine server errors" (R §1), and corruption is one the operator must see, while a 404 would hide it as absence. |
| **Range** | Not supported. A `Range` header is ignored, and the answer is 200 with the full chunk (RFC 9110 §14.2). No `Accept-Ranges` header is sent. | Not applicable. | Provider. The chunk is the resumption unit, and resumable TS state is out of scope. |
| **Provider memory** | At most `maxChunkBytes` buffered per in-flight request. | Per request. | Provider. Buffering is forced, because the status line cannot be sent before the hash is known (the bazel-remote scar above). `maxChunkBytes` defaults to `DEFAULT_CHUNK_SIZE_BYTES` (4 MiB) and must lie in 1..2147483647. The host bounds concurrency (server `maxConnections` or a reverse proxy), and the README says so. |
| **Reader result / error** | A `Uint8Array` of the whole body, or one `CultMeshContentError` with `code` ∈ {`manifest-invalid`, `too-large`, `insecure-endpoint`, `webcrypto-unavailable`, `network`, `http-status`, `length-mismatch`, `chunk-hash-mismatch`, `content-hash-mismatch`}, plus `chunkHash` and `status` where they apply. A caller abort rejects with the signal's own `reason`, unwrapped. Bad options reject with `RangeError`. | Transient, held in memory only. No byte of a failed chunk or body is returned, in whole or in part (`verify-before-use`). | `fetchVerifiedContent` (and `fetchVerifiedChunk`) in `cultmesh-browser`. Concurrency defaults to 4 and is bounded 1..32 (the C# `CultMeshContentTransferOptions` bounds). `maxBytes` defaults to 256 MiB and is checked before any request. The scheme must be `https:`, or `http:` where `isLoopbackEndpoint` (F22) holds. Only status 200 is accepted. **`Content-Length` is not consulted** (F20). The body is read through a stream into the chunk's slot, a body that runs past the chunk's size is cancelled at the first excess byte, and one that ends short is refused. |
| **Endpoint** | A URL the caller resolved through CultMesh. | Per call; never persisted in content. | The caller. **Forbidden:** any manifest, chunk-ref or record field carrying it (`no-urls-in-content`). Error messages never echo it, because an endpoint query may carry a tenant or a token. |
| **Vector files** | `content-vectors.cs-written.json` (C#), `.rs-written.json` (Rust), `content-vectors.ts-written.json` (TS, manifests only, own inputs) and `cdn-record-key-vectors.json` (C#, record-key traps), all landed by C1. | Each is rewritten only by its writer under `CULTNET_WRITE_VECTORS=1`. CI never sets the variable: the committed files are the contract. | Each writer owns its file. C2 and C3 add no vector file. HTTP behaviour is pinned by tests and, in C4, by the C# transfer service driving the unmodified connector (F29). |
Two cells rest on reasoning rather than a probe: that the host, not the
provider, bounds concurrency, and the cost of hashing on HEAD. Both are
defaults a later cut can revise without changing any shape.
## 3. Rationale
**Where the CDN shapes live: `cultnet-ts/src/content.ts`, exported as
`cultnet-ts/content`.** C1 landed this at `b8b825e1`. Both consumers already
depend on `cultnet-ts`, and its subpaths are the repo's established
browser-safe entry points (F14). The placement matches Rust, where
`cultnet-rs/src/content.rs` owns the same rules.
**Hashing is WebCrypto in one place.** `crypto.subtle` exists in browsers
(secure contexts) and in Node 20+, so one hasher serves the packer, the
provider and the reader. WebCrypto has no incremental digest, so every hash
needs its bytes in memory. That fits all three users: the reader holds the
body (the target keeps artifacts in memory), the provider holds one chunk at a
time, and the packer holds the body.
**Decoding follows the siblings.** C1's report records two places where its
decoder is stricter than MessagePack-CSharp: an exact field count, and refusal
of nil strings. Ruling `leaf-acceptance-follows-reference` may make that a
finding. That belongs to C1's Soul pass, not to C2 or C3, which take manifest
values and never decode bytes.
**No TS CultCache document definition for manifests yet.** This is still
follow-up `ts-manifest-document-definition`.
**Provider: `cultmesh-ts/content`, a request-to-answer core, one store and one
Node adapter, in one file.** `createContentChunkHandler(store, options)`
returns `(request: { method, url }) => Promise<{ status, headers, body? }>`.
The request carries no headers, because the provider honours none: Range is
ignored, and conditional requests have no answer now that ETag is cut. The
core reads only the query, never the path, which is what makes
`chunk-only-route` structural rather than checked. `toNodeListener(handler)`
is about 15 lines for `node:http(s)`, and Express, Fastify or Next mount the
core the same way. The name says "chunk" because the handler answers chunks
only. StreamPixels' sketch name, `createHttpsContentHandler` (F23), implied it
does TLS, and it does not: the host terminates TLS. One file, not three,
because the store and adapter are about 50 lines between them and have no
consumer apart from the core. `cultmesh-ts` is already a Node package, so
`node:fs` in the file costs nothing.
**Store layout: per-chunk flat files.** This is the reference layout (F21),
and it needs no index that can go stale. It coincides with StreamPixels'
whole-artifact files, because every StreamPixels artifact is a single chunk
(F23). The alternative is a store indexed from manifests, mapping a chunk hash
to (artifact file, offset, size), so that multi-chunk artifacts could be
stored whole. It is not shipped, because it adds a derived index that must be
rebuilt when manifests change, and no consumer needs it. A host that wants it
writes about 30 lines against the port, and the handler still verifies every
byte. **Consumer note for StreamPixels B3:** its `catalog.cc` and
`character-artifacts.cc` manifests must stay single-chunk, so their files are
the stored chunk. Either they are packed with `chunkSizeBytes` at least the
file size and the provider's `maxChunkBytes` is raised to match, or B3
publishes the chunk files.
**Verify on every request, not once at boot.** The invariant says "serves
only bytes whose SHA-256 equals the requested chunk hash", and a boot check
cannot see a file replaced after boot. C# verifies per request (F21). A
"verified at boot" cache would be a cache pretending to be truth. StreamPixels'
boot check is its own deploy gate (missing blob means failed health), and it
stays useful beside per-request verification. The cost is one SHA-256 of at
most 4 MiB per request, and the immutable one-year caching keeps repeats away
from the origin.
**Reader: `fetchVerifiedContent(endpoint, manifest, options)` and
`fetchVerifiedChunk(endpoint, chunkHash, sizeBytes, options)` in
`cultmesh-browser`, over one internal per-chunk primitive.** The second is the
"fetch by hash" a general consumer expects for a single-chunk artifact. It
builds no manifest: it validates the hash and size, sends
`recordKey=chunkRecordKey(hash)` so its URL equals a C#-packed manifest's, and
runs the same primitive. The whole-body check is the chunk check, because for
a single chunk the two hashes are the same. Before any request,
`fetchVerifiedContent` runs `validateManifest`, `parseSha256Hex` on every
hash, the zero-size rule (F18) and the `maxBytes` cap. It then fills a private
buffer of `sizeBytes` through a sliding window of at most
`maxConcurrentChunkRequests` requests. It uses `credentials: "omit"` and an
internal `AbortController` chained to the caller's signal, so the first
failure cancels every request still in flight. It hashes each chunk in place
and returns the buffer only after the whole-body hash matches.
**Why the reader ignores `Content-Length`.** The target says the reader
"refuses a response whose length differs from the advertised size before
reading it all". The header cannot carry that duty. Fetch decodes content
codings, and a proxy that gzips in spite of `no-transform` turns a correct
1,000-byte chunk into a 29-byte `Content-Length` (F20, R §6). In a browser the
reader cannot even see `Content-Encoding` cross-origin, because it is not
safelisted, so it cannot tell that case apart. What the invariant protects is
bounded reading and no acceptance of a wrong length. The streaming cap
delivers both from the bytes actually received: the reader stops at the first
byte past the chunk's size, and refuses a short body at its end. The target
sentence should be reworded (reported to Self as a contradiction).
**C3 does not wait for C2.** C3's live test serves from a stub `node:http`
loopback server written in the test (about 30 lines). The stub sends what the
model page says, plus the misbehaviours: gzip, a short body, an endless body,
and wrong bytes. The real provider meets the real reader in C4. This keeps
the two cuts parallel, and it lets C3 test misbehaviours the real provider can
never produce.
**C4: the proof lives in `cultmesh-browser/test/interop`, with the C# half
in `GameCult.Mesh.Tests`.** The orchestrator is Node, following the QUIC
precedent (F13): it starts the provider, runs both consumers and judges what
the provider saw. It lives in `cultmesh-browser` and not `cultmesh-ts`,
because the workspace build order compiles `cultmesh-ts` first (F31). The
browser package can import `cultmesh-ts/content` with types. The reverse
would need either an untyped script or a build-order exception. The provider
runs in its own process rather than the orchestrator's, so each of the three
parties is a separate OS process, and the orchestrator never shares an event
loop with the server it judges. The C# half drives the connector through
`CultMeshContentTransferService` (F29), because the connector alone does not
verify (F6). A parity test that stopped at the connector would show that C#
receives bytes, not that C# refuses bad ones. TLS trust is a committed
fixture certificate, as the QUIC fixture is (F25). The C# side anchors it
through `CustomRootTrust`, which keeps chain and name validation and adds no
code path that can accept a different certificate. A hash-pinning
`ServerCertificateCustomValidationCallback` would skip name validation and
add a callback that has to be read to be trusted. Corruption comes in two
kinds. On disk, the real provider refuses it with 500 (C2's
verify-before-serve). In flight, a test wrapper flips bytes after the
provider has verified them. The in-flight leg is what proves that both
consumers verify.
**"Released"** is ruled: `release-shape-gitlink` (Defaulted by Self).
## 4. Cut sequence
C1 (`cut-c1-shapes.r1`, report `h1` under a Soul pass), C2
(`cut-c2-provider.r1`) and C3 (`cut-c3-reader.r1`) are specified in the mind.
C2 and C3 run in parallel once C1 merges, each in its own worktree from C1's
merge. Both are based on `b8b825e1`, and they touch disjoint files except
`cultmesh-portability.yml`, where their hunks lie 15 lines apart. What
remains:
4. **C4 `c4-parity`** (`cut-c4-parity.r1`, branch
`hands/cultmesh-ts-content-c4-parity` from `7adbf3cb` plus the C3 merge),
waiting on C2 and C3. The spec carries the plan. In one paragraph: a
node:test orchestrator in `cultmesh-browser/test/interop` packs three
artifacts (multi-chunk, single and empty) and spawns the C2 provider as its
own `node:https` process with three mounts: honest, tampered and corrupt
store. It runs the C3 reader in a child with `NODE_EXTRA_CA_CERTS`, once
more without it as a control, and then `dotnet test` over a new
`CultMeshHttpsContentInteropTests`. That suite drives the unmodified
connector through `CultMeshContentTransferService` and trusts the committed
fixture through `CustomRootTrust`. The orchestrator then judges URL parity
and the provider's fault log. CI gains the step on both OSes, and the
browser and `cultmesh-ts` steps swap to follow the workspace order (F31).
5. **C5 `c5-release`**, waiting on C4 and the Soul passes of C2 to C4. Ruling
`release-shape-gitlink`: merged to CultLib `main` with version bumps,
CHANGELOGs and git tags, and no npm publish. Line-level plan:
- Bumps (minor; every change is additive): `cultnet-ts` 0.2.0 to 0.3.0
(`./content`), `cultmesh-ts` 0.1.1 to 0.2.0 (`./content`), and `cultmesh-browser` 0.3.0 to 0.4.0 (the reader).
`@gamecult/cultcache-ts` gains only the `checkPayloadDepth` re-export:
a patch bump, 0.14.0 to 0.14.1 (its version at `b8b825e1`).
- Dependency ranges: `cultmesh-ts` and `cultmesh-browser` move
`cultnet-ts` to `^0.3.0`. `cultnet-ts`'s `^0.14.0` on
`@gamecult/cultcache-ts` already admits 0.14.1 and stays. C4 gives
`cultmesh-browser` a devDependency `cultmesh-ts: ^0.1.1`, which does not
admit 0.2.0, so it moves to `^0.2.0` in the same commit. Otherwise npm
looks for 0.2.0 on the registry and the install fails. `npm install
--no-package-lock`, the four workspace tests and the browser's
`test:interop` run after the bumps.
- A `CHANGELOG.md` per bumped package. None exists in any package at
`b8b825e1`, so C5 creates four, each naming the live surface and not the
history (Keep a Changelog headings).
- A Soul pass over the release diff, then four tags: `cultcache-ts-v…`,
`cultnet-ts-v0.3.0`, `cultmesh-ts-v0.2.0` and `cultmesh-browser-v0.4.0`.
These go in **two pushes** (at most three tags per push, SKILL.md):
`cultcache-ts` first, because its tag triggers `publish-packages.yml`
(F16), and the three plane tags second. Whether the `cultcache-ts` tag's
npm publish job still runs, and to where, is checked before that push;
follow-up `npm-publish-ts-packages` owns the rest.
- The consumer's pickup (StreamPixels bumping `vendor/CultLib` to the
tagged commit) is StreamPixels' B-cuts, not this campaign.
- Size: about 60 lines of version and CHANGELOG text; a short Hands pass.

101
docs/cultmesh-ts-content-target.md Normal file
View file

@ -0,0 +1,101 @@
# TypeScript CultMesh Content Plane Target
Campaign `cultmesh-ts-content`. Self wrote this on 2026-10-19 for the
StreamPixels M1 live character preview. That preview is a three.js page that
loads glTF bodies and accessories by content hash from Yggdrasil.
## Why now
CultMesh moves large immutable artifacts as content-addressed chunks described
by a `gamecult.mesh.cdn_artifact_manifest`. C# packs manifests, serves chunks
over TCP (`CultMeshContentServer`), fetches them over HTTPS
(`CultMeshHttpsContentTransportConnector`), and verifies each chunk and the
whole body (`CultMeshContentTransferService`). Rust speaks the chunk messages
through `cultnet-rs` `fetch_content` and `answer_content_chunk_request`.
TypeScript has none of this: no manifest type, no packer, no provider and no
reader. And no runtime has an HTTPS content *server*; the C# HTTPS connector
has never had a peer.
Standing ruling (StreamPixels `state/map.yaml`, "Content plane", operator
2026-09-15): HTTPS content transport serves both browser and overlay.
Yggdrasil plus SHA-256 verification is the trust boundary for now.
Distributed zero-trust distribution is future work. The catalog never stores
URLs; clients resolve content through CultMesh.
## Target state
- **One TypeScript owner of the CDN document shapes.** It holds the manifest
and chunk-ref types, their MessagePack wire form, `normalizeHash`, the
record-key grammar (`mesh:cdn:chunk:<hex>` and
`mesh:cdn:artifact:<token>:<version>:<hex>`), and the packer. A manifest
that TypeScript packs is byte-identical to the one C# packs from the same
bytes and options. The owner can be imported in a browser, because the
reader needs it there.
- **An HTTPS content provider in `cultmesh-ts`.** It answers
`GET <endpoint>?chunkHash=<hex>&recordKey=<key>` with exactly the chunk's
bytes and a `Content-Length` equal to the chunk's size: the request shape
the C# connector sends (`CultMeshHttpsContentTransport.cs` `BuildChunkUri`).
Its core is framework-neutral: a request maps to an answer, and a host can
mount it on `node:https` or on any HTTP framework. It reads from a content
store port. The store shipped with it serves content-addressed files from a
directory outside git. The provider never serves a byte it has not
verified against its hash, and it never serves anything but a chunk:
there is no generic static-file route.
- **A verifying reader in `cultmesh-browser`.** Given a resolved endpoint and
a manifest, it fetches the manifest's chunks over HTTPS with the C# query
shape. It checks each response's length, each chunk's SHA-256 and the whole
body's SHA-256 with WebCrypto, and returns the bytes or throws a typed
error. No byte that failed its hash reaches the caller, in whole or in part.
It runs in browsers and in Node 20+.
- **Endpoints come from CultMesh, not from content.** The reader takes the
endpoint as an argument that the caller resolved through CultMesh. Manifests
carry no URL.
## Invariants
- `wire-parity`: manifests, chunk refs and record keys are byte-identical
across C#, Rust and TypeScript for the same input. The C# runtime is the
reference. TypeScript writes and reads the shared vectors in
`contracts/cultmesh/`.
- `https-shape-parity`: the C# `CultMeshHttpsContentTransportConnector`
fetches content from the TypeScript provider unmodified, in a cross-process
test. The TypeScript reader fetches the same chunks from the same provider.
- `verify-before-use`: the reader releases no byte that failed its chunk hash
or the whole-body hash. It refuses a body longer than the advertised size at
its first excess byte, and a shorter one at its end. It does not trust
response length headers, because Fetch decodes content codings and a proxy
can rewrite them (research 2026-10-19, section 6).
- `verify-before-serve`: the provider serves only bytes whose SHA-256 equals
the requested chunk hash.
- `chunk-only-route`: the provider answers only chunk requests. A path, a
filename or a hash outside the store can never select a file.
- `no-urls-in-content`: no manifest, chunk ref or record carries an endpoint.
- `one-owner-per-shape`: one TypeScript module owns the CDN shapes, and both
packages consume it. Neither package keeps a private copy.
- `general-purpose`: the plane is judged by what a reasonable consumer
expects, not only by StreamPixels' slice (operator, 2026-09-30). Multi-chunk
artifacts, empty artifacts, cancellation, bounded concurrency and browser
cross-origin use all work.
- `released`: the npm packages StreamPixels vendors are published at versions
that contain this plane. The release is its own cut with a Soul pass before
the tag.
## Not in this campaign
- Odin route verification for content. Single-host trust is ruled for now.
- Resumable or durable transfer state in TypeScript. The browser holds
artifacts in memory.
- Distributed or zero-trust distribution, and peer serving.
- A TypeScript TCP or RUDP content server, and an HTTPS content server in Rust
or C#.
- Garbage collection of stored blobs.
- StreamPixels' own export, publish and service wiring (its B2 and B3 cuts).
StreamPixels is the consumer, not part of this change.
## Consumer
StreamPixels `map/web-customize` (dfbe2a6),
`docs/web-character-customization-map.md` section B1. Each option's artifact is
10-125 KB, and a full avatar is 0.5-0.8 MB. The browser fetches them. The
service serves them from `/srv/streampixels/content/` on Yggdrasil, outside
git, and hashes every referenced blob at boot.

2
docs/nuget-packaging.md
View file

@ -25,5 +25,5 @@ closure through UPM rather than NuGet restore.
Install the Unity package from the immutable release tag:
```text
https://github.com/GameCult/CultLib.git?path=/unity/org.gamecult.cultlib#cultlib-unity-v1.0.57
https://github.com/GameCult/CultLib.git?path=/unity/org.gamecult.cultlib#cultlib-unity-v2.0.0
```

170
docs/research/2026-10-19-content-addressed-http-answers.md Normal file
View file

@ -0,0 +1,170 @@
# Content-addressed HTTP answers: what the specs and sources say
Fork served: **cultmesh-ts-content C2 provider HTTP answers** (session label
self-2026-10-19-cm-content). Eyes pass, 2026-10-19. Facts with evidence pointers
only; no recommendation. Line numbers refer to the raw files fetched on
2026-10-19 (master/main at that date), so they drift.
Source shorthands:
- **TG** = IPFS Trustless Gateway spec, https://specs.ipfs.tech/http-gateways/trustless-gateway/ (source: https://raw.githubusercontent.com/ipfs/specs/main/src/http-gateways/trustless-gateway.md)
- **PG** = IPFS Path Gateway spec, https://specs.ipfs.tech/http-gateways/path-gateway/ (source: https://raw.githubusercontent.com/ipfs/specs/main/src/http-gateways/path-gateway.md)
- **OCI** = OCI Distribution Spec, https://raw.githubusercontent.com/opencontainers/distribution-spec/main/spec.md
- **Fetch** = WHATWG Fetch Standard, https://fetch.spec.whatwg.org/
- **RFC 9110/9111/9112**, https://www.rfc-editor.org/rfc/rfc9110.txt (and 9111, 9112, 8246)
Two things the draft model page said "from memory" are checked at the end
(section 8).
## 1. IPFS Trustless Gateway (`application/vnd.ipld.raw`)
| Case | What the spec says | Pointer |
|---|---|---|
| Found | Body is "An opaque bytes matching the requested block CID"; "The Body hash MUST match the Multihash from the requested CID." | TG "Block Responses (application/vnd.ipld.raw)" (src l.341-346) |
| Unknown/absent hash | `404 Not Found` MUST be returned when the root block is not in the gateway's storage; applies to HEAD for any CID and GET of raw blocks. Body not specified. For non-recursive gateways it "definitively signals" the content is not in the dataset. | TG "Response Status Codes / 404 Not Found" (src l.274-283); "Recursion" (l.606-612) |
| Malformed request | TG defines no status for a malformed CID. It says servers SHOULD "validate CID format and encoding before processing requests". PG says `400 Bad Request` is a "generic client error", example "malformed CID". TG gives 400 for missing `Accept` in strict trustless mode (l.108) and for an out-of-range `entity-bytes` (l.210). | TG "Security Considerations / HTTP Servers" (src l.616-628); PG "Response Status Codes" |
| Stored object failing its digest | **Not specified for servers.** The only requirement is client-side: clients MUST verify block digest against the CID before processing or storing. TG `500` is "only for genuine server errors, not for content unavailability" (storage failure, resource exhaustion, unexpected internal errors); corruption is not named. | TG "HTTP Clients / Block Validation" (src l.629-631); "500 Internal Server Error" (l.294-296) |
| Range | TG defines no Range support for raw blocks. PG says Range is "for requesting specific byte range of UnixFS files and raw blocks"; gateways SHOULD support single-range, multi-range optional; success is `206` with `Content-Range`; `Accept-Ranges` is optional, `none` is the example for non-deterministic CAR order. | PG "Range (request header)" (src l.184-193), "Content-Range" (l.626), "Accept-Ranges" (l.632-640) |
| HEAD | `HEAD /ipfs/{cid}` is GET without payload; missing root block is 404 for HEAD too. PG: gateways SHOULD do minimal work to produce headers such as `Content-Length`. For the probe path, all HEAD headers are optional. | TG "HEAD /ipfs/{cid}" (src l.68-72), "404" (l.274-283), "Dedicated Probe Paths" (l.525-548); PG HEAD text |
| CORS | **TG defines none.** PG only says gateways serving cross-origin traffic SHOULD list `Ipfs-Uri` (and `X-Ipfs-Path`) in `Access-Control-Expose-Headers`. Kubo's config doc says its default `Gateway.HTTPHeaders` is `{}` "+ implicit CORS headers from `boxo/gateway#AddAccessControlHeaders` and ipfs/specs#423"; the values themselves are not in that file. | PG header sections; https://raw.githubusercontent.com/ipfs/kubo/master/docs/config.md l.1484-1488 |
| Cache-Control | Response value is in PG, not TG: `Cache-Control: public, max-age=29030400, immutable` "MUST be returned for every immutable resource under `/ipfs/` namespace". (Request-side `Cache-Control: only-if-cached` returns `412` when the root block is not immediately available.) | PG "Cache-Control (response header)" (src l.488-500); TG "Cache-Control: only-if-cached" (l.120-126) |
| ETag | TG: "MUST be returned and follow the recommendations in" PG. PG default is the requested CID in quotes: `Etag: "bafy…foo"`; format-specific variants add a suffix (`"bafy…foo.raw"`); non-deterministic streams use `W/`. PG: gateway MUST compare `If-None-Match` against its `Etag` (answer 304). TG: clients SHOULD NOT assume ETags are portable across gateways. | TG "Etag (response header)" (src l.324-338); PG l.130-136, l.453-486 |
| Content-Type | "MUST be returned"; for raw blocks the type is `application/vnd.ipld.raw` (IANA registered). No charset parameter is specified. | TG "Content-Type (response header)" (src l.300-308); IANA entry linked at l.343 |
| Other headers | `Content-Disposition` MUST be `attachment` with `filename="<cid>.bin"` for raw blocks, "to ensure requested bytes are not rendered by a web browser". PG: `X-Content-Type-Options: nosniff` should be returned with `application/vnd.ipld.raw` and `.car` (from the PG fetch summary; line not re-grepped). | TG "Content-Disposition" (src l.310-316); PG "X-Content-Type-Options" |
| Content-Encoding / Length | PG: `Content-Length` "Represents the length of returned HTTP payload", and warns "the value may differ from the real size of requested data if compression or chunked `Transfer-Encoding` are used", citing ipfs/specs#461 (an open TODO in the spec). | PG "Content-Length (response header)" (src l.613-624) |
| Size cap | Clients SHOULD cap block size; the spec names 2 MiB as the p2p block limit (see Block Limits). | TG "Block Limits" (src l.558-573) and "HTTP Clients" (l.629-640) |
## 2. OCI Distribution Spec, blob `GET`/`HEAD /v2/<name>/blobs/<digest>`
| Case | What the spec says | Pointer |
|---|---|---|
| Found (GET) | "MUST provide the expected blob, with a response code that MUST be `200 OK`." Response MUST contain `Docker-Content-Digest`; "If present, the value of this header MUST be a digest matching that of the response body." Clients MAY ignore it, but if used MUST verify it; "Clients SHOULD verify that the response body matches the requested digest." | OCI "Pulling blobs" (src l.180-195) |
| Unknown | `404 Not Found` MUST be returned (GET and HEAD). The endpoint table lists only `200` success and `404` failure. Error code names `BLOB_UNKNOWN` (code-1) exist in the error table; this section does not require a JSON body. | OCI l.194, l.209-216; Endpoints table l.846 (end-2); Error Codes table l.891 |
| Malformed digest | **Not specified for blob pull.** The table lists only 404. (Other endpoints: PUT blob 400 on digest problems; DELETE blob 404/400/405.) `DIGEST_INVALID` is defined as "provided digest did not match uploaded content" (upload semantics). | OCI table l.846-858; code-4 l.894 |
| Stored object failing its digest | **Not specified for servers.** Client-side verification only (above). | OCI l.188-193 |
| Range | "A registry SHOULD support the `Range` request header in accordance with RFC 9110 (section 14)." No `206`/`416`/`Accept-Ranges` text for blob pull (416 appears only in the upload flow, l.392, 421). | OCI l.196 |
| HEAD | HEAD on an existing blob MUST return `200`; MUST contain `Docker-Content-Digest` and `Content-Length` ("the size in bytes"); not found is `404`. Note: older registries may omit the digest header; clients "can reasonably assume the digest algorithm used is sha256". | OCI "Checking if content exists" (src l.198-216) |
| CORS | **Not in spec.** | whole OCI spec (no `Access-Control` occurrence found in the `grep` of the file) |
| Cache-Control/ETag | **Not in spec.** | OCI spec |
| Content-Type | **Not specified for blob GET**; `application/octet-stream` appears only for upload requests. | OCI upload sections |
| Redirects | Registries may answer blob GET with a redirect; clients must not forward `Authorization` across hosts. | OCI pull text (from the WebFetch of the same file; not re-grepped) |
Known failures near this area (NVD records, fetched 2026-10-19):
- CVE-2024-3727 (containers/image, Red Hat): "integrity-check flaw" allowing unexpected authenticated registry access, resource exhaustion, local path traversal. It affects the client library, not a registry server. https://services.nvd.nist.gov/rest/json/cves/2.0?cveId=CVE-2024-3727
- CVE-2026-85747 (distribution/distribution, fixed 3.1.2): the client's manifest `Get()` does not hash the body against the requested digest, so a malicious registry can substitute content. https://services.nvd.nist.gov/rest/json/cves/2.0?cveId=CVE-2026-85747 (status "Awaiting Analysis")
- CVE-2026-93317 (Docker build): a registry or OCI-layout blob source could supply content not matching the claimed digest and have it cached under that digest. https://services.nvd.nist.gov/rest/json/cves/2.0?cveId=CVE-2026-93317 (status "Awaiting Analysis")
- Searched for a path traversal CVE in a registry server's digest-to-path mapping: none surfaced.
## 3. Nix binary cache (HTTP)
The manual's Binary Cache page defines layout only (a `nix-cache-info` file, a
`.narinfo` per object with "a (usually relative) URL to the corresponding
compressed NAR", the NAR). It states no status codes, headers, CORS, Range or
Cache-Control. https://nix.dev/manual/nix/2.35/protocols/binary-cache/ (page head).
The behavior below is from the Nix client/writer source.
| Case | What the source shows | Pointer |
|---|---|---|
| Found | Writer stores NARs at `nar/<FileHash nix32>.nar<ext>`, i.e. named by the hash of the compressed file; narinfo records `FileHash`, `FileSize`, plus `NarHash`. | https://raw.githubusercontent.com/NixOS/nix/master/src/libstore/binary-cache-store.cc l.156-181 |
| Absent | Client maps HTTP `404` and `410` to `NotFound` ("The file is definitely not there"); `403` to `Forbidden`; `408`/`429` to `Transient`. `fileExists` issues a **HEAD** and treats `NotFound` **and `Forbidden`** as absent, "S3 buckets return 403 if a file doesn't exist and the bucket is unlistable". | https://raw.githubusercontent.com/NixOS/nix/master/src/libstore/filetransfer.cc l.826-838; .../http-binary-cache-store.cc l.184-202 |
| Malformed request | Not specified. | n/a |
| Failing digest | Client side only; the writer's `FileHash`/`NarHash` are checked by the consumer. No server answer defined. | binary-cache-store.cc l.175-181, l.313 |
| Range | Client records `acceptRanges` for retries (client-side use of Range). No server rule. | filetransfer.cc l.219 |
| HEAD | Used for existence probing (above). | http-binary-cache-store.cc l.190 |
| CORS / Cache-Control / ETag | Not in spec or the files read. | n/a |
| Content-Type | Writer uploads `text/x-nix-narinfo` (narinfo), `text/x-nix-cache-info`, and `application/x-nix-nar` (NAR). | binary-cache-store.cc l.57, l.135, l.270 |
| Content-Encoding | Writer can upload with a `Content-Encoding` header naming the compression (http-binary-cache-store.cc l.232). Client sets `CURLOPT_ACCEPT_ENCODING ""` (accept any supported) and tracks a `hasContentEncoding` flag. The NAR itself is stored already compressed (named by compressed-file hash), so transport encoding and stored compression are separate. | filetransfer.cc l.223-224, l.616 |
## 4. Bazel remote cache, HTTP
| Case | What the source/spec says | Pointer |
|---|---|---|
| Protocol | Bazel docs: "Binary data (BLOB) is uploaded via PUT requests and downloaded via GET requests"; action results under `/ac/`, files under `/cas/`; any "HTTP/1.1 server that supports PUT and GET" can serve. The doc page states **no** status codes, no digest verification on PUT, no Content-Type, no compression, no Range. HEAD is not mentioned there. | https://bazel.build/remote/caching "HTTP caching protocol" |
| bazel-remote README | CAS key is "the lowercase SHA256 hash of the entry"; `/cas/<key>` or `/<instance>/cas/<key>` (instance "optional but ignored"); "HEAD requests can be used to confirm whether a key exists"; the empty blob is "always available, even if the cache is empty" (HEAD example: `200` with `Content-Length: 0`); GET with `Accept-Encoding: zstd` "may" return zstd data; PUT of zstd requires `Content-Encoding: zstd` and `X-Digest-SizeBytes`. | https://raw.githubusercontent.com/buchgr/bazel-remote/master/README.md (HTTP section) |
| Found | `200`, `Content-Type: application/octet-stream`, `Content-Length` set to the stored size (uncompressed path). | https://raw.githubusercontent.com/buchgr/bazel-remote/master/server/http.go l.275-282 |
| Absent | `404`, body "Not found" via `http.Error`. | http.go l.268-271 |
| Malformed | Path must match `^/?(.*/)?(ac/\|cas/)([a-f0-9]{64})$` (lowercase 64-hex); otherwise `400` with the escaped URL echoed in the message (`html.EscapeString`). | http.go l.33, l.101-112, l.216-223 |
| Failing digest (read) | No digest check on the GET path in this handler (reader returned by `cache.Get`, `io.Copy` to the client). A failure after headers are sent is logged and cannot change the status ("we've already started writing data"). | http.go l.284-297 |
| Digest check (write) | PUT checks size match (`400` "sizes don't match") and `Content-Encoding` is `identity`/`zstd`/absent (else `400`); the remaining digest verification is in code not read here. | http.go l.306-345, l.372-388 |
| Compression | When zstd is chosen the handler sets `Content-Encoding: zstd` and **omits `Content-Length`** ("TODO: calculate Content-Length for compressed blobs too"). | http.go l.246-250, l.275-281 |
| HEAD | `200` with `Content-Length` of the size, or `404`. | http.go l.468-491 |
| Range / CORS / Cache-Control / ETag | Not present in the handler excerpts read. | n/a |
## 5. Git over HTTP
Smart HTTP does not address single objects. Dumb HTTP does: URL
`$GIT_URL/objects/XX/YYYY…` for a loose object.
https://git-scm.com/docs/http-protocol (introduction example).
| Case | What the spec/source says | Pointer |
|---|---|---|
| Absent | Server "MUST NOT respond with `200 OK`"; "SHOULD respond with `404 Not Found`, `410 Gone`, or any other suitable HTTP status code which does not imply the resource exists". Access denied MUST be `403`. | git http-protocol "General Request Processing" |
| Caching | Clients MAY reuse a cached entity per `Cache-Control`/`Expires` (follow RFC 2616); servers MAY return `304`; clients treat `304` as `200`. Smart services must prevent caching. The page states no Cache-Control value for object files. | same section |
| Content-Type | Specified only for `info/refs` (`text/plain; charset=utf-8` SHOULD, clients MUST NOT validate); nothing for object files. | same page |
| Failing digest | Client side: the dumb walker compares the received object's computed id with the requested id and errors `File %s has bad hash`. | https://raw.githubusercontent.com/git/git/master/http-walker.c l.540-541 |
| Missing target | The client treats a missing-target result as `HTTP_MISSING_TARGET` (http.c l.1944). | https://raw.githubusercontent.com/git/git/master/http.c |
| Content-Encoding | Client sets `CURLOPT_ENCODING ""` at http.c l.2284 (accept any encoding curl supports, curl decodes transparently) and sends `Pragma: no-cache` (l.1452). The http-protocol page says nothing about gzip for objects. | http.c |
| Range, CORS, ETag | Not in the page. | n/a |
## 6. Browser fetch facts (verifying reader)
| Fact | Evidence |
|---|---|
| CORS-safelisted response-header names are exactly `Cache-Control`, `Content-Language`, `Content-Length`, `Content-Type`, `Expires`, `Last-Modified`, `Pragma`. So `Content-Length` is readable cross-origin from a CORS response without `Access-Control-Expose-Headers`. `ETag` is **not** on the list; it needs `Access-Control-Expose-Headers` to be visible. | Fetch 2.2.2 "Headers", CORS-safelisted response-header name (https://fetch.spec.whatwg.org/#cors-safelisted-response-header-name) |
| The CORS-exposed header-name list comes from `Access-Control-Expose-Headers`; a CORS filtered response exposes safelisted names plus that list. | Fetch 2.2.6 "Responses" (https://fetch.spec.whatwg.org/#concept-response-cors-exposed-header-name-list); 3.3.3 |
| CORS check: failure if `Access-Control-Allow-Origin` is absent; "If request's credentials mode is not "include" and origin is `*`, then return success." With credentials mode `include`, `*` is not enough (needs exact origin plus `Access-Control-Allow-Credentials: true`). | Fetch 4.10 "CORS check"; 3.3.5 |
| A CORS-preflight happens for non-safelisted methods/headers (`GET`/`HEAD` with no custom headers are simple requests). | Fetch 3.3 (general); not re-quoted here |
| Fetch decodes content codings before exposing the body. In HTTP-network fetch: it records encoded size from transmitted bytes, sets `bytes` to "the result of handling content codings", then increases decoded size by the decoded length. Unsupported codings: bytes are returned as-is; decoding error: failure. | Fetch 4.7 "HTTP-network fetch"; 2.2.4 "handle content codings" (https://fetch.spec.whatwg.org/#handle-content-codings) |
| The spec itself notes: "This makes the `Content-Length` header unreliable to the extent that it was reliable to begin with." (the header describes the encoded body; the body the page reads is decoded.) | Fetch 4.7 (fetch_t.txt l.9670) |
| When a request has a `Range` header, the browser appends `Accept-Encoding: identity`, "This avoids a failure when handling content codings with a part of an encoded response." | Fetch 4.6 "HTTP-network-or-cache fetch" (l.8760-8767) |
| HTTP framing: with a valid `Content-Length`, a response is incomplete if fewer octets arrive than `Content-Length`; a client MUST record it as incomplete. `Content-Length` "refers specifically to the amount of data enclosed" (encoded form on the wire). | RFC 9112 section 8 "Handling Incomplete Messages"; RFC 9110 section 8.6 |
| `Cache-Control: no-transform`: an intermediary "MUST NOT transform the content". | RFC 9111 section 5.2.2.6 |
| `immutable` extension: identifies responses that will not be updated during freshness lifetime so clients need not revalidate. | RFC 8246 section 2 |
## 7. HTTP core facts that apply to all five systems
| Fact | Evidence |
|---|---|
| Range: "A server MAY ignore the Range header field"; a server MUST ignore Range for methods without defined range handling (GET is the only one). | RFC 9110 section 14.2 "Range" (l.6476-6484) |
| HEAD: "server SHOULD send the same header fields" as for GET; a `Content-Length` on HEAD MUST equal the GET length if sent. | RFC 9110 section 9.3.2 (l.3993), section 8.6 (l.3225) |
| `304`: a `Content-Length` in 304 MUST equal what a 200 would carry, if present. | RFC 9110 section 8.6 |
| `Content-Length` MUST NOT be sent on `204`. | RFC 9110 section 8.6 |
## 8. Cross-check against the draft model page's from-memory claims
| Draft claim (map l.44-49) | Finding |
|---|---|
| IPFS trustless gateway answers raw blocks with `Cache-Control: public, max-age=29030400, immutable` | In **PG**, not TG: the `/ipfs/` immutable value is a MUST there; TG itself does not state the response value. |
| IPFS uses the CID as `ETag` | PG: default `Etag: "<cid>"` quoted; variants add suffixes (raw: `"<cid>.raw"`). |
| IPFS returns 400 for an unparseable CID | PG lists `400` as "generic client error" with malformed-CID as the example. TG itself lists no malformed-CID rule. |
| OCI `GET`/`HEAD` returns 200 with the digest header | Confirmed (`Docker-Content-Digest` MUST on GET and HEAD; `Content-Length` MUST on HEAD). |
| OCI 404 `BLOB_UNKNOWN` | 404 confirmed; `BLOB_UNKNOWN` exists as a code name; the blob-pull text does not mandate a JSON body. |
| OCI 400 `DIGEST_INVALID` on a bad digest | **Not in the blob-pull text.** Endpoint table lists only 404 for blob GET/HEAD; `DIGEST_INVALID` means "digest did not match uploaded content". |
| OCI Range (206/416) optional | Spec says "SHOULD support" Range per RFC 9110; no 206/416/Accept-Ranges text for pull. |
| Nix `<hash>.narinfo` 200 or 404, probed with HEAD | Client source: HEAD for existence; 404 and 410 both map to NotFound; 403 also treated as absent in `fileExists`. |
| Map's own claims (open `*` CORS, 500 on corrupt, Range ignored, Cache-Control immutable) | `*` with non-`include` credentials passes the CORS check (Fetch 4.10). No system surveyed specifies a status for a corrupt stored object. Ignoring Range is permitted (RFC 9110 14.2). |
## 9. Where the systems agree and differ (matrix of what was found)
| Item | IPFS TG/PG | OCI | Nix | Bazel (bazel-remote) | Git dumb |
|---|---|---|---|---|---|
| Absent | 404 (MUST, TG) | 404 (MUST) | 404/410 -> NotFound; 403 also "absent" in HEAD probe | 404 "Not found" | 404/410/other non-200 |
| Malformed id | PG 400 generic; TG silent | silent for blob pull | silent | 400 with message | silent |
| Corrupt stored object (server) | silent; client MUST verify | silent; client SHOULD verify | silent; client verifies hashes | no read-path check in handler | silent; client errors "bad hash" |
| Range | PG: 206 supported (raw blocks); TG silent | SHOULD support | client-side retry only | not in handler | not in spec |
| HEAD | same as GET minus body; 404 | 200 + digest + Content-Length MUST | used to probe | 200 + Content-Length or 404 | not in spec |
| CORS | none in TG/PG (Kubo adds implicit headers) | none | none | none | none |
| Cache-Control on content | PG: `public, max-age=29030400, immutable` MUST | none | none | none | defers to RFC 2616 |
| ETag | PG: quoted CID | none | none | none | 304 allowed |
| Content-Type | `application/vnd.ipld.raw` | unspecified (pull) | `application/x-nix-nar` etc. | `application/octet-stream` | unspecified (objects) |
| Content-Length vs encoding | PG warns they may differ | n/a | transport `Content-Encoding` separate from stored compression | zstd path omits Content-Length | client auto-decodes |
| nosniff / attachment | TG: Content-Disposition attachment MUST; PG: nosniff | none | none | none | none |
## 10. Limits of this pass
- Fetch spec 4.7 was read from the HTML text; the exact algorithm line that fails the fetch on a body shorter than Content-Length was not found in the Fetch text (framing is RFC 9112 section 8, which says the client records the message incomplete).
- bazel-remote digest verification on PUT was not read to the end; the read-path (GET) has no digest check in the handler.
- Kubo's actual default CORS header values were not retrieved (only the pointer to boxo and ipfs/specs#423).
- git CVEs were not searched; Nix and Bazel CVEs were not searched.
- OCI "CORS: not in spec" is from a search of the spec file, not from a registry implementation (distribution/distribution's own headers were not read).

103
docs/research/device-capture-runners.md Normal file
View file

@ -0,0 +1,103 @@
# Device capture runners: how established projects capture real-GPU results, and what a browser driver can silently get wrong
For `cultmath-tapes:question:device-run-driver` (campaign `cultmath-tapes`) and follow-up `cultmath-tapes:follow_up:glsl-browser-parity-run`.
Gathered by eyes-ag3-device-driver, session self-2026-10-15-ag3, 2026-10-16. Facts only; no recommendation.
Strength marks: [src] read from a primary source text this pass; [doc] vendor or standards documentation read this pass; [web] search-result or fetch-summary of a page (a small model summarised it; check before quoting); [abs] searched, not found.
Chromium doc pages were read at `main` or at an old pinned revision as noted; Chromium GPU bot details drift, so the dates matter.
## 0. What the question asks (the record)
- [src] Question `device-run-driver` (Imagination, 2026-10-16): a static page (`packages/cultmath/tools/device-runner`) compiles each family's call against `CultMath.glsl`, reads back through an RGBA32UI target, emits raw bits; C# judges. Options: `committed-captures` (page + stdlib Python server, any WebGL2 browser is a driver, captures committed with renderer, revision, shader sha, args hash, result bits, judged by `DeviceCaptureTests` on Linux), `playwright-live` (Microsoft.Playwright test dependency, Windows-only, ANGLE D3D11, must refuse SwiftShader), `both`.
- [src] Prior precedent named in the question: three Soul/Hands passes through the desktop app's built-in browser on Starfire, renderer `ANGLE (NVIDIA GeForce GTX 1070 Direct3D11)`.
- [src] Follow-up `glsl-browser-parity-run` records both GLSL files probed on a GTX 1070 (phacelle within 3.1e-6 of max(|v|,1), frustum check 0/256).
## 1. How established projects capture real-GPU results for CI
| Project | What runs where | What is stored, and where | What identifies the device | Source |
|---|---|---|---|---|
| Khronos WebGL conformance suite | Static HTML pages. Open `webgl-conformance-tests.html` in the browser under test (`?run=1` auto-starts), then "display text summary". Run by hand in any browser. | Formal results go as a pull request to the `KhronosGroup/WebGLConformanceSubmissions` repo. Older README copies said email `webgl_conformance_submissions@khronos.org`. Only WebGL implementors submit. | README instruction: "Verify that the User Agent and WebGL renderer strings identify your browser and target correctly." The summary text carries them. | [src] raw.githubusercontent.com/KhronosGroup/WebGL/main/sdk/tests/README.md (read 2026-10-16); [web] search summary of CONFORMANCE_RULES.txt |
| Khronos WebGL conformance rules | Results carry forward: "results submitted for an earlier version of the browser carry forward to later versions ... that do not cause any previously passing test to fail"; for OS versions likewise (a new OS version needs a new submission). | The carry-forward rule is the only staleness rule found. | | [web] search summary of CONFORMANCE_RULES.txt, not read in full |
| Vulkan CTS conformance submission | Run on the vendor's driver builds, every fraction. | Submission package must contain full `TestResults.qpa` logs per driver build and fraction, `git status` and `git log --first-parent <release tag>^..HEAD` output of a clean CTS checkout, `git format-patch` output of any CTS changes over the release tag, and a statement file with a `CONFORM_VERSION` line carrying the CTS git tag, plus PRODUCT, CPU, OS. `verify_submission.py` checks the package against the current mustpass list. | PRODUCT/CPU/OS lines in the statement. | [web] search summaries of `external/vulkancts/README.md` in Fuchsia/Android/Chromium mirrors (mirror copies differ by version) |
| Chromium GPU bots | Physical machines in Swarming; most tests via Telemetry launching the full browser; `webgl_conformance_tests` and pixel tests. `chromium.gpu.fyi` tests top-of-tree ANGLE. | Expected outcomes in per-test expectation files (`pixel_expectations.txt` under `content/test/gpu/gpu_tests/test_expectations/`, `Failure` entries). Pixel images are not in the repo (Skia Gold, below). Some pixel tests check only a few pixels with handwritten code so one check serves all vendors. | Swarming task dimensions: `gpu` = PCI vendor:device-driver version (example `10de:2184-440.100`), plus `os`, `cpu`, `pool`. The page does not describe a runtime renderer-string check. | [src] chromium.googlesource.com/chromium/src/+/main/docs/gpu/gpu_testing.md (read 2026-10-16) |
| Skia Gold (Chromium pixel tests) | goldctl client sends the produced image, the test name and hardware/software config metadata to a service. | Approved images live in a service (storage bucket + triage UI), not git. A test passes if the hash of the produced image is in that test's approved-hash list; default exact matching, multiple baselines per test. An unknown hash fails until a human triages it. Doc says local runs "differ slightly from bot images" and uploads need allow-listing; `--local-pixel-tests` disables upload. | Config metadata uploaded with each image. | [web] search summary of `docs/gpu/gpu_pixel_testing_with_gold.md` (pinned old revisions); [src] gpu_testing.md for the local-run flags |
| WebGPU CTS in Chromium | gpuweb/cts compiled to JS, run through WPT and Telemetry; Telemetry covers non-virtual devices with real GPUs. A pinned revision is auto-rolled. | Expectations per platform: Fail/Skip may be any CTS query; Slow/Crash/Timeout must list parameters in loader order. | | [web] search summaries of Chromium `web_tests/webgpu/README.md` and a 2022 W3C public-gpu post by Kai Ninomiya (not read in full) |
| Dawn node runner | `dawn.node` binding runs the CTS without a browser; `run-cts` can instead drive a Chrome instance. | `--output <path>` writes a results file, `--expect <path>` diffs a later run against it. Dawn keeps `webgpu-cts/expectations.txt`, pruned when tests start passing, with wide `Failure` wildcards for flaky ones. | | [web] search summaries of Dawn `src/dawn/node/README.md` and commit `f78edf56` ("Expand CTS expectations for Win11 QC CI bot") |
| three.js | `test/e2e/puppeteer.js` screenshots almost all examples and compares to stored references; helpers `deterministic-injection.js`, `clean-page.js`, `check-coverage.js` exist in an older release's `test/e2e`. | Reference screenshots in the repo. | A 2020 issue: screenshots were black on macOS with the GPU-path flags; a maintainer floated using SwiftShader everywhere "to avoid GPU-specific bugs". Current flags and CI setup not confirmed. | [web] search summaries of discourse.threejs.org and a lightrun mirror of the GitHub issue; [abs] current `puppeteer.js` flags not read |
| Babylon.js / Babylon Native | Visualization tests list scenes in `tests/test/visualization/config.json` (Babylon Native: `Apps\ValidationTests\Scripts\config.json`); each entry names a reference image, optional per-scene pixel-difference threshold. | Reference images in `ReferenceImages/` in the repo; `-u` regenerates. Babylon Native CI fails on too-different images and stores rendered/error images as an artifact. | | [web] search summaries of forum.babylonjs.com and a Babylon Native `ContinuousIntegration.md` fork; [abs] an `errorRatio` key was not found |
| glslang / SPIRV-Cross | No GPU. SPIRV-Cross round-trips `shaders/` through glslangValidator/spirv-as and SPIRV-Cross, diffs output text against committed `reference/` files; `test_shaders.py` exits non-zero on difference, `--update` regenerates references that then ride in the PR; says to use the right glslang and SPIRV-Tools versions. Only hardware option: `--malisc` static cycle counts. | Reference text in the repo. | | [src] raw.githubusercontent.com/KhronosGroup/SPIRV-Cross/main/README.md (read 2026-10-16); [abs] glslang's own test practice not read |
| dEQP GLES3 builtin precision tests | Random inputs through built-ins in a shader; outputs via transform feedback or framebuffer pixel; compared to a reference interval, not a single value. | Spec text of tolerances (section 4). | | [src] doc/testspecs/GLES3/functional.shaders.builtin_functions.precision.txt (Fuchsia mirror, read 2026-10-16) |
Observations across the rows (no ranking):
- [src/web] Where real-GPU outputs are exact or large, the approved results are stored outside a source repo (Gold, Swarming expectations) or as submission packages (WebGL, Vulkan CTS). Where outputs are text from a deterministic tool (SPIRV-Cross) the reference is committed beside the source and `--update` regenerates it.
- [web] Only the Vulkan CTS submission package was found to record the exact source revision (tag plus `git log`) inside the result bundle; WebGL's rule is the carry-forward rule above. No project was found that refuses a committed capture automatically because a shader source hash changed. [abs]
- [src] Chromium's GPU doc keys a bot's hardware by PCI id and driver version rather than by a renderer string read inside the page.
## 2. Headless browser drivers on real GPUs
### 2.1 Fallback to a software renderer
- [src] Chromium SwiftShader doc (read at `refs/heads/main`): flags `--use-gl=angle --use-angle=swiftshader` (SwANGLE as the ES driver); `--use-gl=angle --use-angle=swiftshader-webgl --enable-unsafe-swiftshader` (WebGL fallback). "Automatic fallback to SwiftShader-backed WebGL is deprecated ... WebGL context creation will soon fail instead of falling back." `--enable-unsafe-swiftshader` opts in and suppresses the console warning. The page does not describe how to verify the renderer.
- [web] Blink-dev "Intent to Remove: SwiftShader Fallback": deprecation period began Chrome 130; estimated desktop ship Chrome 137; a later reply from Rick Byers says API owners consider it high-risk and may phase it, so the milestone was unconfirmed in the thread; Finch feature `AllowSwiftShaderFallback`; headless SwiftShader use "will still be supported by opting in"; desktop only. Sources disagree on the shipped milestone (a Winaero Chrome 137 write-up says WARP replaces it on Windows, which the blink-dev page does not mention; Samsung's browser policy page says M139). Not resolved. [web]
- [src] Chrome developer blog "Supercharge Web AI model testing": "By default, Headless Chrome disables GPU." `chrome://gpu` showed "Software only, hardware acceleration unavailable" until drivers were fixed; "Some combinations of flags allow detection of the GPU, but don't allow you to actually use the GPU." Failing renderer string: `ANGLE (Google, Vulkan 1.3.0 (SwiftShader Device (Subzero) (0x0000C0DE)), SwiftShader driver-5.0.0)`; working: `ANGLE (NVIDIA Corporation, Tesla T4/PCIe/SSE2, OpenGL ES 3.2 NVIDIA 525.105.17)`. Linux flags there: `--headless=new --use-angle=vulkan --enable-features=Vulkan --disable-vulkan-surface --enable-unsafe-webgpu`. Setup was Colab, not CI.
- [web] A Playwright-on-GPU-runner write-up (createit.com) reported renderer ANGLE-on-Vulkan with SwiftShader; headed Chromium under a virtual display used the GPU; a Vulkan flag set later gave `llvmpipe`, still software.
- [src] Chromium GPU testing doc: example bot flags `--use-gl=angle --use-angle=gl --use-cmd-decoder=passthrough --force_high_performance_gpu`, passed via `--extra-browser-args`.
- [src] Playwright `chromiumSwitches.ts` at `main` (read 2026-10-16) contains no GPU, SwiftShader, ANGLE or headless switch. [abs] What the downloaded headless shell binary itself defaults to was not found.
- [abs] No source was found on `--use-angle=d3d11` combined with headless Playwright on Windows, and none on the renderer string a Windows GPU reports under headless.
- [web] `--disable-gpu` is reported by one source to force software rendering.
### 2.2 Detecting a fallback from inside the page
- [doc] `WEBGL_debug_renderer_info` exposes `UNMASKED_RENDERER_WEBGL` through `gl.getExtension(...)` then `gl.getParameter(...)`; MDN says it may be limited to privileged contexts depending on privacy settings, and Firefox disables it when `privacy.resistFingerprinting` is true, so `getExtension` can return null (MDN, via search result).
- [doc] `failIfMajorPerformanceCaveat: true` makes `getContext` return null if the browser would use a dramatically slow implementation (MDN, via search result). [web, unconfirmed here] whether Chrome treats SwiftShader as such a caveat was not found in a source this pass.
- [src] Strings observed in sources: SwiftShader `ANGLE (Google, Vulkan 1.3.0 (SwiftShader Device (Subzero) ...))` and `Google SwiftShader`; hardware `ANGLE (NVIDIA ..., Direct3D11)` form as in the question's precedent.
- [src] Chromium's own bots do not read a string; they identify hardware by Swarming dimensions (section 1).
- [web] `WebGLShaderPrecisionFormat` from `getShaderPrecisionFormat(FRAGMENT_SHADER, HIGH_FLOAT)`: `precision` 23 means real highp float; it reports mantissa bits, nothing about subnormal handling; Safari mis-reported it as of April 2020 (WebKit bug via webglfundamentals).
- [web] A sample WebGL report line: `ANGLE (Intel(R) HD Graphics 4600 Direct3D11 vs_5_0 ps_5_0)`, highp/highp, precision 23.
### 2.3 Windows service and session 0
- [web] Microsoft XPDM-vs-WDDM table: on WDDM, Session 0 isolation means a Direct3D 9 HAL device is never available from a service (NULLREF, HAL, REF "No"; WARP10 "Yes").
- [web] Microsoft support article 978635: starting with Windows 8.1 / Server 2012 R2, Direct3D 11 supports use from a Windows service. WARP is recommended by Microsoft for service/server environments.
- [web] VTK issue 18657 (GitLab): swap-chain creation failed on Windows CI runners because the process ran in Session 0, which has no graphics hardware access; runners were moved from system services to scheduled tasks.
- [abs] No source found on headless Chrome or ANGLE behaviour specifically inside a service. The pattern the sources support is an interactive user session or a scheduled task that runs only when the user is logged on.
## 3. Committed-capture designs: versioning against source hashes and refusing stale captures
- [src] SPIRV-Cross: references are text regenerated by the tool; staleness is detected by re-running the tool and diffing, not by a stored hash. The tool versions (glslang, SPIRV-Tools) are named in the README as the ones to use.
- [web] Skia Gold: the approved set is keyed by test name plus image hash; an unknown hash fails until triaged. Config metadata is uploaded with the image. No source-revision binding was found.
- [web] Vulkan CTS: the result package carries the CTS tag, `git log` and patches; a checker script validates the package against the mustpass list for the version.
- [web] WebGL: carry-forward rule (earlier passing results hold for later browser versions that fail no previously passing test).
- [web] Babylon / three.js: reference image names live in a config list; regeneration is a flag (`-u`); a missing reference is an error (a 404 in a Karma run, Babylon forum).
- [src] Chromium expectation files: per-test `Failure` entries are removed once a test passes (gpu_testing.md); Dawn prunes `expectations.txt` the same way.
- [abs] No project was found whose committed device capture embeds a hash of the shader source it ran and is rejected by CI when that hash no longer matches the checked-in source. (Search was for the pattern, not an exhaustive survey of repositories.)
## 4. Float behaviour that bears on judging captures across ANGLE/D3D11, native GL and mobile GPUs
### 4.1 What the standards guarantee
- [doc] Qualifiers are minimums. highp float "is at least a 32 bit value", range about +-2^62 in the WebGL2Fundamentals table (exponents stripped in extraction, inferred); mediump at least 16 bit, range +-2^14, relative precision 2^-10; lowp at least 9 bit, range +-2 (webgl2fundamentals precision lesson; asawicki.info April 2015 for the 2^-10 and 2^-8 figures). On a desktop GPU mediump "will still most likely use 32bits internally"; the lesson's author's Pixel 2 XL used 16 bit for mediump and lowp.
- [doc] Highp in fragment shaders is optional in OpenGL ES 2.0 (a shader declaring it fails to compile where unsupported). [web] GL_ARB_shader_precision text: internal processing need not match IEEE 754; rounding mode cannot be set and is undefined; "any denormalized value input into a shader or potentially generated by an operation in a shader can be flushed to 0."
- [src] dEQP GLES3 precision spec tolerances (ULP unless noted): `+ - *` under 1 ULP; `/` 2.5 ULP for 2^-126 <= |y| <= 2^127-1; sin/cos 2^-11 abs on [-pi, pi]; asin/acos 4; atan2 6; atan 5; exp 3+2|x| highp; inversesqrt 2; log 3 ULP highp outside the central range; exact (0 ULP) abs, sign, floor, trunc, roundEven, ceil, min, max, clamp, step; derived functions (tan, pow, sqrt, mix, smoothstep, fract, mod ...) judged by their expansions. The page does not mention flush to zero. Lowp and mediump results on overflow "must produce an infinity or the max/min value".
- [web] Olli Etuaho on the WebGL list, 14 Nov 2014: proposes testing only at the spec minimums, with subnormals flushed to zero, since code that works there will likely work with more bits; concedes not simulating built-in internals is a gap.
### 4.2 Direct3D 11 (the ANGLE/D3D11 path)
- [doc] Microsoft Learn "Floating-point rules (Direct3D 11)" (updated 2025-04-15), 32-bit rules: "Denorms are flushed to sign-preserved zero on input and output of any floating-point mathematical operation" (exceptions: I/O or data movement that does not manipulate the data). Add/sub/mul within 0.5 ULP, and the page says hardware may truncate; sqrt and rcp 1 ULP; rcp and rsq have a separate relaxed requirement; "Fused operations (such as mad, dp3) produce results that are no less accurate than the worst possible serial ordering of evaluation of the unfused expansion", each unfused step allowed 1 ULP (D3D10 page: 1 ULP for mul/div). NaN bit patterns are not preserved. min/max flush denorms for comparison but the result "may or may not be denorm flushed". Doubles preserve denorms; 16-bit floats preserve denorms.
- [abs] No source found that states how ANGLE's D3D11 translation, fxc/dxc flags, or NVIDIA's D3D11 driver treat denormals or contraction beyond the D3D rules above.
- [web] ANGLE's `FeaturesD3D.h` workaround list contains HLSL-compiler-specific rewrites (for example expanding `pow` into multiplies on an affected compiler, per search summary); a Wine D3D11 trig test failed on one NVIDIA configuration with roughly 0.1% relative error, not on AMD (Wine bug thread, cause not identified).
- [web] ANGLE D3D11 vs D3D9 rendering differences are reported in Mozilla bug 1122912 (unresolved in the excerpt).
### 4.3 Mobile and other native GPUs
- [web] ARM blog "Benchmarking floating point precision in mobile GPUs, Part III": Vivante GC4000 and Imagination SGX 554 "do not support subnormal values" (below 2^-126 treated as zero); Mali-T604 supports subnormals, and the authors say Mali Midgard is the only mobile family they know to; the measuring shader divides by 2 per exponent step and multiplies back, drawing underflow pixels red. No ULP measurements in that part; Part II (not read) is said to find GPUs taking "the easy (round-to-zero) way out".
- [web] asawicki.info: `length`, `normalize`, `distance` and dot products in a `mediump` shader can overflow at (74,74,74) because 3*74^2 = 16428 exceeds 2^14; a GPU may pass because it uses more precision than required.
- [abs] No source read this pass on Adreno, Apple, current Mali or PowerVR denormal and contraction behaviour; the ARM post covers 2013-era parts.
- [abs] FMA contraction in GLSL: no WebGL or GLSL ES source was read here on whether `a*b+c` may be fused (see `docs/research/cycles-fma-contraction.md` for the Cycles CPU case and the D3D "mad" fused-op tolerance above).
## 5. Gaps left by this pass
- Current three.js `test/e2e/puppeteer.js` flags and Babylon `config.json` tolerance keys were not read from the repositories (search summaries only).
- Gold, WebGPU CTS and Dawn node pages are search summaries, not full reads.
- Whether Chrome's shipped (not 130-era) behaviour fails WebGL context creation when no GPU is usable, versus using WARP on Windows, is unresolved between the sources in section 2.1.
- Nothing here was measured on Starfire; no run of any driver was made.

49
docs/research/element-id-header-flags.md Normal file
View file

@ -0,0 +1,49 @@
# Prior art: a header flag one writer sets and content another writer cannot see
Serves: `variants:question:non-csharp-writers-mark-ids` (raised in
`variants:follow_up:foreign-ids-under-v1-header`, campaign `variants`). Gathered by
Imagination, session self-2026-10-14b-eureka, 2026-10-14. Facts with pointers; the
recommendation lives in the question, not here.
The CultLib case: the store header (`cultcache.store.v1` / `v3`) says whether any record
holds an element id, so a reader older than ids refuses the store instead of rewriting
elements without them. An element id is an ordinary keyed string in the payload, so a
runtime that does not know which member is the id (cultcache-py, -ts, -rs today) cannot
tell from content whether to mark. The persistence contract
(`src/GameCult.Caching/Contracts/cultcache-persistence-format.md:424-427` at 6afa8ed9)
says those runtimes keep a `v3` header they found and do not decide.
## How established formats split the job
| System | Who sets the flag | What an implementation that does not understand the feature does | Pointer |
|---|---|---|---|
| ext4 superblock | The code that first uses a feature sets its bit in `s_feature_compat`, `_ro_compat` or `_incompat`. | Unknown `incompat` bit: refuse to mount (fsck stops too). Unknown `ro_compat` bit: mount read-only, never write. Unknown `compat` bit: read and write freely; fsck should not modify. | https://www.kernel.org/doc/html/latest/filesystems/ext4/super.html (feature fields) |
| SQLite file header | The writer's journal mode sets the write/read version bytes (offsets 18, 19); the schema format number (offset 44) is set by the writer of the newer schema features. | Read version 1-2 with write version > 2: open read-only. Read version > 2: neither read nor write. | https://www.sqlite.org/fileformat.html section 1.3.3, 1.3.10 |
| git repository format | A repository declares `core.repositoryformatversion = 1` and the `extensions.*` it uses; the tool enabling a feature writes its extension key. | Any unimplemented `extensions.*` key in a v1 repo: the operation must not proceed (read or write). | https://git-scm.com/docs/repository-version |
| Protocol Buffers | No flag. New fields are self-delimited by tag. | Parsers keep unknown fields and write them back (proto2 always; proto3 again since 3.5), so an old reader does not drop a newer writer's data on rewrite. | https://protobuf.dev/programming-guides/proto3/#unknowns |
| Avro container files | The writer embeds its full schema in the file header. | Reader resolves writer schema against its own; a writer field the reader lacks is ignored, so it is lost if the reader rewrites. | https://avro.apache.org/docs/current/specification/ ("Schema Resolution") |
| Parquet | Writer stamps `created_by` (application and version) in FileMetaData. | Readers keep a list of known-bad writer versions and distrust their metadata (parquet-mr ignores min/max statistics from writers before 1.8.0, PARQUET-251). | https://github.com/apache/parquet-format/blob/master/src/main/thrift/parquet.thrift (`created_by`); https://issues.apache.org/jira/browse/PARQUET-251 |
## Patterns, as facts
- Flag-based formats (ext4, SQLite, git) put the duty to set the flag on the writer that
introduces the content. None of them has a writer that carries feature content while
not knowing it is that feature: an implementation that does not understand a feature
either does not write (ro_compat, SQLite read-only) or does not touch the store (git).
- Formats without a flag (protobuf, Avro) handle the older reader by preservation
(protobuf keeps unknown fields) or accept loss (Avro drops unknown writer fields on
rewrite). CultCache's C# 1.x readers skip unknown keyed slots and do not preserve
them, which is why the `v3` header exists.
- Parquet's `created_by` is a third option: readers judge content by who wrote it and
compensate for writers known to get a flag or statistic wrong.
- No system surveyed asks a reader to infer a feature flag from content it cannot
recognise; where content cannot be recognised, the burden is on writers.
## Mechanism facts from this pass (probes at 6afa8ed9 on Yggdrasil)
- A C# cache that pulls the Python-written `py-store/leaf-record.cc` (v1, element ids
`000000000021/22`) holds that record with `HoldsIds = true`, `IdsInMemoryOnly = false`;
a C# write that adds an unrelated record copies it forward and leaves the header `v1`.
The ids survive in C#; a pre-id reader with a type lacking the id slot would drop them.
- cultcache-py, -ts and -rs read `v3` and keep it on rewrite; none has a declaration of
which member is an element id.

115
docs/research/ground-motion-law.md Normal file
View file

@ -0,0 +1,115 @@
# Ground motion law: prior art (facts only)
Serves fork: cultmath-tapes, site-ground motion law (invariant `one-motion-owner`, target `cultmath-tapes:target:r2`), blocks `cut-site-ground`.
Gathered by Eyes, session self-2026-10-14b-eureka, 2026-10-14. No recommendation. Committed by Self before the fork was put.
Method: primary PDFs were downloaded to the session scratchpad and read with pdftotext ("read"). "[secondary]" marks facts taken from a web-search summary, forum or blog, not read at the primary source. "[not verified]" marks things looked for and not reached.
## 1. How shipped volumetric clouds move, and what breaks
### 1.1 Horizon Zero Dawn, Schneider and Vos, SIGGRAPH 2015
Source: https://www.advances.realtimerendering.com/s2015/The%20Real-time%20Volumetric%20Cloudscapes%20of%20Horizon%20-%20Zero%20Dawn%20-%20ARTR.pdf (99 slides, read)
- Density = low-frequency 3D Perlin-Worley base shape times height gradient and coverage, eroded by a high-frequency 3D Worley texture (slides ~34-37).
- A 128^2, 3-channel 2D curl noise texture "is non divergent and is used to fake fluid motion"; it distorts the cloud shapes and distorts the sample position of the second (detail) 3D noise "to fake the swirly distortions from atmospheric turbulence" (slides 33, 37). The curl texture is a static distortion of the detail lookup, not a time-integrated flow.
- Summary slide (48): noises are custom, "made from Perlin, Worley and Curl noise", and "all animated in a given wind direction". Wind direction varies over altitude (slide 21). High cirrus/alto layers are 2D textures "tiling and scrolling at different speeds and directions" (slide 87).
- Large-scale evolution is not noise motion: coverage, precipitation and cloud type come from a 2D weather map driven by a simulation, with a wind-speed control (slides 39-44).
- Reprojection (slides 92-93): quarter-res buffer updates 1 of 16 pixels per 4x4 block each frame; "We reproject the previous frame to ensure we have something persistent"; where reprojection fails (screen edges) the low-res result is substituted. Stated as making the shader "10x faster or more"; raw cost ~20 ms, target ~2 ms.
### 1.2 Nubis, Evolved, Schneider, SIGGRAPH 2022
Source: https://advances.realtimerendering.com/s2022/SIGGRAPH2022-Advances-NubisEvolved-NoVideos.pdf (read)
- Slide "Skies / Vertical Profile Model / Animating Density": the motion law for sky density is a sample-position offset, `noise_sample_position = sample_position - wind_direction * scroll_offset` (comment "Deform sample coordinates").
- Slide family "VFX / Animating Superstorm Density / The Mesocyclone": each ring samples noise at `GetRotatedPosition(sample_position, superstorm_center, time_offset * RingRotationSpeed[n])`, later with a per-ring `ring_skew[n]`: different rotation rates per layer about one center, no flow field.
- Slide "VFX / Animating Cloud Density For Skies" (code listing): the reprojection pass reads a stored per-pixel `cloud_distance` from a "CloudAttrWorkingBuffer", reconstructs `cloud_world_space = invView * (eyeRay * cloud_distance)`, applies the SAME motion law to that world position (rotate about the storm center blended by `superstorm_mask`; scroll `cloud_world_space + scroll_direction_2D * deltaTime`), then projects it with `inReprojectionMatrix` to get the previous-frame UV. The history lookup is displaced by the density's own motion, using one depth per pixel written by the march.
- The 2022 and 2023 decks show an "Aero Cloud Solver" frame (Apply Buoyant Force, Advect Density, Advect Velocity, Non-Divergence, Sourcing) among the history slides, as the offline VFX approach.
### 1.3 Nubis3 (Nubis Cubed), Schneider, SIGGRAPH 2023
Source: https://advances.realtimerendering.com/s2023/Nubis%20Cubed%20(Advances%202023).pdf (read)
- Voxel clouds, slide "Sampling Density": `inSamplePosition -= float3(cCloudWindOffset.x, cCloudWindOffset.y, 0.0) * voxel_cloud_animation_speed`. Wind offset only; no curl, no flow field.
- A comparison table lists "Evolution: Yes" for the Vertical Profile method and "Pseudomotion Only" for the Envelope method, and "Pseudomotion Only" in later tables for the voxel methods.
- March is SDF-accelerated: "Hybrid SDF, Adaptive and Jittered samples"; jitter is `animated_hash` within 250 m of camera, `static_hash` beyond.
- No temporal reprojection slide found in the extracted text.
### 1.4 Frostbite, Hillaire, SIGGRAPH 2016
Source: https://media.contentapi.ea.com/content/dam/eacom/frostbite/files/s2016_pbs_frostbite_sky_clouds.pdf (read; section 5.5.3 and the cloud intro)
- 5.5.3: clouds rendered in a single pass; each frame samples are randomly offset within their step range; current frame blended with previous "according to a constant blend factor" (EMA); "For the final frame to not look blurry when the camera is moving/rotating very fast, we re-project the previous result according to previous and current camera properties". Camera-only reprojection; no density-motion term stated.
- Text cites Guerette's visual-flow technique to give a panoramic cloud texture the illusion of motion along the wind direction, noting it has no variation of cloud shape, weather or lighting.
- Frostbite volumetric fog temporal blend, as reproduced by Flax Engine: Halton jitter, 7% current / rest history (https://flaxengine.com/?p=941) [secondary]. Godot docs: higher blend "makes ghosting much worse", moving fog volumes and lights leave a trail (https://docs.godotengine.org/en/stable/tutorials/3d/volumetric_fog.html) [secondary].
- Hillaire 2015 (Frostbite unified volumetrics) temporal section: [not verified] (158 MB PPTX; SlideShare copy cuts off before it).
### 1.5 Toft, Bowles, Zimmermann (Studio Gobo), "Optimisations for Real-Time Volumetric Cloudscapes", arXiv 2016
Source: https://arxiv.org/abs/1609.05344 (PDF read)
- Calls the Horizon 1/16-pixel scheme "a fairly complex technique which may introduce visual artifacts as a result of the 4x4 pattern by which pixels are updated".
- Alternative: far fewer march steps + random per-pixel start offset + analytic integration per step + Playdead-style TAA (Pedersen, INSIDE, 2016) on a low-res buffer. Timings, 1080p GTX 1080 (Fig. 3): half res 128 steps 128.0 ms; half res 8 steps jitter+TAA 7.5 ms; quarter res 8 steps jitter+TAA 2.4 ms.
### 1.6 UE Volumetric Clouds
- Official component docs list View/Reflection/Shadow sample-count scales and a transmittance stop threshold; nothing on temporal behavior: https://dev.epicgames.com/documentation/unreal-engine/volumetric-cloud-component-properties-in-unreal-engine [secondary].
- Forum: Mode 2 traces 1 pixel per 4x4 block and reprojects 15, "lots of latency", cloud "would leak after reprojection over opaque meshes": https://forums.unrealengine.com/t/volumetriccloud/688516 [secondary].
- Forum hypothesis (unconfirmed): a VolumetricCloud writes neither depth nor velocity, so TSR has nothing to reject history on, causing trailing and banding inside a cloud: https://forums.unrealengine.com/t/tsr-ghosting-against-a-volumetriccloud-the-player-stands-inside-every-rejection-knob-measured-none-help/2837437 [secondary].
- Bug UE-198490 (fixed 5.4): horizontal flicker lines from the temporal component; workaround `r.VolumetricRenderTarget.Mode=1`: https://issues.unrealengine.com/issue/UE-198490 [secondary].
### 1.7 Red Dead Redemption 2 clouds (Bauer, SIGGRAPH 2019)
Talk exists ("Creating the Atmospheric World of Red Dead Redemption 2: A Complete and Integrated Solution", Advances in Real-Time Rendering in Games 2019). Slides [not verified]: not among the PDF/PPTX links on https://www.advances.realtimerendering.com/s2019/index.htm; a guessed URL returned 404. Nothing about its motion or reprojection was read.
### 1.8 Shadertoy-scale nebulae
[not verified]. No Shadertoy source was read in this pass.
### 1.9 What the sources say breaks
- Pulsing and repetition: Vlachos names these as the two problems of two-phase flow-map animation (3.1).
- Stretching: a texture advected by a non-uniform flow stretches without bound unless regenerated (Neyret, 3.3).
- 4x4 update-pattern artifacts (Toft et al., 1.5); latency and leaking over opaque geometry in 1/16 modes (UE forum, 1.6).
- Camera-only reprojection fails when content moves: https://www.wayline.io/blog/altos-1-1-better-temporal-reprojection-for-volumetric-clouds-in-unity ("When the player moves their camera, the history becomes useless") [secondary]. A hobby engine skips a velocity buffer "since the clouds move slowly", uses bilinear history sampling and EMA: https://handmade.network/p/75/monter/blog/p/7201-engine_optimization__fast_high-quality_cloudscape_through_noise_and_reprojection [secondary].
## 2. Temporal reprojection of raymarched volumes whose density moves
- A volume has no single surface depth. Nubis 2022 writes a per-pixel `cloud_distance` during the march, rebuilds a world point from it, applies the density motion law to that point, then projects with the previous view-projection (1.2). This is the only shipped density-motion-aware reprojection found at a primary source.
- Horizon 2015 and Frostbite 2016 reproject with camera properties only (1.1, 1.4).
- AtmoPlugin (hobby project, GitHub PR #15): raymarch writes a "contribution-weighted" reprojection depth into alpha, taken where opacity exceeds 5%, with fallbacks surface depth, scene depth, mid-atmosphere, infinity; temporal pass reprojects the pixel center with previous camera and planet transforms: https://github.com/Upa2sdo2l/AtmoPlugin/pull/15 [secondary].
- Two hybrid depth-of-field papers reproject semi-transparent regions by average world position of far-field hits, with a 3x3 neighborhood fallback when there is no hit, and lerp by latest hit ratio during motion to reduce ghosting: https://arxiv.org/pdf/2210.04981 and https://arxiv.org/pdf/2210.06158 [secondary].
- A search for a published "transmittance-weighted depth" reprojection for clouds returned nothing; it is not an established term in anything reached.
- Reported costs and artefacts of history: higher history weight = more ghosting for moving volumes (Godot, 1.4); EMA "stable in static scenes but performs poorly under dynamic volumetric lighting", and variance clipping plus low-discrepancy jitter tested on froxel data, implemented in Frostbite (a thesis at https://gupea.ub.gu.se/items/e05613c2-e1d4-48b0-b41e-c32f4b0ecbae, blocked by bot protection; only a search summary was seen) [secondary]; UE TSR trailing when no depth/velocity is written (1.6).
- Playdead's TAA (Pedersen, GDC 2016) is the history-reprojection reference cited by both Frostbite and Toft et al.
## 3. Flow maps, curl noise, advected textures
### 3.1 Flow maps, Vlachos, Valve, SIGGRAPH 2010 "Water Flow in Portal 2"
Source: https://www.advances.realtimerendering.com/s2010/Vlachos-Waterflow(SIGGRAPH%202010%20Advanced%20RealTime%20Rendering%20Course).pdf (read)
- Builds on Max and Becker 1995 "Flow visualization using moving textures": offset the UV by the 2D flow vector scaled by time; "distortion looks reasonable for the first 1/3 of uv space".
- "Blend the short animated segment in two layers. Each layer is offset half a phase so we can hide the restart for each layer."
- Two named problems: repetition ("the same normals will flow through the same point") and pulsing ("the surface appears to pulse in a repeating pattern"). Repetition solved by offsetting the second layer's sample; pulsing solved by a noise texture that offsets phase per pixel. Cost vs two scrolling normal maps: +2 texture fetches, +21 ALU instructions.
- Colors (Portal 2): distortion interval offset to -fraction..+fraction so the peak is at rest, favouring less distortion than for normals. Listed future work: dynamic per-frame flow vectors.
- Catlike Coding (https://catlikecoding.com/unity/tutorials/flow/texture-distortion/) [secondary]: p=frac(time), weight `1 - |1 - 2p|`, second pattern shifted half a period so weights sum to 1; Perlin noise in the flow map's A channel added to time staggers the pulse as a wave; per-phase "jump" vector times floor(time) shifts UVs only while weight is zero, limited to about +/-0.25, loop length is LCM of jump denominators; strong deformation makes compressed flow maps blocky; large-area flow maps have low effective resolution.
- Golus flow-map notes: [not verified].
### 3.2 Curl noise, Bridson, Hourihan, Nordenstam, TOG 26(3) 2007
Source: https://www.cs.ubc.ca/~rbridson/docs/bridson-siggraph2007-curlnoise.pdf (read, 3 pages)
- Velocity = curl of a noise-valued potential (2D scalar potential rotated 90 degrees; 3D three noise functions, "the same noise function evaluated at large offsets"). Divergence-free by construction: "No sources or sinks ('gutters')"; contrasted with plain noise as velocity, which "will generally contain many sinks" where particles accumulate.
- Derivatives by finite difference (displacement 10^-4 of the domain "works fine in single precision"). With noise at length scale L: vortices of diameter about L, speeds O(1/L). Several octaves with power-law amplitude falloff resemble Kolmogorov turbulence.
- Time variation: "achieved very simply by using time-varying noise"; pseudo-advection of Flow Noise mentioned as untried.
- Boundaries: potential multiplied by smooth ramp of distance to solids, `ramp(r) = 15/8 r - 10/8 r^3 + 3/8 r^5`.
- The paper names particle advection as a primary use. It says Perlin and Neyret's Flow Noise textures "cannot be used for moving particle systems".
- Perlin and Neyret, "Flow Noise" (SIGGRAPH 2001 sketch): rotating gradients over time plus pseudo-advection of small octaves by large ones; needs noise with analytic derivatives. https://evasion.inrialpes.fr/~Fabrice.Neyret/flownoise/index.gb.html ; Gustavson's simplex noise code exposes the derivative and a rotation parameter `t` for it (https://www.itn.liu.se/~stegu76/simplexnoise/DSOnoises.html) [secondary].
### 3.3 Advected textures, Neyret, SCA 2003
Source: https://evasion.inrialpes.fr/Publications/2003/Ney03/neyret161.pdf (read)
- A noise texture advected by a flow stretches; periodic regeneration prevents stretch from growing on steady flows, but the regeneration rate (latency) is hard to tune and blending regenerated layers causes ghosting and contrast loss.
- Method: accumulate local deformation (time integral of the strain-tensor norm) per layer since its last regeneration; blend layers of different latency by a criterion on that accumulated deformation against a user target (analogous to MIP-mapping); each layer blends three phase-shifted regenerated textures; proposes a procedural-texture blending scheme that avoids contrast loss.
- Needs a velocity field to compute deformation; the paper's flow came from a low-resolution fluid simulation.
### 3.4 Does a span / Lipschitz bound survive advection or a time-dependent warp
No source found states it. What was read:
- Seyb, Jacobson, Nowrouzezahrai, Jarosz, "Non-linear sphere tracing for rendering deformed signed distance fields", TOG 38(6) 2019 (PDF read): stepping in deformed space needs the Lipschitz constant of S o D^-1; "for affine transformations this is trivial [Stander and Hart 1994], but for more complicated deformations Lipschitz constants can be tedious or even impossible to compute analytically"; a global constant degrades step size where it is large; local constants "require a non-negligible computation whenever the deformation changes, i.e., every frame". Their remedy: trace in undeformed space as an ODE, avoiding the inverse deformation. https://par.nsf.gov/servlets/purl/10172295
- Galin et al., "Segment Tracing Using Local Lipschitz Bounds", CGF 39(2) 2020: local Lipschitz bounds per ray segment, bounds derived per operator and primitive (abstract/search summary only) https://diglib.eg.org/handle/10.1111/cgf13951 [secondary].
- Bridson 2007: speed of a noise-derived velocity is O(1/L) per octave (informal amplitude bound).
- Neyret 2003: the quantity that grows under advection is the time-integrated strain-tensor norm.
- A bound on the time derivative of a warped field, or a space-time Lipschitz constant: [not found].
## 4. Particle layers coherent with a volumetric field
- No shipped system found that drives volumetric density and a particle layer from one stated shared motion law.
- Niagara Curl Noise Force: reads a pre-baked, medium-resolution tiling curl-noise field by default, or computes a Perlin-derived curl at higher cost; docs do not state the divergence-free property: https://dev.epicgames.com/documentation/unreal-engine/particle-spawn-group-reference-for-niagara-effects-in-unreal-engine [secondary].
- Frostbite 2015 couples particles to fog by voxelizing particle extinction into the same froxel volume, so they are lit together (shared volume, not shared velocity): https://www.ea.com/news/physically-based-unified-volumetric-rendering-in-frostbite [secondary].
- Flax: volume particles write into the fog's 3D textures: https://docs.flaxengine.com/manual/graphics/fog-effects/volumetric-fog.html [secondary]. Godot demo of particle-faked fog animated with wind/attractors; "billboards make the fog appear to rotate with the camera": https://github.com/Bauxitedev/volumetric-fog-demo [secondary].
- Bridson's curl noise is built for particle advection and the divergence-free property is its stated reason (3.2).
- Nubis 2022 / Nubis3 slides read carry no particle layer sharing the density motion law.
## 5. Not reached
RDR2 (Bauer 2019) slides; Hillaire 2015 temporal section; Golus flow-map notes; any Shadertoy nebula source; UE internals beyond forum reports; any source bounding the time derivative of a warped noise field.

11
docs/research/lattice-site-weight/weighted-node/a.py Normal file
View file

@ -0,0 +1,11 @@
import numpy as np, probe as P
g = np.linspace(-3200, 3200, 260); X, Z = np.meshgrid(g, g); q = np.stack([X, Z], -1)
rng = np.random.default_rng(3); ang = rng.uniform(0, 6.283, X.shape); nrm = np.stack([np.cos(ang), np.sin(ang)], -1)
for boost, S in [(1.6, 1), (1.6, 2), (4, 1), (4, 2), (9, 1)]:
ref = P.node(q, 100.0, 9, boost, S, nrm)
row = []
for W in (2, 3, 4, 5):
r = P.node(q, 100.0, W, boost, S, nrm)
miss = np.mean((np.abs(r[0]-ref[0])>1e-12)|(np.abs(r[1]-ref[1])>1e-12))
row.append(f"W{W}: miss {miss:.5f} dOut {np.abs(r[3]-ref[3]).max():.3f}")
print(f"boost {boost} S {S} ratio {2*(1+boost):.1f}x{S}:", "; ".join(row))

10
docs/research/lattice-site-weight/weighted-node/c.py Normal file
View file

@ -0,0 +1,10 @@
import numpy as np, probe as P
g = (np.arange(-12, 12) + .5) * 260; X, Z = np.meshgrid(g, g); c = np.stack([X, Z], -1).reshape(-1, 2)
for ex in (False, True):
prev = None; worst = 0; big = 0
for t in np.arange(0, 300, 1/60):
Pw = np.exp(-(P.border(c, P.bodies(t), ex) / P.FALL) ** 2)
if prev is not None:
s = np.abs(Pw - prev); worst = max(worst, s.max()); big += (s > .01).sum()
prev = Pw
print("exact" if ex else "first-order", f"cell-centre profile: max step per 1/60 s {worst:.4f}, steps > .01: {big} (576 centres x 18000 ticks)")

9
docs/research/lattice-site-weight/weighted-node/d3.py Normal file
View file

@ -0,0 +1,9 @@
import numpy as np, probe as P
rng = np.random.default_rng(9); n = 15000
kx = rng.integers(-12, 12, n); z = rng.uniform(-3100, 3100, n); eps = 1e-6
qa = np.stack([kx * 260.0 - eps, z], -1)[None]; qb = np.stack([kx * 260.0 + eps, z], -1)[None]
for W, fold, boost, S in [(6,"f1kw",1.6,2),(6,"her",1.6,2),(6,"f1kw",4,2)]:
ang = rng.uniform(0, 6.283, n); nrm = np.stack([np.cos(ang), np.sin(ang)], -1)[None]
a = P.node(qa, 100.0, W, boost, S, nrm, True, fold); b = P.node(qb, 100.0, W, boost, S, nrm, True, fold)
j = np.abs(a[3] - b[3]); js = np.abs(a[2]-b[2]); jf = np.abs(a[0]-b[0])
print(f"W{W} {fold} B{boost} S{S}: cell-boundary jump out max {j.max():.4f} (>.01: {np.mean(j>.01):.4f}), soft max {js.max():.4f}, F1 max {jf.max():.2e}")

13
docs/research/lattice-site-weight/weighted-node/e.py Normal file
View file

@ -0,0 +1,13 @@
import numpy as np, probe as P
g = np.linspace(-3200, 3200, 200); X, Z = np.meshgrid(g, g); q = np.stack([X, Z], -1)
rng = np.random.default_rng(3); ang = rng.uniform(0, 6.283, X.shape); nrm = np.stack([np.cos(ang), np.sin(ang)], -1)
wmin = .5
for boost, S in [(1.6, 1), (1.6, 2), (4, 2), (9, 1)]:
f1, f2, soft, out = P.node(q, 100.0, 6, boost, S, nrm)
C = np.maximum(f2, f1 + P.KW)
k = np.floor(C / wmin).astype(int) + 1 # first ring whose lower bound (k-1)*wmin >= C is excluded... rings visited 0..k
k = np.ceil(C / wmin).astype(int) # rings 0..k visited (ring k+1 has lb k*wmin >= C)
cells = (2 * k + 1) ** 2
wmax = 1.0 * (1 + boost); Usound = 1.58 * wmax * S
Wsound = int(np.ceil((Usound + P.KW) / wmin))
print(f"B{boost} S{S}: rings needed p50 {np.median(k):.0f} p99 {np.percentile(k,99):.0f} max {k.max()} -> cells p50 {np.median(cells):.0f} max {cells.max()}; worst-case sound W {Wsound} ({(2*Wsound+1)**2} cells)")

14
docs/research/lattice-site-weight/weighted-node/g.py Normal file
View file

@ -0,0 +1,14 @@
import numpy as np, probe as P, sys
fold = sys.argv[1]; rng = np.random.default_rng(2); jm = js = 0; nsw = 0
for k in range(3):
o = rng.uniform(-2500, 2500, 2); a = rng.uniform(0, 3.1416); d = np.array([np.cos(a), np.sin(a)])
s = np.arange(0, 15 * 260, .5); P.node((o + s[:, None] * d)[None], 100.0, 3, 1.6, 1.0, None, True, fold); a1 = P.node.a1[0].copy()
idx = np.nonzero(np.diff(a1))[0][:40]
lo = s[idx]; hi = s[idx + 1]; ref = a1[idx]
for _ in range(40):
mid = .5 * (lo + hi); P.node((o + mid[:, None] * d)[None], 100.0, 3, 1.6, 1.0, None, True, fold)
same = P.node.a1[0] == ref; lo = np.where(same, mid, lo); hi = np.where(same, hi, mid)
ra = P.node((o + lo[:, None] * d)[None], 100.0, 3, 1.6, 1.0, None, True, fold); ma = P.node.m[0].copy()
rb = P.node((o + hi[:, None] * d)[None], 100.0, 3, 1.6, 1.0, None, True, fold); mb = P.node.m[0]
jm = max(jm, np.abs(ma - mb).max()); js = max(js, np.abs(ra[2] - rb[2]).max()); nsw += len(idx)
print(fold, f"F1 swaps {nsw}: max jump m {jm:.2e}, soft {js:.2e}")

6
docs/research/lattice-site-weight/weighted-node/h.py Normal file
View file

@ -0,0 +1,6 @@
import numpy as np, probe as P
g = np.linspace(-3200, 3200, 200); X, Z = np.meshgrid(g, g); q = np.stack([X, Z], -1)
rng = np.random.default_rng(3); ang = rng.uniform(0, 6.283, X.shape); nrm = np.stack([np.cos(ang), np.sin(ang)], -1)
for S in (1, 2):
a = P.node(q, 100.0, 6, 1.6, S, nrm, True, "her"); b = P.node(q, 100.0, 6, 1.6, S, nrm, True, "f1kw")
d = np.abs(a[3] - b[3]); print(f"S{S} look her vs f1kw: mean {d.mean():.4f} p99 {np.percentile(d,99):.3f} max {d.max():.3f}; mean out {a[3].mean():.3f}")

60
docs/research/lattice-site-weight/weighted-node/probe.py Normal file
View file

@ -0,0 +1,60 @@
"""Weighted lattice cellular probe (Imagination, lattice-weighted-cellular). numpy, float64.
Bodies orbit (period = distance s). Cell weight = base(hash)*(1+BOOST*exp(-(border(centre)/FALL)^2)),
border over the bodies' power scores (site.w = sigma). Her fold: row-major, m from 8; variant m0f1 starts m at F1."""
import numpy as np, sys
LAT, FALL, KW, EW, OMEGA = 260.0, 120.0, .35, .10, .05
GAP0 = .25 * KW / (1 + 3 * KW)
rng = np.random.default_rng(7)
NB = 6
DIST = np.array([0.0, 700, 1200, 1800, 2400, 3000]); PH = rng.uniform(0, 6.283, NB)
SIG = np.array([1500, 300, 450, 350, 900, 600]) / np.sqrt(128.0) * 4 # sigma ratio 5:1, edges well inside
def bodies(t):
a = PH + np.where(DIST > 0, 2 * np.pi * t / np.maximum(DIST, 1), 0)
return np.stack([np.cos(a) * DIST, np.sin(a) * DIST], -1)
def h32(x):
x = x.astype(np.uint32)
x ^= x >> 16; x *= np.uint32(0x7feb352d); x ^= x >> 15; x *= np.uint32(0x846ca68b); x ^= x >> 16
return x
def hash01(cx, cz, s):
k = ((cx.astype(np.int64) * 73856093) ^ (cz.astype(np.int64) * 19349663) ^ (s * 83492791)) & 0xffffffff
return (h32(h32(k)) >> 8) / 16777216.0
def border(q, B, exact):
"""q: (...,2). Power border distance over bodies B."""
sc = ((q[..., None, :] - B) ** 2).sum(-1) - SIG ** 2
o = np.argsort(sc, -1); i1 = o[..., 0]; i2 = o[..., 1]
s1 = B[i1]
if not exact:
s2 = B[i2]; return (np.take_along_axis(sc, i2[..., None], -1)[..., 0] - np.take_along_axis(sc, i1[..., None], -1)[..., 0]) / (2 * np.linalg.norm(s2 - s1, axis=-1))
best = np.full(q.shape[:-1], 1e30)
w1 = SIG[i1]
for k in range(NB):
d = B[k] - s1; L = np.linalg.norm(d, axis=-1)
pl = np.maximum((((s1 - q) * d).sum(-1) + .5 * ((d * d).sum(-1) + w1 ** 2 - SIG[k] ** 2)) / np.where(L > 0, L, 1), 0)
best = np.where((L > 0) & (pl < best), pl, best)
return best
def node(q, t, W, boost, S=1.0, nrm=None, exact=True, fold="her"):
B = bodies(t); p = q / LAT; n = np.floor(p); f = p - n
if nrm is None: nrm = np.zeros_like(q); nrm[..., 0] = 1
ds = []
for j in range(-W, W + 1):
for i in range(-W, W + 1):
cx, cz = n[..., 0] + i, n[..., 1] + j
base = .5 + .5 * hash01(cx, cz, 1)
P = np.exp(-(border(np.stack([(cx + .5) * LAT, (cz + .5) * LAT], -1), B, exact) / FALL) ** 2)
wt = base * (1 + boost * P)
ox = .5 + .5 * np.sin(OMEGA * t + 6.2831 * hash01(cx, cz, 2)); oz = .5 + .5 * np.sin(OMEGA * t + 6.2831 * hash01(cx, cz, 3))
vx, vz = i - f[..., 0] + ox, j - f[..., 1] + oz
a = vx * nrm[..., 0] + vz * nrm[..., 1]; c = -vx * nrm[..., 1] + vz * nrm[..., 0]
ds.append(np.hypot(S * a, c) * wt)
D = np.stack(ds, -1); srt = np.sort(D, -1); f1, f2 = srt[..., 0], srt[..., 1]; a1 = np.argmin(D, -1)
m = np.full(f1.shape, 8.0) if fold == "her" else (f1 + KW if fold == "f1kw" else f1.copy())
for k in range(D.shape[-1]):
d = D[..., k]
h = np.clip(.5 + .5 * (m - d) / KW, 0, 1); h = h * h * (3 - 2 * h)
mn = m + (d - m) * h - h * (1 - h) * KW / (1 + 3 * KW)
m = np.where((fold == "m0f1") & (a1 == k), m, mn)
soft = np.clip((f1 - m) / GAP0, 0, 1); node.m = m; node.a1 = a1
return f1, f2, soft, np.maximum(soft ** 2, np.exp(-((f2 - f1) / EW) ** 2))

143
docs/research/native-shader-probes.md Normal file
View file

@ -0,0 +1,143 @@
# Shader runner probes: the HLSL-to-GLSL ES toolchain, lavapipe on Yggdrasil, native ANGLE on Starfire
For the cultmath-tapes questions that replace `device-run-driver`: `glsl-es-translation-owner`,
`conforming-shader-run` and `real-device-shader-run`. These are body facts measured by
imagination-ag4-shader-runner (session self-2026-10-16-ag4) on 2026-10-16, between about 14:10 and
15:20 UTC (16:10 to 17:20 CEST). Every probe read CultLib at f697e03f (origin/main). Companion to
`docs/research/native-shader-testing.md` (f697e03f) and `docs/research/device-capture-runners.md` (c9d3e672).
The probe scripts are in this directory under `p/`: `probe.sh` and `lib.sh` (Yggdrasil jobs), `fam.py`
(the per-family shader generator and bit judge), `vkrun.c` (an 87-line Vulkan compute host) and
`angle_native.py` (a 96-line native ANGLE host). They are throwaway probe code, not a proposed
implementation. The logs are `q12-run3.log` (lavapipe) and `lib-run4.log` (library route). The earlier
runs are superseded: run1 used a raw-ulp metric, run2 died on CRLF, and lib-run1 to lib-run3 failed on
the capability and grep problems recorded below.
## Metric
Each fixture family has 256 cases, and the fixture records the C# result bits. "Exact" means all 256
cases are bit-identical to the C# fixture. Otherwise the worst error is given in units of
2^-23 * max(|expected|, 1) (written "u1" below), the scale the fixture's ulp-bounded checks use. NaN and
Inf cases are compared only as NaN-versus-not-NaN. These figures describe the data. They are not the
fixture's own judge, which is C# and per family: GlslMirrorTests tolerances, and the `check` text on 10
families.
## The fixture surface (glsl-parity.json at f697e03f)
- 109 families, each with 256 cases, encoded `args -> result` as hex 32-bit words. There are at most 32
words per case.
- 105 families use only float, int, vecN, ivecN and mat3 (mat3 appears as a return only). The other 4
take or return structs: `CultCellular cultmath_cellular(vec3)`, `CultPhasor cultmath_phacelle(vec3,
vec3, float, float)`, and `SpanNoise cultmath_span_fbm/ridged(SpanPath, SpanStage, int, float, float)`.
Any runner needs per-struct flatteners in field order for these 4.
- Tolerance classes: 89 are `ulp-bounded`, 2 are `exact`, 7 are `platform` (exp, pow, log, asin, acos,
atan, and phacelle's sin and cos), and 10 carry a `check` (enclosure).
- What a harness needs: generate one shader per family that reads the argument words, bit-casts them to
the signature's types, calls the function and writes the result words. The probe generator does this
for both HLSL and GLSL in 112 lines of Python, judge included.
## Q1. Can dxc emit a SPIR-V library that SPIRV-Cross lowers to GLSL ES for CultMath's real surface?
Yggdrasil, ubuntu:24.04. Tools: dxc v1.9.2607 Linux (`linux_dxc_2026_07_29`, the version get-dxc.ps1
pins on Windows), and SPIRV-Cross built from HEAD 300681bb (2026-10-14).
| Step | Result |
|---|---|
| `dxc -T lib_6_6 -spirv -fspv-target-env=vulkan1.2 CultMath.hlsl`, no change to the source | fails: "No OpEntryPoint ... only allowed if the Linkage ... capability is being used" |
| `-default-linkage external` (lib_6_3, universal1.5), no `export` | fails the same way: no exports in SPIR-V |
| `export` added to every top-level `cultmath_` definition, vulkan1.2 | fails: "Capability Linkage is not allowed by Vulkan 1.2" |
| same, `-fspv-target-env=universal1.5` | compiles, but declares 14 capabilities, ray query, interlock, quad control and partitioned subgroups among them |
| SPIRV-Cross `--es --version 310` on that module | fails: "RayQuery requires Vulkan GLSL 460" |
| ray query stripped | fails: "Cannot find mapping for combined sampler parameter". The Texture2D/SamplerState functions were exported |
| `export` only on the 173 non-texture definitions, all capabilities but Shader and Linkage stripped, every OpExtension stripped, then `spirv-as` | assembles; SPIRV-Cross `--es --version 310` exits 0 |
| SPIRV-Cross `--es --version 300` | fails: "At least ESSL 3.10 required for compute shaders" (a library gets a GLCompute placeholder entry) |
| ES 3.10 output, `#version` guarded by `#ifdef SPIRV_CROSS_LIBRARY_HEADER`, placeholder main removed, compiled by glslang as a `#version 300 es` fragment shader | compiles |
Shape of the output against the committed lowering:
| | committed (CultMath.glsl, .Cycles, .Phacelle) | dxc + SPIRV-Cross |
|---|---|---|
| lines | 1035 + Cycles + Phacelle | 3458, one file (the MPL and Apache files are merged) |
| function definitions | 173 (121 + 51 + 1) | 127 |
| overloads | kept | **46 dropped**: one definition per name survives (for example `cultmath_snoise(vec2)` and `cultmath_iv_snoise_ball(vec2, float)` are gone) |
| SSA temporaries `_NNN` | 0 | 5632 |
| min and max | as written | NaN-guarded `mix(mix(min(a,b),b,isnan(a)),a,isnan(b))` |
In `cultmath_snoise(vec3)` as emitted, the body is one temporary per SSA value (`vec3 _1157 = floor(...)`)
and constants are spelt to 20 digits.
**Answer:** yes, but only with three owned workarounds: an `export` annotation on 173 definitions,
which FXC (Unity's D3D11 compiler) does not accept, so it would have to be a macro; a capability and
extension strip pass; and unique names for 46 overloads, which breaks the public API for both the HLSL
and the GLSL consumers. The output is generated text, not a readable library.
## Q2. Does a conforming Vulkan implementation run the families in a Linux container on Yggdrasil?
Yggdrasil, ubuntu:24.04, apt packages: mesa-vulkan-drivers 25.2.8 (lavapipe), glslang-tools 15.1.0,
spirv-tools 2025.1~rc1, libvulkan-dev, and the same dxc. There is no GPU.
- Device: `llvmpipe (LLVM 20.1.2, 256 bits)`, Vulkan 1.4. Float controls for f32: denormPreserve 0,
flushToZero 0, RTE 1, RTZ 0, signedZeroInfNanPreserve 1.
- HLSL route: a generated `cs_6_0` compute shader that includes CultMath.hlsl, through `dxc -spirv
-fspv-target-env=vulkan1.2`. 105 of 105 families compile and run. Fma instructions in the SPIR-V: 0.
- GLSL route: the committed CultMath.glsl, .Cycles.glsl and .Phacelle.glsl concatenated after `#version
310 es` and highp precision, as a compute shader through `glslangValidator -V`. 105 of 105 compile and
run.
- Library route: the dxc + SPIRV-Cross library from Q1 through glslang. 103 of 105 run; the 2 missing
ones are the dropped overloads. Against the fixture: 75 exact, 4 within 4 u1, 7 within 64, 17 above
64. Its worst rows are the same figures as the HLSL route's.
- Time: all 105 dispatches together take 8 s on a re-run. Each job runs about 4 minutes, almost all of
it apt, the dxc download and the SPIRV-Cross build.
- Determinism: re-running all 105 SPIR-V modules gave bit-identical outputs.
- The HLSL and GLSL routes are bit-identical to each other in 85 of 105 families.
- Against the C# fixture, HLSL route: 75 exact, 6 within 4 u1, 7 within 64, 17 above 64. GLSL route:
75, 4, 8 and 18. The largest errors are in the `coordinate bands 1e2 to 4e6` families of snoise_grad
and snoise_hessian (13e6 and 1.6e8 u1), then fbm_grad and ridged_grad (about 3.5e3 to 5e3 u1), Cycles
arcsine (2.7e3) and arccosine (1.0e3), and the other hessian families (1e3 to 2.4e3).
**Answer:** yes. The image needs nothing beyond distro packages and the pinned dxc. It is
deterministic and fast, and it runs both the HLSL and the lowered GLSL on the same implementation.
## Q3. A real-device run on Starfire without a browser: native ANGLE
Starfire. The host is `angle_native.py`, Python ctypes over the libEGL.dll and libGLESv2.dll that ship
with Docker Desktop 4.71.0 (an Electron app): ANGLE 2.1.26626, git hash a4490148bd3d. Display:
`EGL_PLATFORM_ANGLE_TYPE_D3D11`. Context: ES 3.0, optionally `EGL_CONTEXT_WEBGL_COMPATIBILITY_ANGLE`.
Each family is a `#version 300 es` fragment shader, with arguments read by texelFetch from an RGBA32UI
texture and results written to up to 3 RGBA32UI targets over a 16x16 grid, then read back with
glReadPixels. The same RGBA32UI readback is used by the scratchpad browser pages (gpu.js).
- `GL_RENDERER`: `ANGLE (NVIDIA, NVIDIA GeForce GTX 1070 (0x00001B81) Direct3D11 vs_5_0 ps_5_0, D3D11)`.
`GL_VERSION`: `OpenGL ES 3.0 (ANGLE 2.1.26626 git hash: a4490148bd3d)`.
- 105 of 105 families compile and run. All 105 together take 6.3 s wall, process start included. The GL
error at the end is 0.
- The WebGL-compatibility context and the plain ES 3.0 context are bit-identical in 105 of 105 families.
- Against the C# fixture: 38 exact, 31 within 4 u1, 17 within 64, 19 above 64.
- It agrees with the earlier browser measurements on the same GPU. `cultmath_iv_exp`: 19 of 256 exact,
worst 12 ulp; the browser run of finding iv-exp-gpu-not-enclosed measured "up to 12 ulp".
`cultmath_iv_frustum_ball`: 213 of 256 exact, so 43 off, by at most 2 ulp; finding
frustum-exact-unsound-on-webgl2 measured "43/256 cases by up to 2 ulp".
- The dxc + SPIRV-Cross library from Q1 also runs on this ANGLE, as a WebGL-compatibility ES 3.00
fragment shader: 103 of 105 families (the 2 dropped overloads fail to compile), with 41 exact. It is
bit-identical to the committed lowering's ANGLE output in 71 of 103 families.
- Largest differences: the `coordinate bands 1e2 to 4e6` families again. 173 cases of snoise_grad and
415 component values of snoise_hessian differ by more than 1e-3 relative, up to 2 and 20. These
families exist for the span allowance's device margin (follow-up span-allowance-gpu-margin, cut
stage-shaders-device). Their judge is the allowance check, not an ulp bound.
**Answer:** yes. A pinned ANGLE build, loaded natively, runs the site's real translation path
(ES 3.00, then ANGLE, then HLSL, then D3DCompile, then the NVIDIA D3D11 driver) with no browser and no
SwiftShader fallback, because the backend is named explicitly and the renderer string is recorded. The
result agrees bit for bit with the WebGL-compatibility context, and it reproduces the two browser
measurements on record.
## What was not probed
- SwiftShader as a second conforming implementation.
- Amber as the dispatch host instead of a small owned one.
- WARP and native D3D11 HLSL on Starfire, which is the Unity consumer path.
- naga with dxc SPIR-V as input.
- Chrome's own ANGLE feature flags, which the native host does not reproduce.
- Mobile or mediump devices.
- An Electron release's ANGLE pinned by download. The probe used the copy Docker Desktop happens to
ship.

96
docs/research/native-shader-probes/angle_native.py Normal file
View file

@ -0,0 +1,96 @@
"""Probe: run the GLSL fixture families through a native ANGLE (DLLs shipped with an Electron app), D3D11
backend, ES 3.0 context, fragment-shader readback through RGBA32UI targets. No browser.
usage: py -3 -I angle_native.py <angle-dll-dir> <fixture.json> <shaders-dir> <outdir> [webgl]"""
import ctypes as C, json, os, struct, sys
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import fam
dll, fx, sh, out = sys.argv[1:5]; webgl = len(sys.argv) > 5
os.add_dll_directory(dll)
egl = C.CDLL(os.path.join(dll, 'libEGL.dll')); gl = C.CDLL(os.path.join(dll, 'libGLESv2.dll'))
egl.eglGetProcAddress.restype = C.c_void_p
for n in ['eglInitialize', 'eglChooseConfig', 'eglCreatePbufferSurface', 'eglCreateContext', 'eglMakeCurrent', 'eglBindAPI', 'eglGetError']:
getattr(egl, n).restype = C.c_void_p if n.startswith('eglCreate') else C.c_uint
egl.eglCreatePbufferSurface.argtypes = [C.c_void_p, C.c_void_p, C.POINTER(C.c_int)]
egl.eglCreateContext.argtypes = [C.c_void_p, C.c_void_p, C.c_void_p, C.POINTER(C.c_int)]
egl.eglMakeCurrent.argtypes = [C.c_void_p, C.c_void_p, C.c_void_p, C.c_void_p]
egl.eglInitialize.argtypes = [C.c_void_p, C.c_void_p, C.c_void_p]
egl.eglChooseConfig.argtypes = [C.c_void_p, C.POINTER(C.c_int), C.POINTER(C.c_void_p), C.c_int, C.POINTER(C.c_int)]
GPD = C.CFUNCTYPE(C.c_void_p, C.c_uint, C.c_void_p, C.POINTER(C.c_int))(egl.eglGetProcAddress(b'eglGetPlatformDisplayEXT'))
def ia(*v): return (C.c_int * (len(v) + 1))(*v, 0x3038)
disp = GPD(0x3202, None, ia(0x3203, 0x3208)) # EGL_PLATFORM_ANGLE_TYPE_ANGLE = D3D11
assert egl.eglInitialize(disp, None, None), 'eglInitialize %x' % egl.eglGetError()
cfg = C.c_void_p(); ncfg = C.c_int()
egl.eglChooseConfig(disp, ia(0x3033, 0x1, 0x3040, 0x40), C.byref(cfg), 1, C.byref(ncfg)) # PBUFFER, ES3
surf = egl.eglCreatePbufferSurface(disp, cfg, ia(0x3057, 16, 0x3056, 16))
egl.eglBindAPI(0x30A0)
ctx_attr = (0x3098, 3) + ((0x33AC, 1) if webgl else ()) # CLIENT_VERSION 3 [, WEBGL_COMPATIBILITY_ANGLE]
ctx = egl.eglCreateContext(disp, cfg, None, ia(*ctx_attr))
assert ctx and egl.eglMakeCurrent(disp, surf, surf, ctx), 'context %x' % egl.eglGetError()
gl.glGetString.restype = C.c_char_p
print('GL_RENDERER', gl.glGetString(0x1F01).decode()); print('GL_VERSION', gl.glGetString(0x1F02).decode())
print('webgl compatibility context', webgl)
gl.glCreateShader.restype = C.c_uint; gl.glCreateProgram.restype = C.c_uint
def shader(kind, src):
s = gl.glCreateShader(kind); b = C.c_char_p(src.encode()); gl.glShaderSource(s, 1, C.byref(b), None); gl.glCompileShader(s)
ok = C.c_int(); gl.glGetShaderiv(s, 0x8B81, C.byref(ok))
if not ok.value:
buf = C.create_string_buffer(4000); gl.glGetShaderInfoLog(s, 4000, None, buf); raise RuntimeError(buf.value.decode()[:300])
return s
VS = '#version 300 es\nvoid main() { vec2 p = vec2((gl_VertexID << 1) & 2, gl_VertexID & 2); gl_Position = vec4(p * 2.0 - 1.0, 0.0, 1.0); }\n'
lib = open(os.environ['LIBGLSL']).read() if os.environ.get('LIBGLSL') else ''.join(open(os.path.join(sh, f)).read() for f in ['CultMath.glsl', 'CultMath.Cycles.glsl', 'CultMath.Phacelle.glsl'])
vao = C.c_uint(); gl.glGenVertexArrays(1, C.byref(vao)); gl.glBindVertexArray(vao)
d = json.load(open(fx)); man = []; skipped = 0
for idx, f in enumerate(d['functions']):
r, name, ps = fam.parse(f['glsl']); rt = fam.ty(r); pts = [fam.ty(p) for p in ps]
if rt is None or any(p is None for p in pts): skipped += 1; continue
nin = sum(p[1] for p in pts); nout = rt[1]; tw = (nin + 3) // 4; nt = (nout + 3) // 4; tag = 'f%03d' % idx
words = [int(x, 16) for c in f['cases'] for x in c.split('->')[0].split()]
tex = []
for ci in range(256):
row = words[ci * nin:(ci + 1) * nin]; tex += row + [0] * (tw * 4 - nin)
lines, off = [], 0
for j, (p, (ht, n, k)) in enumerate(zip(ps, pts)):
src = ', '.join('w[%d]' % (off + t) for t in range(n)); v = ('uvec%d(%s)' % (n, src)) if n > 1 else src
conv = {'f': 'uintBitsToFloat', 'i': ('ivec%d' % n if n > 1 else 'int'), 'u': ''}[k]
lines.append(' %s a%d = %s(%s);' % (p, j, conv, v)); off += n
lines.append(' %s r = %s(%s);' % (r, name, ', '.join('a%d' % j for j in range(len(ps)))))
comps = ['r[%d][%d]' % (a, b) for a in range(3) for b in range(3)] if r == 'mat3' else ['r'] if nout == 1 else ['r[%d]' % i for i in range(nout)]
for i, c in enumerate(comps):
lines.append(' ow[%d] = %s(%s);' % (i, {'f': 'floatBitsToUint', 'i': 'uint', 'u': ''}[rt[2]], c))
outs = ''.join('layout(location = %d) out highp uvec4 o%d;\n' % (t, t) for t in range(nt))
wr = ''.join(' o%d = uvec4(ow[%d], ow[%d], ow[%d], ow[%d]);\n' % (t, 4 * t, 4 * t + 1, 4 * t + 2, 4 * t + 3) for t in range(nt))
FS = ('#version 300 es\nprecision highp float;\nprecision highp int;\n' + lib + '\nuniform highp usampler2D In;\n' + outs +
'void main() {\n int i = int(gl_FragCoord.y) * 16 + int(gl_FragCoord.x);\n uint w[%d];\n for (int k = 0; k < %d; k++) w[k] = texelFetch(In, ivec2(k / 4, i), 0)[k %% 4];\n'
' uint ow[%d];\n' % (tw * 4, nin, nt * 4) + '\n'.join(lines) + '\n' + ''.join(' ow[%d] = 0u;\n' % z for z in range(nout, nt * 4)) + wr + '}\n')
try:
prog = gl.glCreateProgram(); gl.glAttachShader(prog, shader(0x8B31, VS)); gl.glAttachShader(prog, shader(0x8B30, FS)); gl.glLinkProgram(prog)
except RuntimeError as e:
print('compile fail', f['glsl'], str(e).replace('\n', ' ')[:200]); continue
gl.glUseProgram(prog)
t_in = C.c_uint(); gl.glGenTextures(1, C.byref(t_in)); gl.glActiveTexture(0x84C0); gl.glBindTexture(0x0DE1, t_in)
gl.glTexParameteri(0x0DE1, 0x2801, 0x2600); gl.glTexParameteri(0x0DE1, 0x2800, 0x2600)
buf = (C.c_uint * len(tex))(*tex); gl.glTexImage2D(0x0DE1, 0, 0x8D70, tw, 256, 0, 0x8D99, 0x1405, buf)
gl.glUniform1i(gl.glGetUniformLocation(prog, b'In'), 0)
fbo = C.c_uint(); gl.glGenFramebuffers(1, C.byref(fbo)); gl.glBindFramebuffer(0x8D40, fbo)
tos = (C.c_uint * nt)(); gl.glGenTextures(nt, tos)
for t in range(nt):
gl.glBindTexture(0x0DE1, tos[t]); gl.glTexImage2D(0x0DE1, 0, 0x8D70, 16, 16, 0, 0x8D99, 0x1405, None)
gl.glFramebufferTexture2D(0x8D40, 0x8CE0 + t, 0x0DE1, tos[t], 0)
gl.glDrawBuffers(nt, (C.c_uint * nt)(*[0x8CE0 + t for t in range(nt)]))
gl.glActiveTexture(0x84C0); gl.glBindTexture(0x0DE1, t_in)
gl.glViewport(0, 0, 16, 16); gl.glDrawArrays(0x0004, 0, 3)
res = [[0] * 1024 for _ in range(nt)]
for t in range(nt):
gl.glReadBuffer(0x8CE0 + t); px = (C.c_uint * 1024)(); gl.glReadPixels(0, 0, 16, 16, 0x8D99, 0x1405, px); res[t] = list(px)
flat = []
for ci in range(256):
allw = sum((res[t][ci * 4:ci * 4 + 4] for t in range(nt)), []); flat += allw[:nout]
open(os.path.join(out, tag + '.in'), 'wb').write(b'\0')
open(os.path.join(out, tag + '.angle.out'), 'wb').write(struct.pack('<%dI' % len(flat), *flat))
man.append({'tag': tag, 'glsl': f['glsl'], 'n': 256, 'nout': nout, 'kind': rt[2], 'tol': f['tolerance'], 'platform': f.get('platform')})
gl.glDeleteFramebuffers(1, C.byref(fbo)); gl.glDeleteTextures(nt, tos); gl.glDeleteTextures(1, C.byref(t_in)); gl.glDeleteProgram(prog)
print('gl error at end 0x%x' % gl.glGetError(), '; ran', len(man), 'skipped struct', skipped)
json.dump(man, open(os.path.join(out, 'manifest.json'), 'w'))
fam.cmp(fx, out, 'angle')

112
docs/research/native-shader-probes/fam.py Normal file
View file

@ -0,0 +1,112 @@
#!/usr/bin/env python3
"""Probe: per fixture family, generate an HLSL and a GLSL ES 3.10 compute shader that call the family's
function on the fixture's argument bits, and compare device results with the fixture's result bits.
usage: fam.py gen <fixture> <outdir> | fam.py cmp <fixture> <outdir> <tag>"""
import json, re, struct, sys, os
SCAL = {'float': ('float', 1, 'f'), 'int': ('int', 1, 'i'), 'uint': ('uint', 1, 'u')}
def ty(t):
if t in SCAL: return SCAL[t]
m = re.fullmatch(r'(i|u|)vec([234])', t)
if m:
k = {'': 'f', 'i': 'i', 'u': 'u'}[m.group(1)]; n = int(m.group(2))
return ({'f': 'float', 'i': 'int', 'u': 'uint'}[k] + str(n), n, k)
if t == 'mat3': return ('float3x3', 9, 'f')
return None
def parse(sig):
m = re.fullmatch(r'(\w+)\s+(\w+)\((.*)\)', sig)
ps = [p.strip() for p in m.group(3).split(',') if p.strip()]
return m.group(1), m.group(2), ps
def hlsl_read(k, words):
return {'f': 'asfloat', 'i': 'asint', 'u': ''}[k] + '(' + words + ')'
def gen(fx, out):
d = json.load(open(fx)); man = []; skipped = []
for idx, f in enumerate(d['functions']):
r, name, ps = parse(f['glsl'])
rt = ty(r); pts = [ty(p) for p in ps]
if rt is None or any(p is None for p in pts):
skipped.append(f['glsl']); continue
nin = sum(p[1] for p in pts); nout = rt[1]
tag = 'f%03d' % idx
words = []
for c in f['cases']:
a, b = c.split('->'); aw = a.split(); assert len(aw) == nin, (f['glsl'], len(aw), nin)
words += [int(x, 16) for x in aw]
open(os.path.join(out, tag + '.in'), 'wb').write(struct.pack('<%dI' % len(words), *words) if words else b'\0\0\0\0')
# HLSL
hl, gl, off = [], [], 0
for j, (p, (ht, n, k)) in enumerate(zip(ps, pts)):
idxs = ', '.join('In[b + %d]' % (off + t) for t in range(n))
gidx = ', '.join('In.a[b + %du]' % (off + t) for t in range(n))
if ht == 'float3x3': raise SystemExit('mat3 arg unsupported')
vec = ('uint%d(%s)' % (n, idxs)) if n > 1 else idxs
hl.append(' %s a%d = %s;' % (ht, j, hlsl_read(k, vec) if k != 'u' else vec))
gvec = ('uvec%d(%s)' % (n, gidx)) if n > 1 else gidx
gfun = {'f': 'uintBitsToFloat', 'i': ('ivec%d' % n if n > 1 else 'int'), 'u': ''}[k]
gl.append(' %s a%d = %s(%s);' % (p, j, gfun, gvec))
off += n
args = ', '.join('a%d' % j for j in range(len(ps)))
hl.append(' %s r = %s(%s);' % (rt[0], name, args))
gl.append(' %s r = %s(%s);' % (r, name, args))
if rt[0] == 'float3x3':
comps = ['r[%d][%d]' % (i, j) for i in range(3) for j in range(3)]
elif rt[1] == 1: comps = ['r']
else: comps = ['r[%d]' % i for i in range(rt[1])]
for i, c in enumerate(comps):
hl.append(' Out[o + %d] = %s;' % (i, {'f': 'asuint', 'i': 'asuint', 'u': ''}[rt[2]] + '(' + c + ')'))
gl.append(' Out.a[o + %du] = %s;' % (i, {'f': 'floatBitsToUint', 'i': 'uint', 'u': ''}[rt[2]] + '(' + c + ')'))
n = len(f['cases'])
open(os.path.join(out, tag + '.hlsl'), 'w').write(
'#include "CultMath.hlsl"\n[[vk::binding(0)]] StructuredBuffer<uint> In;\n[[vk::binding(1)]] RWStructuredBuffer<uint> Out;\n'
'[numthreads(64, 1, 1)] void main(uint3 id : SV_DispatchThreadID) {\n uint i = id.x; if (i >= %du) return; uint b = i * %du; uint o = i * %du;\n%s\n}\n'
% (n, nin, nout, '\n'.join(hl)))
open(os.path.join(out, tag + '.body.glsl'), 'w').write(
'layout(local_size_x = 64) in;\nlayout(std430, binding = 0) readonly buffer IB { uint a[]; } In;\nlayout(std430, binding = 1) buffer OB { uint a[]; } Out;\n'
'void main() {\n uint i = gl_GlobalInvocationID.x; if (i >= %du) return; uint b = i * %du; uint o = i * %du;\n%s\n}\n'
% (n, nin, nout, '\n'.join(gl)))
man.append({'tag': tag, 'glsl': f['glsl'], 'n': n, 'nout': nout, 'kind': rt[2], 'tol': f['tolerance'], 'platform': f.get('platform')})
json.dump(man, open(os.path.join(out, 'manifest.json'), 'w'))
print('generated', len(man), 'families; skipped (struct types)', len(skipped), ':', '; '.join(skipped))
def ordered(u): # float bits -> monotonic int for ulp distance
return u if u < 0x80000000 else 0x80000000 - u
def cmp(fx, out, tag):
d = json.load(open(fx)); man = json.load(open(os.path.join(out, 'manifest.json')))
rows = []; buckets = {'exact': 0, '<=4': 0, '<=64': 0, '>64': 0, 'missing': 0}
for m in man:
p = os.path.join(out, m['tag'] + '.' + tag + '.out')
if not os.path.exists(p): buckets['missing'] += 1; rows.append((m['glsl'], 'MISSING', '')); continue
got = struct.unpack('<%dI' % (m['n'] * m['nout']), open(p, 'rb').read())
f = d['functions'][int(m['tag'][1:])]
worst = 0; exact = 0; nanmis = 0
for ci, c in enumerate(f['cases']):
exp = [int(x, 16) for x in c.split('->')[1].split()]
g = got[ci * m['nout']:(ci + 1) * m['nout']]
if list(g) == exp: exact += 1; continue
for e, h in zip(exp, g):
if m['kind'] != 'f': worst = max(worst, 1 << 40); continue
en = (e & 0x7f800000) == 0x7f800000 and e & 0x7fffff; hn = (h & 0x7f800000) == 0x7f800000 and h & 0x7fffff
if en or hn:
if not (en and hn): nanmis += 1
continue
fe = struct.unpack('<f', struct.pack('<I', e))[0]; fh = struct.unpack('<f', struct.pack('<I', h))[0]
if fe != fe or fh != fh or abs(fe) == float('inf') or abs(fh) == float('inf'):
if not (fe == fh or (fe != fe and fh != fh)): nanmis += 1
continue
worst = max(worst, abs(fe - fh) / max(abs(fe), 1.0) / 2.0 ** -23)
b = 'exact' if exact == m['n'] else '<=4' if worst <= 4 and not nanmis else '<=64' if worst <= 64 and not nanmis else '>64'
if worst >= 1 << 40: b = '>64'
buckets[b] += 1
rows.append((m['glsl'], '%d/%d exact' % (exact, m['n']), 'max err %s ulp(max(|v|,1))%s' % (('%.1f' % worst) if worst < 1 << 40 else 'int-mismatch', ', nan-mismatch %d' % nanmis if nanmis else '')))
print(tag, 'buckets', buckets)
for r in sorted(rows, key=lambda r: r[1])[:0]: pass
rows.sort(key=lambda r: -float(re.search(r'max err ([0-9.]+)', r[2]).group(1)) if re.search(r'max err ([0-9.]+)', r[2]) else -1e18)
for r in rows[:int(os.environ.get('CMP_TOP', '25'))]:
if not r[1].startswith('256/256'): print(' ', r[0], '|', r[1], '|', r[2])
if __name__ == '__main__':
{'gen': lambda: gen(sys.argv[2], sys.argv[3]), 'cmp': lambda: cmp(sys.argv[2], sys.argv[3], sys.argv[4])}[sys.argv[1]]()

52
docs/research/native-shader-probes/lib.sh Normal file
View file

@ -0,0 +1,52 @@
#!/usr/bin/env bash
# Probe job, runs in ubuntu:24.04 with /src = CultLib at the probed revision. Q1 library path, Q2 lavapipe runs.
set -u
P=/tmp/p; W=/tmp/w; mkdir -p $W/fam $W/tools; cd $W
step() { echo; echo "=== $*"; }
step apt
export DEBIAN_FRONTEND=noninteractive
apt-get update -qq >/dev/null && apt-get install -y -qq --no-install-recommends ca-certificates curl git cmake g++ make python3 \
mesa-vulkan-drivers libvulkan-dev vulkan-tools glslang-tools spirv-tools >/dev/null 2>&1; echo "apt exit $?"
dpkg-query -W -f='${Package} ${Version}\n' mesa-vulkan-drivers glslang-tools spirv-tools libvulkan-dev
step dxc
url=$(curl -s https://api.github.com/repos/microsoft/DirectXShaderCompiler/releases/tags/v1.9.2607 | python3 -c "import json,sys;print([a['browser_download_url'] for a in json.load(sys.stdin)['assets'] if 'linux' in a['name'] and a['name'].endswith('.tar.gz')][0])")
echo "$url"; curl -sL --fail -o dxc.tgz "$url" && mkdir -p tools/dxc && tar xzf dxc.tgz -C tools/dxc; DXC=$(find tools/dxc -name dxc -type f | head -1); echo "DXC=$DXC"; $DXC --version 2>&1 | head -2
step spirv-cross
git clone -q --depth 1 https://github.com/KhronosGroup/SPIRV-Cross.git tools/sc && git -C tools/sc log -1 --format='SPIRV-Cross %H %cs' && \
cmake -S tools/sc -B tools/scb -DCMAKE_BUILD_TYPE=Release -DSPIRV_CROSS_ENABLE_TESTS=OFF >/dev/null && cmake --build tools/scb --target spirv-cross -j4 >/dev/null 2>&1; SC=tools/scb/spirv-cross; ls -la $SC | cut -c1-120
SH=/src/packages/cultmath/shaders
python3 - "$SH" <<'PY'
import re,sys,os,shutil
src=sys.argv[1]; dst='/tmp/w/shx'; shutil.copytree(src,dst,dirs_exist_ok=True); n=0
for fn in os.listdir(dst):
if fn.endswith('.hlsl'):
t=open(os.path.join(dst,fn)).read(); t2,k=re.subn(r'(?m)^(?!static|export)(\w+(?:\dx\d)?\s+cultmath_\w+\s*\(([^)]*)\))', lambda m: m.group(1) if re.search('Texture2D|SamplerState', m.group(2)) else 'export ' + m.group(1), t); k=len(re.findall(r'(?m)^export ', t2)); n+=k; open(os.path.join(dst,fn),'w').write(t2)
print('export keywords added', n)
PY
step "L1 dxc lib_6_3 universal1.5, export on non-texture defs"
$DXC -T lib_6_3 -HV 2021 -spirv -fspv-target-env=universal1.5 -I shx shx/CultMath.hlsl -Fo lib.spv 2>&1 | head -3; echo "exit ${PIPESTATUS[0]}"
spirv-dis lib.spv -o lib.spvasm; echo "exports $(grep -c 'LinkageAttributes.*Export' lib.spvasm); capabilities $(grep -c OpCapability lib.spvasm)"
step "L1b -default-linkage external, no export keyword"
$DXC -T lib_6_3 -HV 2021 -spirv -fspv-target-env=universal1.5 -default-linkage external -I $SH $SH/CultMath.hlsl -Fo libd.spv 2>&1 | head -3; echo "exit ${PIPESTATUS[0]}"
[ -s libd.spv ] && { spirv-dis libd.spv -o libd.spvasm; echo "exports $(grep -c 'LinkageAttributes.*Export' libd.spvasm)"; }
step "L2 strip every capability but Shader and Linkage, every extension; SPIRV-Cross ES 310 and 300"
grep -vP '^\s*OpCapability (?!Shader$|Linkage$)|^\s*OpExtension ' lib.spvasm > s.spvasm; spirv-as --target-env spv1.5 s.spvasm -o s.spv; echo "as exit $?"
for v in 310 300; do $SC s.spv --es --version $v --output s.$v.glsl 2>&1 | head -2; echo "es$v exit ${PIPESTATUS[0]}"; done
G=s.310.glsl; echo "G=$G"; [ -s "$G" ] || { echo PROBE-DONE; exit 0; }
step "L3 shape of $G"
echo "lines $(wc -l < $G); defs cultmath_: $(grep -cE '^\w+ cultmath_\w+\(' $G); distinct: $(grep -oE '^\w+ cultmath_\w+\(' $G | sed -E 's/^\w+ //' | sort -u | wc -l); renamed _N: $(grep -oE '^\w+ cultmath_\w+_[0-9]+\(' $G | wc -l); _NNN temporaries: $(grep -oE '\b_[0-9]+\b' $G | wc -l); non-cultmath defs: $(grep -cE '^\w+ (?!cultmath_)\w+\(' -P $G)"
grep -nE '^#|^layout|precision|^struct|^void main' $G | head -12
echo "--- renamed defs sample:"; grep -oE '^\w+ cultmath_\w+_[0-9]+\([^)]*\)' $G | head -6
echo "--- snoise(vec3):"; awk '/^(highp )?float cultmath_snoise\((highp )?vec3/{p=1} p{print; if(++n>=22) exit}' $G
sed -E '/^#version/d; /^layout\(local_size/d' $G | awk 'BEGIN{s=0} /^void main\(\)/{s=1} s==0{print} s==1&&/^\}/{s=0}' > lib.body.glsl
{ printf '#version 300 es\nprecision highp float;\nprecision highp int;\n'; cat lib.body.glsl; printf 'out highp vec4 c;\nvoid main() { c = vec4(cultmath_snoise(gl_FragCoord.xyz)); }\n'; } > es300.frag
glslangValidator es300.frag > es300.txt 2>&1; echo "glslang ES 3.00 fragment with the library: exit $? $(grep -m3 ERROR es300.txt | tr '\n' ' ' | cut -c1-300)"
step "L4 library through glslang on lavapipe"
gcc -O2 -o vkrun $P/vkrun.c -lvulkan 2>/dev/null; python3 $P/fam.py gen /src/packages/cultmath/tests/CultMath.Tests/fixtures/glsl-parity.json fam >/dev/null
okl=0; for b in fam/*.body.glsl; do t=${b%.body.glsl}; tag=$(basename $t); n=$(python3 -c "import json;m={x['tag']:x for x in json.load(open('fam/manifest.json'))}['$tag'];print(m['n'],m['nout'])")
{ printf '#version 310 es\nprecision highp float;\nprecision highp int;\n'; cat lib.body.glsl $b; } > $t.l.comp
glslangValidator -V --target-env vulkan1.2 -o $t.l.spv $t.l.comp >$t.l.log 2>&1 && ./vkrun $t.l.spv $t.in $t.lib.out $n && okl=$((okl+1)); done
echo "library path ran $okl of 105"; grep -h -m1 ERROR fam/*.l.log | sed -E 's/[0-9]+:/N:/g' | sort | uniq -c | sort -rn | head -6
CMP_TOP=8 python3 $P/fam.py cmp /src/packages/cultmath/tests/CultMath.Tests/fixtures/glsl-parity.json fam lib
echo "LIBGZ-BEGIN"; gzip -9c lib.body.glsl | base64 -w0; echo; echo "LIBGZ-END"
echo PROBE-DONE

53
docs/research/native-shader-probes/od-temporaries/byvalue.py Normal file
View file

@ -0,0 +1,53 @@
#!/usr/bin/env python3
"""Probe: by-value parameter lowering on a SPIR-V module. dxc -Od passes every parameter as a Function pointer and the
caller copies in; a callee that assigns its parameter then comes out of SPIRV-Cross as `inout`. This pass makes each
Function-pointer parameter a value parameter: the old param id becomes a local variable in the entry block, stored from the
new value parameter; each call loads the pointer it passed. usage: byvalue.py <in.spv> <out.spv>"""
import struct, sys
def load(p):
b = open(p, 'rb').read(); w = list(struct.unpack('<%dI' % (len(b) // 4), b)); ins = []; i = 5
while i < len(w): n = w[i] >> 16; ins.append(w[i:i + n]); i += n
return w[:5], ins
def op(x): return x[0] & 0xffff
def mk(o, *ws): return [((len(ws) + 1) << 16) | o, *ws]
h, ins = load(sys.argv[1]); bound = h[3]
ptr = {x[1]: (x[2], x[3]) for x in ins if op(x) == 32} # id -> (storage, pointee)
ftype = {x[1]: x for x in ins if op(x) == 33}
fnparams = {}; cur = None; fntype = {}
for x in ins: # analysis: per function, which params are Function pointers
if op(x) == 54: cur = x[2]; fnparams[cur] = []; fntype[cur] = x[4]
elif op(x) == 55 and cur is not None:
t = x[1]; conv = t in ptr and ptr[t][0] == 7
fnparams[cur].append((x[2], t, ptr[t][1] if conv else None, bound if conv else None)); bound += conv
newtype = {}; typeins = []; sig = {tuple(x[2:]): x[1] for x in ins if op(x) == 33}
for f, ps in fnparams.items():
if any(p[2] for p in ps):
ft = ftype[fntype[f]]; key = (ft[2], *[p[2] if p[2] else p[1] for p in ps])
if key not in sig: bound += 1; sig[key] = bound; typeins.append(mk(33, bound, *key))
newtype[f] = sig[key]
names = {x[1]: x for x in ins if op(x) == 5}
out = []; cur = None; pend = []; need_stores = False; first = True; pc = 0
for x in ins:
o = op(x)
if o == 54 and first: out += typeins; first = False
if need_stores and o != 59:
out += [mk(62, p, q) for p, _, q in pend]; pend = []; need_stores = False
if o == 54:
cur = x[2]; pc = 0
if cur in newtype: x = x[:4] + [newtype[cur]] + x[5:]
elif o == 55:
_, t, pointee, q = fnparams[cur][pc]; pc += 1
if pointee: out.append(mk(55, pointee, q)); pend.append((x[2], t, q)); continue
elif o == 248 and pend:
out.append(x); out += [mk(59, t, p, 7) for p, t, _ in pend]; need_stores = True; continue
elif o == 57:
callee = x[3]; args = x[4:]; new = []
for a, (_, _, pointee, _) in zip(args, fnparams[callee]):
if pointee: bound += 1; out.append(mk(61, pointee, bound, a)); new.append(bound)
else: new.append(a)
x = x[:4] + new; x[0] = (len(x) << 16) | 57
out.append(x)
if o == 5 and x[1] in [p[0] for ps in fnparams.values() for p in ps if p[2]]:
q = next(p[3] for ps in fnparams.values() for p in ps if p[0] == x[1]); out.append([x[0], q] + x[2:])
h[3] = bound + 1; w = h + [v for x in out for v in x]; open(sys.argv[2], 'wb').write(struct.pack('<%dI' % len(w), *w))
print('byvalue: %d functions changed, %d params' % (len(newtype), sum(1 for ps in fnparams.values() for p in ps if p[2])))

84
docs/research/native-shader-probes/od-temporaries/od.sh Normal file
View file

@ -0,0 +1,84 @@
#!/usr/bin/env bash
# Imagination probe: what dxc -Od leaves in the SPIRV-Cross GLSL and what cleans it. ubuntu:24.04, /src = CultLib, /cache persists tools.
set -u; P=/src/docs/research/native-shader-probes; O=$P/overload-wrappers; W=/tmp/w; mkdir -p $W /cache; cd $W
step() { echo; echo "=== $*"; }
export DEBIAN_FRONTEND=noninteractive
apt-get update -qq >/dev/null && apt-get install -y -qq --no-install-recommends ca-certificates curl git cmake g++ make python3 gcc \
mesa-vulkan-drivers libvulkan-dev glslang-tools spirv-tools >/dev/null 2>&1; echo "apt exit $?"
dpkg-query -W -f='${Package} ${Version}\n' spirv-tools glslang-tools mesa-vulkan-drivers
if [ ! -x /cache/sc/spirv-cross ]; then
git clone -q --depth 1 --branch vulkan-sdk-1.4.363.0 https://github.com/KhronosGroup/SPIRV-Cross.git /cache/scsrc
cmake -S /cache/scsrc -B /cache/scb -DCMAKE_BUILD_TYPE=Release -DSPIRV_CROSS_ENABLE_TESTS=OFF -DSPIRV_CROSS_CLI=ON >/dev/null && cmake --build /cache/scb --target spirv-cross -j8 >/dev/null 2>&1
mkdir -p /cache/sc; cp /cache/scb/spirv-cross /cache/sc/
fi
if [ ! -x /cache/dxc/bin/dxc ]; then
curl -sL --fail -o /cache/dxc.tgz https://github.com/microsoft/DirectXShaderCompiler/releases/download/v1.9.2607/linux_dxc_2026_07_29.x86_x64.tar.gz
mkdir -p /cache/dxc && tar xzf /cache/dxc.tgz -C /cache/dxc
fi
DXC=$(find /cache/dxc -name dxc -type f | head -1); SC=/cache/sc/spirv-cross; export LD_LIBRARY_PATH=$(dirname $DXC)/../lib:${LD_LIBRARY_PATH:-}
if [ ! -x /cache/st/spirv-opt ]; then
git clone -q --depth 1 --branch vulkan-sdk-1.4.363.0 https://github.com/KhronosGroup/SPIRV-Tools.git /cache/stsrc
git clone -q --depth 1 --branch vulkan-sdk-1.4.363.0 https://github.com/KhronosGroup/SPIRV-Headers.git /cache/stsrc/external/spirv-headers
cmake -S /cache/stsrc -B /cache/stb -DCMAKE_BUILD_TYPE=Release -DSPIRV_SKIP_TESTS=ON -DSPIRV_WERROR=OFF >/dev/null && cmake --build /cache/stb --target spirv-opt spirv-val spirv-dis spirv-as -j8 >/dev/null 2>&1
mkdir -p /cache/st; cp /cache/stb/tools/spirv-* /cache/st/
fi
export PATH=/cache/st:$PATH; echo "spirv-opt from $(which spirv-opt)"
echo "dxc $DXC"; $DXC --version 2>&1|head -2; spirv-opt --version | head -1
SH=/src/packages/cultmath/shaders; FX=/src/packages/cultmath/tests/CultMath.Tests/fixtures/glsl-parity.json
step "1 build the -Od library as in Pass 9b"
python3 $O/wrapgen.py $SH shw
$DXC -T lib_6_3 -HV 2021 -spirv -fspv-target-env=universal1.5 -Od -I shw shw/cm_wrappers.hlsl -Fo w.spv 2>&1 | head -3
python3 $O/spvedit.py edit w.spv e.spv shw/wrappers.tsv export; spirv-val --target-env spv1.5 e.spv && echo "val ok"
nf() { spirv-dis "$1" | grep -c ' = OpFunction '; }; echo "functions in e.spv: $(nf e.spv)"
step "2 variants: by-value lowering, then spirv-opt (never an inliner)"
cp e.spv v0.spv
python3 $P/od-temporaries/byvalue.py e.spv b0.spv; spirv-val --target-env spv1.5 b0.spv && echo "b0 val ok"
spirv-opt --target-env=spv1.5 --ssa-rewrite --eliminate-dead-code-aggressive --simplify-instructions e.spv -o v2.spv
SSA="--ssa-rewrite --eliminate-dead-code-aggressive"
declare -A VP=([b1]="--eliminate-local-single-store --eliminate-dead-code-aggressive" [b2]="$SSA" [b3]="$SSA --simplify-instructions" [b4]="--eliminate-local-single-block --eliminate-local-single-store --eliminate-local-multi-store --eliminate-dead-code-aggressive --merge-blocks")
for v in b1 b2 b3 b4; do spirv-opt --target-env=spv1.5 ${VP[$v]} b0.spv -o $v.spv 2>$v.err; echo "$v opt exit $? functions $(nf $v.spv 2>/dev/null) $(head -c 160 $v.err)"; done
cp b1.spv b1r.spv; VS="v0 b0 b1 b1r b3"
for v in $VS; do [ -s $v.spv ] || continue
spirv-val --target-env spv1.5 $v.spv >$v.val 2>&1; vx=$?
$SC $v.spv --es --version 310 $([ $v = b1r ] && echo --relax-nan-checks) --output $v.glsl >$v.sc 2>&1; sx=$?
[ -s $v.glsl ] || { echo "$v val $vx sc exit $sx"; head -3 $v.val $v.sc; continue; }
python3 - $v $vx $sx <<'PY'
import re,sys
v,vx,sx=sys.argv[1:4]; g=open(f'/tmp/w/{v}.glsl').read()
defs=re.findall(r'(?m)^(?:\w+ )*?(\w+) (cultmath_\w+)\(([^)]*)\)\n\{',g)
ids=re.findall(r'\b_\d+\b',g)
print(f'{v}: val={vx} sc={sx} lines={g.count(chr(10))} bytes={len(g)} cultmath_defs={len(defs)} distinct={len(set(d[1] for d in defs))} id_tokens={len(ids)} distinct_ids={len(set(ids))} param_var={g.count("param_var_")} inout={len(re.findall(r"inout",g))} suffixed={len(re.findall(r"cultmath_\w+_\d+\(",g))}')
PY
done
mkdir -p /cache/out; cp *.glsl *.spv /cache/out/ 2>/dev/null
step "3 SPIRV-Cross options"
for opt in "--remove-unused-variables"; do for v in v0 b2; do $SC $v.spv --es --version 310 $opt --output $v.o.glsl >/dev/null 2>&1; echo "$v + $opt: lines $(wc -l < $v.o.glsl)"; done; done
step "4 lerp(vec3) as emitted: v0 vs b2, vs committed"
for v in v0 b2; do echo "--- $v lerp(vec3)"; awk '/^(highp )?vec3 cultmath_lerp\((highp )?vec3/{p=1} p{print} p&&/^}/{exit}' $v.glsl; done
echo "--- committed lerp(vec3)"; grep -n -A3 -E '^(highp )?vec3 cultmath_lerp\(' $SH/CultMath.glsl | head -6
for v in v0 b2; do echo "--- $v snoise(vec3) head"; awk '/^(highp )?float cultmath_snoise\((highp )?vec3/{p=1} p{print; if(++n>=40) exit}' $v.glsl; done
step "5 conformance on lavapipe, every family, per variant vs committed GLSL and HLSL"
mkdir -p fam; gcc -O2 -o vkrun $P/vkrun.c -lvulkan && python3 $P/fam.py gen $FX fam >/dev/null
cat $SH/CultMath.glsl $SH/CultMath.Cycles.glsl $SH/CultMath.Phacelle.glsl > all.glsl
body() { sed -E '/^#version/d; /^#ifdef SPIRV_CROSS/d; /^#endif/d; /^layout\(local_size/d' $1 | awk 'BEGIN{s=0} /^void main\(\)/{s=1} s==0{print} s==1&&/^\}/{s=0}'; }
for v in $VS; do [ -s $v.glsl ] && body $v.glsl > $v.body; done
run() { local t=fam/$1 n; n=$(python3 -c "import json;m={x['tag']:x for x in json.load(open('fam/manifest.json'))}['$1'];print(m['n'],m['nout'])")
{ printf '#version 310 es\nprecision highp float;\nprecision highp int;\n'; cat $3 $t.body.glsl; } > $t.$2.comp
glslangValidator -V --target-env vulkan1.2 -o $t.$2.spv $t.$2.comp >$t.$2.log 2>&1 && ./vkrun $t.$2.spv $t.in $t.$2.out $n; }
for b in fam/*.body.glsl; do tag=$(basename ${b%.body.glsl})
n=$(python3 -c "import json;m={x['tag']:x for x in json.load(open('fam/manifest.json'))}['$tag'];print(m['n'],m['nout'])")
$DXC -T cs_6_0 -E main -HV 2021 -spirv -fspv-target-env=vulkan1.2 -I $SH fam/$tag.hlsl -Fo fam/$tag.h.spv >/dev/null 2>&1 && ./vkrun fam/$tag.h.spv fam/$tag.in fam/$tag.h.out $n
run $tag g all.glsl; for v in $VS; do [ -s $v.body ] && run $tag $v $v.body; done; done
export VS; python3 - <<'PY'
import os,filecmp
d='/tmp/w/fam'; tags=sorted(f[:-10] for f in os.listdir(d) if f.endswith('.body.glsl'))
def same(a,b): return [t for t in tags if os.path.exists(f'{d}/{t}.{a}.out') and os.path.exists(f'{d}/{t}.{b}.out') and filecmp.cmp(f'{d}/{t}.{a}.out',f'{d}/{t}.{b}.out',shallow=False)]
print('families',len(tags),'hlsl ran',len([t for t in tags if os.path.exists(f'{d}/{t}.h.out')]),'committed glsl ran',len([t for t in tags if os.path.exists(f'{d}/{t}.g.out')]))
for v in os.environ['VS'].split():
ran=[t for t in tags if os.path.exists(f'{d}/{t}.{v}.out')]
sg=same(v,'g'); print(v,'ran',len(ran),'bit-identical to committed glsl',len(sg),'to hlsl',len(same(v,'h')),'| differs from g:',[t for t in ran if t not in sg][:8])
print('committed vs hlsl',len(same('g','h')),'of',len(tags))
PY
for v in v0 b2; do { printf '#version 300 es\nprecision highp float;\nprecision highp int;\n'; cat $v.body; printf 'out highp vec4 c;\nvoid main() { vec3 p = gl_FragCoord.xyz; c = vec4(cultmath_lerp(p, p.yzx, vec3(0.5)), cultmath_lerp(1.0, 2.0, p.x)) + cultmath_saturate(vec4(p, 1.0)) + vec4(cultmath_snoise(p)); }\n'; } > $v.es300.frag; glslangValidator $v.es300.frag >$v.es300.txt 2>&1; echo "$v ES 3.00 glslang exit $? $(grep -m2 ERROR $v.es300.txt | tr '\n' ' ' | cut -c1-200)"; done
for t in g $VS; do echo "--- cmp vs C# fixture: $t"; CMP_TOP=3 python3 $P/fam.py cmp $FX fam $t 2>&1 | grep -v "^ *$" | head -8; done
echo PROBE-DONE

29
docs/research/native-shader-probes/overload-wrappers/lava.sh Normal file
View file

@ -0,0 +1,29 @@
#!/usr/bin/env bash
# Imagination probe, lavapipe leg: every fixture family through three routes on lavapipe.
# h = HLSL by dxc cs_6_0, g = committed GLSL by glslang, r = restored overloaded library GLSL (lib.body.glsl) by glslang.
set -u; P=/tmp/p; W=/tmp/w; cd $W; SH=/src/packages/cultmath/shaders; FX=/src/packages/cultmath/tests/CultMath.Tests/fixtures/glsl-parity.json
echo; echo "=== 2b lavapipe: same computation"
mkdir -p fam; gcc -O2 -o vkrun $P/vkrun.c -lvulkan && python3 $P/fam.py gen $FX fam >/dev/null
cat $SH/CultMath.glsl $SH/CultMath.Cycles.glsl $SH/CultMath.Phacelle.glsl > all.glsl
ok=(0 0 0); first=1
for b in fam/*.body.glsl; do t=${b%.body.glsl}; tag=$(basename $t)
n=$(python3 -c "import json;m={x['tag']:x for x in json.load(open('fam/manifest.json'))}['$tag'];print(m['n'],m['nout'])")
$DXC_PATH -T cs_6_0 -E main -HV 2021 -spirv -fspv-target-env=vulkan1.2 -I $SH $t.hlsl -Fo $t.h.spv >$t.h.log 2>&1 && ./vkrun $t.h.spv $t.in $t.h.out $n && ok[0]=$((ok[0]+1))
for r in g r; do src=all.glsl; [ $r = r ] && src=lib.body.glsl
{ printf '#version 310 es\nprecision highp float;\nprecision highp int;\n'; cat $src $b; } > $t.$r.comp
glslangValidator -V --target-env vulkan1.2 -o $t.$r.spv $t.$r.comp >$t.$r.log 2>&1 && ./vkrun $t.$r.spv $t.in $t.$r.out $n && { [ $r = g ] && ok[1]=$((ok[1]+1)) || ok[2]=$((ok[2]+1)); }
done
done
echo "ran on lavapipe: hlsl ${ok[0]}, committed glsl ${ok[1]}, restored glsl ${ok[2]} (of $(ls fam/*.body.glsl | wc -l))"
grep -h -m1 ERROR fam/*.r.log | sed -E 's/[0-9]+:/N:/g' | sort | uniq -c | sort -rn | head -5
python3 - <<'PY'
import os,filecmp
d='/tmp/w/fam'; tags=sorted(f[:-10] for f in os.listdir(d) if f.endswith('.body.glsl'))
def same(a,b): return [t for t in tags if os.path.exists(f'{d}/{t}.{a}.out') and os.path.exists(f'{d}/{t}.{b}.out') and filecmp.cmp(f'{d}/{t}.{a}.out',f'{d}/{t}.{b}.out',shallow=False)]
rg=same('r','g'); rh=same('r','h'); gh=same('g','h')
print('bit-identical: restored vs committed glsl',len(rg),'| restored vs hlsl',len(rh),'| committed glsl vs hlsl',len(gh),'of',len(tags))
print('restored differs from committed glsl in:',[t for t in tags if t not in rg][:12])
print('restored differs from hlsl but committed glsl matches hlsl:',[t for t in gh if t not in rh][:12])
PY
CMP_TOP=10 python3 $P/fam.py cmp $FX fam r 2>&1 | head -16

39
docs/research/native-shader-probes/overload-wrappers/split.py Normal file
View file

@ -0,0 +1,39 @@
import re,sys,os,collections
sh,g,out=sys.argv[1:4]; os.makedirs(out,exist_ok=True)
origin={}; owner={'CultMath.hlsl':'core','CultMath.Interval.hlsl':'core','CultMath.Affine.hlsl':'core','CultMath.Cycles.hlsl':'cycles','CultMath.Phacelle.hlsl':'phacelle'}
for fn,o in owner.items():
t=open(os.path.join(sh,fn)).read()
for m in re.finditer(r'(?m)^(?:export\s+)?\w+\s+(cultmath_\w+)\s*\(',t): origin.setdefault(m.group(1),o)
for m in re.finditer(r'(?m)^struct\s+(\w+)',t): origin.setdefault(m.group(1),o)
text=open(g).read()
# top-level chunks: struct blocks and function definitions; everything else is prologue
lines=text.split('\n'); chunks=[]; i=0; pro=[]
while i<len(lines):
l=lines[i]
m=re.match(r'^struct (\w+)$',l) or re.match(r'^\w+ (\w+)\(.*\)$',l)
if m and m.group(1)!='main':
j=i
while lines[j]!='};' and lines[j]!='}': j+=1
chunks.append((m.group(1),'\n'.join(lines[i:j+1]))); i=j+1
elif m:
j=i
while lines[j]!='}': j+=1
i=j+1
else:
if l.strip(): pro.append(l)
i+=1
print('prologue lines',len(pro),':',' | '.join(pro[:6])[:300])
cnt=collections.Counter(); unknown=[]; bad=collections.Counter(); files=collections.defaultdict(list)
rank={'core':0,'cycles':1,'phacelle':1}
for name,body in chunks:
o=origin.get(name)
if o is None: unknown.append(name); o='core'
cnt[o]+=1; files[o].append(body)
for callee in set(re.findall(r'\b(cultmath_\w+|[A-Z]\w+)\b',body))-{name}:
co=origin.get(callee)
if co and co!=o and not (rank[co]==0): bad[(o,co,name,callee)]+=1
print('chunks',len(chunks),dict(cnt),'unknown-origin',len(unknown),unknown[:8])
print('cross-licence calls (non-core callee):',len(bad),list(bad)[:6])
seen=collections.Counter(n for n,_ in chunks); print('duplicate chunk names',[n for n,c in seen.items() if c>1][:6])
for o,b in files.items(): open(os.path.join(out,o+'.glsl'),'w').write('\n'.join(b)+'\n')
print('split sizes', {o:len('\n'.join(b).split('\n')) for o,b in files.items()})

71
docs/research/native-shader-probes/overload-wrappers/spvedit.py Normal file
View file

@ -0,0 +1,71 @@
#!/usr/bin/env python3
"""Probe: instruction-level SPIR-V edit (binary, no text regex).
stats <spv> <wrappers.tsv> how each wrapper looks after dxc
edit <in> <out> <wrappers.tsv> <mode> mode drop: strip caps (keep Shader, Linkage) and extensions, delete
wrapper functions and their names/decorations, set each callee's OpName to the overloaded source name.
mode export: as drop, but each wrapper's Export decoration moves to its callee instead of being deleted."""
import struct, sys, collections
def load(p):
b = open(p, 'rb').read(); w = list(struct.unpack('<%dI' % (len(b) // 4), b)); ins = []; i = 5
while i < len(w): n = w[i] >> 16; ins.append(w[i:i + n]); i += n
return w[:5], ins
def s_dec(ws):
b = struct.pack('<%dI' % len(ws), *ws); return b[:b.index(0)].decode()
def s_enc(s):
b = s.encode() + b'\0'; b += b'\0' * (-len(b) % 4); return list(struct.unpack('<%dI' % (len(b) // 4), b))
def op(x): return x[0] & 0xffff
def funcs(ins):
fs = {}; cur = None
for k, x in enumerate(ins):
if op(x) == 54: cur = x[2]; fs[cur] = [k, None, [], set()]
if cur is not None:
if op(x) == 57: fs[cur][2].append(x[3])
o = op(x)
if o == 248: fs[cur][3].add(x[1])
elif o not in (8, 317, 62, 246, 247, 249, 250, 253, 254, 252) and len(x) > 2: fs[cur][3].add(x[2])
if op(x) == 56: fs[cur][1] = k; cur = None
return fs
def names(ins): return {x[1]: s_dec(x[2:]) for x in ins if op(x) == 5}
def wrappers(tsv): return [l.split('\t') for l in open(tsv).read().splitlines() if l]
def analyse(ins, tsv):
nm = names(ins); fs = funcs(ins); byname = {v: k for k, v in nm.items() if k in fs}; res = []
for wn, name, fn, ps in wrappers(tsv):
fid = byname.get(wn); callees = fs[fid][2] if fid else None
res.append((wn, name, fid, callees))
return nm, fs, res
if sys.argv[1] == 'stats':
h, ins = load(sys.argv[2]); nm, fs, res = analyse(ins, sys.argv[3])
missing = [r[0] for r in res if r[2] is None]; inl = [r[0] for r in res if r[2] and not r[3]]
multi = [r[0] for r in res if r[3] and len(r[3]) != 1]
callee = collections.Counter(r[3][0] for r in res if r[3] and len(r[3]) == 1)
wrong = [(r[0], nm.get(r[3][0])) for r in res if r[3] and len(r[3]) == 1 and nm.get(r[3][0]) != r[1]]
print('functions %d, wrappers found %d of %d, missing %d, inlined (no call) %d, not-one-call %d' % (len(fs), len(res) - len(missing), len(res), len(missing), len(inl), len(multi)))
print('distinct callees %d, callee shared by >1 wrapper %d' % (len(callee), sum(1 for c in callee.values() if c > 1)))
print('callee OpName != source name: %d %s' % (len(wrong), wrong[:4]))
calls = sum(len(f[2]) for f in fs.values()); print('total OpFunctionCall %d; non-wrapper functions %d' % (calls, len(fs) - len(res) + len(missing)))
print('sample', [(r[0], nm.get(r[3][0]) if r[3] else None) for r in res[:3]], 'missing sample', missing[:3], 'inlined sample', inl[:3])
sys.exit(0)
_, src, dst, tsv, mode = sys.argv[1:6]
h, ins = load(src); nm, fs, res = analyse(ins, tsv)
drop_k = set(); drop_ids = set(); setname = {}; retarget = {}
for wn, name, fid, callees in res:
if fid is None or not callees or len(callees) != 1: sys.exit('refuse: wrapper %s is not one call' % wn)
a, b, _, ids = fs[fid]; drop_k.update(range(a, b + 1)); drop_ids |= ids | {fid}
setname[callees[0]] = name; retarget[fid] = callees[0]
drop_ids |= set(retarget)
outi = []; named = set()
for k, x in enumerate(ins):
if k in drop_k: continue
o = op(x)
if o == 17 and x[1] not in (1, 5): continue
if o == 10: continue
if o == 5 and x[1] in setname: x = [0] + [x[1]] + s_enc(setname[x[1]]); x[0] = (len(x) << 16) | 5; named.add(x[1])
elif o in (5, 6) and x[1] in drop_ids: continue
if o == 71 and x[1] in drop_ids:
if mode == 'export' and x[1] in retarget and x[2] == 41: x = [x[0], retarget[x[1]]] + x[2:]
else: continue
outi.append(x)
for fid, name in setname.items():
if fid not in named: outi.insert(next(i for i, x in enumerate(outi) if op(x) == 5), [((2 + len(s_enc(name))) << 16) | 5, fid] + s_enc(name))
w = h + [v for x in outi for v in x]; open(dst, 'wb').write(struct.pack('<%dI' % len(w), *w))
print('edit %s: dropped %d wrapper functions, renamed %d callees, kept %d instructions' % (mode, len(res), len(setname), len(outi)))

50
docs/research/native-shader-probes/overload-wrappers/wrap.sh Normal file
View file

@ -0,0 +1,50 @@
#!/usr/bin/env bash
# Imagination probe: build-only export wrappers -> dxc lib_6_3 -spirv -> SPIR-V edit -> SPIRV-Cross ES. /src = CultLib.
set -u; P=/tmp/p; W=/tmp/w; mkdir -p $W/tools; cd $W
step() { echo; echo "=== $*"; }
export DEBIAN_FRONTEND=noninteractive
apt-get update -qq >/dev/null && apt-get install -y -qq --no-install-recommends ca-certificates curl git cmake g++ make python3 \
mesa-vulkan-drivers libvulkan-dev glslang-tools spirv-tools >/dev/null 2>&1; echo "apt exit $?"
curl -sL --fail -o dxc.tgz https://github.com/microsoft/DirectXShaderCompiler/releases/download/v1.9.2607/linux_dxc_2026_07_29.x86_x64.tar.gz
echo "dxc sha ok: $(sha256sum dxc.tgz | cut -c1-64 | grep -c 55665c87824051ed4774ff3280a79ccbbb7d39243b9736ca5e98222134112d54)"
mkdir -p tools/dxc && tar xzf dxc.tgz -C tools/dxc; DXC=$(find tools/dxc -name dxc -type f | head -1)
git clone -q --depth 1 --branch vulkan-sdk-1.4.363.0 https://github.com/KhronosGroup/SPIRV-Cross.git tools/sc
cmake -S tools/sc -B tools/scb -DCMAKE_BUILD_TYPE=Release -DSPIRV_CROSS_ENABLE_TESTS=OFF -DSPIRV_CROSS_CLI=ON >/dev/null && cmake --build tools/scb --target spirv-cross -j4 >/dev/null 2>&1; SC=tools/scb/spirv-cross; ls $SC >/dev/null && echo "spirv-cross built"
SH=/src/packages/cultmath/shaders
step "1 wrappers"
python3 $P/wrapgen.py $SH shw; head -3 shw/cm_wrappers.hlsl | cut -c1-160; grep -c '' shw/wrappers.tsv
echo "source defs per overloaded name: $(cut -f2 shw/wrappers.tsv | sort | uniq -c | awk '$1>1{n++; d+=$1} END{print n" names, "d" defs"}')"
step "2 dxc lib_6_3 on source plus wrappers"
for v in default Od; do f=""; [ $v = Od ] && f="-Od"
$DXC -T lib_6_3 -HV 2021 -spirv -fspv-target-env=universal1.5 $f -I shw shw/cm_wrappers.hlsl -Fo w.$v.spv 2>&1 | head -3; echo "[$v] dxc exit ${PIPESTATUS[0]}"
[ -s w.$v.spv ] && python3 $P/spvedit.py stats w.$v.spv shw/wrappers.tsv; done
V=default; python3 $P/spvedit.py stats w.default.spv shw/wrappers.tsv | grep -q 'inlined (no call) 0,' || V=Od; echo "using variant $V"
step "3 SPIR-V edit, validate, SPIRV-Cross ES 310"
for m in drop export; do python3 $P/spvedit.py edit w.$V.spv e.$m.spv shw/wrappers.tsv $m
spirv-val --target-env spv1.5 e.$m.spv 2>&1 | head -2; echo "[$m] val exit ${PIPESTATUS[0]}"
$SC e.$m.spv --es --version 310 --output e.$m.glsl 2>&1 | head -2; echo "[$m] spirv-cross exit ${PIPESTATUS[0]}, lines $(wc -l < e.$m.glsl 2>/dev/null)"
spirv-dis e.$m.spv | grep -E 'OpCapability|OpExtension' | tr -s ' ' | sort | uniq -c | tr '\n' ' '; echo; done
cmp -s e.drop.glsl e.export.glsl && echo "drop and export outputs identical" || echo "drop and export outputs DIFFER"
for m in drop export; do G=e.$m.glsl; [ -s $G ] || continue; python3 - $G shw/wrappers.tsv <<'PY'
import re,sys,collections
g=open(sys.argv[1]).read(); rows=[l.split('\t') for l in open(sys.argv[2]).read().splitlines() if l]
src=collections.Counter(r[1] for r in rows)
defs=re.findall(r'(?m)^(?:\w+ )*?(\w+) (\w+)\(([^)]*)\)\n\{',g); out=collections.Counter(d[1] for d in defs)
print(sys.argv[1],'defs',len(defs),'cultmath_ defs',sum(v for k,v in out.items() if k.startswith('cultmath_')),'source defs',sum(src.values()))
print(' names whose def count differs from source:',[(k,src[k],out.get(k,0)) for k in src if src[k]!=out.get(k,0)][:8])
print(' non-source names emitted:',[k for k in out if k not in src][:10],' cmw in text:',g.count('cmw'),' suffixed _N cultmath names:',len(re.findall(r'\bcultmath_\w+_\d+\(',g)))
print(' prologue:',' | '.join(l for l in g.split('\n')[:6] if l.startswith('#'))[:200], ' main:', g.count('void main'))
PY
done
G=e.export.glsl; echo "--- cultmath_lerp and cultmath_saturate as emitted:"; grep -E '^\w+ cultmath_(lerp|saturate)\(' $G
step "3b glslang ES 3.00 fragment, call sites resolve per overload"
sed -E '/^#version/d; /^#ifdef SPIRV_CROSS/d; /^#endif/d; /^layout\(local_size/d' $G | awk 'BEGIN{s=0} /^void main\(\)/{s=1} s==0{print} s==1&&/^\}/{s=0}' > lib.body.glsl
{ printf '#version 300 es\nprecision highp float;\nprecision highp int;\n'; cat lib.body.glsl; printf 'out highp vec4 c;\nvoid main() { vec3 p = gl_FragCoord.xyz; float a = cultmath_lerp(1.0, 2.0, p.x); vec3 b = cultmath_lerp(p, p.yzx, vec3(0.5)); vec4 s = cultmath_saturate(vec4(p, a)); c = vec4(b, cultmath_saturate(a)) + s + vec4(cultmath_snoise(p)); }\n'; } > es300.frag
glslangValidator es300.frag > es300.txt 2>&1; echo "glslang ES 3.00 fragment: exit $? $(grep -m3 ERROR es300.txt | tr '\n' ' ' | cut -c1-300)"
sed -i 's/cultmath_saturate(a))/cultmath_saturate(ivec2(1)))/' es300.frag; glslangValidator es300.frag > neg.txt 2>&1; echo "negative (no ivec2 overload) refused: exit $? $(grep -m1 ERROR neg.txt | cut -c1-160)"
step "4 split, constants, capability strip"
python3 $P/split.py $SH $G $W/split 2>&1 | head -6
for combo in "core" "core cycles" "core phacelle" "core cycles phacelle"; do f=$W/combo.frag; { printf '#version 300 es\nprecision highp float;\nprecision highp int;\n'; for c in $combo; do cat $W/split/$c.glsl; done; printf 'out highp vec4 c;\nvoid main() { c = vec4(cultmath_snoise(gl_FragCoord.xyz)) + vec4(cultmath_lerp(gl_FragCoord.xyz, vec3(1.0), vec3(0.5)), 1.0); }\n'; } > $f; glslangValidator $f > $W/c.txt 2>&1; echo "glslang [$combo]: exit $? $(grep -m2 ERROR $W/c.txt | tr '\n' ' ' | cut -c1-200)"; done
echo "static const lines: $(grep -hcE '^static const' $SH/*.hlsl | awk '{s+=$1}END{print s}'); of form 'static const float CULTMATH_X = lit;': $(grep -hcE '^static const float CULTMATH_\w+ = [-0-9.eEfx]+;' $SH/*.hlsl | awk '{s+=$1}END{print s}'); CULTMATH_ in output $(grep -c 'CULTMATH_' $G); in committed glsl $(cat $SH/*.glsl | grep -cE '^(const )?\w* ?float CULTMATH_')"
DXC_PATH=$DXC bash $P/lava.sh
echo PROBE-DONE

22
docs/research/native-shader-probes/overload-wrappers/wrapgen.py Normal file
View file

@ -0,0 +1,22 @@
#!/usr/bin/env python3
"""Probe: build-only export wrappers. One exported, uniquely named wrapper per top-level cultmath_ definition
(texture-taking ones skipped). Source is copied unchanged. usage: wrapgen.py <shaders> <outdir>
Writes <outdir>/cm_wrappers.hlsl and <outdir>/wrappers.tsv (wrapper, name, file, params)."""
import os, re, sys, shutil
sh, out = sys.argv[1:3]; shutil.copytree(sh, out, dirs_exist_ok=True)
SIG = re.compile(r'(?m)^(\w+)\s+(cultmath_\w+)\s*\(([^)]*)\)')
files = ['CultMath.hlsl', 'CultMath.Interval.hlsl', 'CultMath.Affine.hlsl', 'CultMath.Cycles.hlsl', 'CultMath.Phacelle.hlsl']
w = ['#include "CultMath.hlsl"']; rows = []; k = 0
for fn in files:
for m in SIG.finditer(open(os.path.join(sh, fn)).read()):
ret, name, ps = m.group(1), m.group(2), m.group(3)
if re.search(r'Texture|Sampler', ps): continue
params = [p.strip() for p in ps.split(',') if p.strip()]
args = [re.split(r'\s+', p)[-1] for p in params]
wn = 'cmw%03d_%s' % (k, name); k += 1
call = '%s(%s)' % (name, ', '.join(args))
w.append('export %s %s(%s) { %s%s; }' % (ret, wn, ', '.join(params), '' if ret == 'void' else 'return ', call))
rows.append('\t'.join([wn, name, fn, ', '.join(params)]))
open(os.path.join(out, 'cm_wrappers.hlsl'), 'w').write('\n'.join(w) + '\n')
open(os.path.join(out, 'wrappers.tsv'), 'w').write('\n'.join(rows) + '\n')
print('wrappers', k)

81
docs/research/native-shader-probes/probe.sh Normal file
View file

@ -0,0 +1,81 @@
#!/usr/bin/env bash
# Probe job, runs in ubuntu:24.04 with /src = CultLib at the probed revision. Q1 library path, Q2 lavapipe runs.
set -u
P=/tmp/p; W=/tmp/w; mkdir -p $W/fam $W/tools; cd $W
step() { echo; echo "=== $*"; }
step apt
export DEBIAN_FRONTEND=noninteractive
apt-get update -qq >/dev/null && apt-get install -y -qq --no-install-recommends ca-certificates curl git cmake g++ make python3 \
mesa-vulkan-drivers libvulkan-dev vulkan-tools glslang-tools spirv-tools >/dev/null 2>&1; echo "apt exit $?"
dpkg-query -W -f='${Package} ${Version}\n' mesa-vulkan-drivers glslang-tools spirv-tools libvulkan-dev
step dxc
url=$(curl -s https://api.github.com/repos/microsoft/DirectXShaderCompiler/releases/tags/v1.9.2607 | python3 -c "import json,sys;print([a['browser_download_url'] for a in json.load(sys.stdin)['assets'] if 'linux' in a['name'] and a['name'].endswith('.tar.gz')][0])")
echo "$url"; curl -sL --fail -o dxc.tgz "$url" && mkdir -p tools/dxc && tar xzf dxc.tgz -C tools/dxc; DXC=$(find tools/dxc -name dxc -type f | head -1); echo "DXC=$DXC"; $DXC --version 2>&1 | head -2
step spirv-cross
git clone -q --depth 1 https://github.com/KhronosGroup/SPIRV-Cross.git tools/sc && git -C tools/sc log -1 --format='SPIRV-Cross %H %cs' && \
cmake -S tools/sc -B tools/scb -DCMAKE_BUILD_TYPE=Release -DSPIRV_CROSS_ENABLE_TESTS=OFF >/dev/null && cmake --build tools/scb --target spirv-cross -j4 >/dev/null 2>&1; SC=tools/scb/spirv-cross; ls -la $SC | cut -c1-120
SH=/src/packages/cultmath/shaders
step "Q1a dxc library SPIR-V, no wrapper (lib_6_6, vulkan1.2)"
$DXC -T lib_6_6 -HV 2021 -spirv -fspv-target-env=vulkan1.2 -I $SH $SH/CultMath.hlsl -Fo lib.spv 2>&1 | head -8; echo "dxc exit ${PIPESTATUS[0]}"
[ -f lib.spv ] && { spirv-dis lib.spv -o lib.spvasm; echo "functions: $(grep -c ' = OpFunction ' lib.spvasm) exports: $(grep -c 'LinkageAttributes.*Export' lib.spvasm) entrypoints: $(grep -c OpEntryPoint lib.spvasm)"; }
step "Q1b dxc library with 'export' added to every top-level cultmath_ definition"
python3 - "$SH" <<'PY'
import re,sys,os,shutil
src=sys.argv[1]; dst='/tmp/w/shx'; shutil.copytree(src,dst,dirs_exist_ok=True)
n=0
for fn in os.listdir(dst):
if not fn.endswith('.hlsl'): continue
t=open(os.path.join(dst,fn)).read()
t2,k=re.subn(r'(?m)^(?!static|export)(\w+(?:\dx\d)?\s+cultmath_\w+\s*\()', r'export \1', t); n+=k
open(os.path.join(dst,fn),'w').write(t2)
print('export keywords added', n)
PY
for v in "universal1.5" "vulkan1.2 -Vd" "vulkan1.3 -Vd"; do echo "--- variant: -fspv-target-env=$v"; rm -f libx.spv
$DXC -T lib_6_6 -HV 2021 -spirv -fspv-target-env=$v -I shx shx/CultMath.hlsl -Fo libx.spv 2>&1 | head -4; echo "dxc exit ${PIPESTATUS[0]}"
[ -s libx.spv ] && { spirv-dis libx.spv -o libx.spvasm; echo "functions: $(grep -c ' = OpFunction ' libx.spvasm) exports: $(grep -c 'LinkageAttributes.*Export' libx.spvasm) OpName cultmath_: $(grep -c 'OpName.*"cultmath_' libx.spvasm) Fma: $(grep -c ' Fma ' libx.spvasm) NoContraction: $(grep -c NoContraction libx.spvasm)"; spirv-val --target-env vulkan1.2 libx.spv 2>&1 | head -2; break; }
done
LIB=libx.spv
step "Q1c SPIRV-Cross ES output of $LIB"
for v in 300 310; do $SC $LIB --es --version $v --output lib$v.glsl 2>&1 | head -3; echo "spirv-cross --es --version $v exit ${PIPESTATUS[0]}"; done
for v in 300 310; do [ -s lib$v.glsl ] || continue
echo "lib$v.glsl: $(wc -l < lib$v.glsl) lines; defs named cultmath_: $(grep -cE '^\w+ cultmath_\w+\(' lib$v.glsl); distinct names: $(grep -oE '^\w+ cultmath_\w+\(' lib$v.glsl | sed -E 's/^\w+ //' | sort -u | wc -l); suffixed _N names: $(grep -oE '^\w+ cultmath_\w+_[0-9]+\(' lib$v.glsl | wc -l); _NNN temporaries: $(grep -oE '\b_[0-9]+\b' lib$v.glsl | wc -l); precision lines: $(grep -c precision lib$v.glsl); main: $(grep -c 'void main' lib$v.glsl)"; done
echo "committed CultMath.glsl: $(wc -l < $SH/CultMath.glsl) lines; defs $(grep -cE '^\w+ cultmath_\w+\(' $SH/CultMath.glsl); distinct names $(grep -oE '^\w+ cultmath_\w+\(' $SH/CultMath.glsl | sed -E 's/^\w+ //' | sort -u | wc -l)"
F=$(ls lib310.glsl lib300.glsl 2>/dev/null | head -1); [ -n "$F" ] && { echo "--- head of $F"; head -12 $F; echo "--- cultmath_snoise(vec3) as emitted (first 25 lines)"; awk '/^float cultmath_snoise\(/{p=1} p{print; if(++n>=25) exit}' $F; }
for v in 300 310; do [ -s lib$v.glsl ] && { glslangValidator -S comp lib$v.glsl >gv$v.txt 2>&1; echo "glslang -S comp lib$v.glsl exit $? : $(head -3 gv$v.txt | tr '\n' ' ' | cut -c1-300)"; }; done
step "Q2 lavapipe runner"
gcc -O2 -o vkrun $P/vkrun.c -lvulkan && echo "vkrun built"
python3 $P/fam.py gen /src/packages/cultmath/tests/CultMath.Tests/fixtures/glsl-parity.json fam
cat $SH/CultMath.glsl $SH/CultMath.Cycles.glsl $SH/CultMath.Phacelle.glsl > all.glsl
python3 - <<'PY'
import json; print('families', len(json.load(open('/tmp/w/fam/manifest.json'))))
PY
ok_h=0; ok_g=0; fail=0; first=1
for h in fam/*.hlsl; do t=${h%.hlsl}; tag=$(basename $t)
n=$(python3 -c "import json;m={x['tag']:x for x in json.load(open('fam/manifest.json'))}['$tag'];print(m['n'],m['nout'])")
if $DXC -T cs_6_0 -E main -HV 2021 -spirv -fspv-target-env=vulkan1.2 -I $SH $h -Fo $t.h.spv >$t.h.log 2>&1; then
if [ $first = 1 ]; then VKRUN_INFO=1 ./vkrun $t.h.spv $t.in $t.hlsl.out $n; first=0; else ./vkrun $t.h.spv $t.in $t.hlsl.out $n; fi && ok_h=$((ok_h+1)) || echo "run fail hlsl $tag"
else fail=$((fail+1)); echo "dxc fail $tag: $(head -2 $t.h.log | tr '\n' ' ' | cut -c1-200)"; fi
{ printf '#version 310 es\nprecision highp float;\nprecision highp int;\n'; cat all.glsl $t.body.glsl; } > $t.comp
if glslangValidator -V --target-env vulkan1.2 -o $t.g.spv $t.comp >$t.g.log 2>&1; then ./vkrun $t.g.spv $t.in $t.glsl.out $n && ok_g=$((ok_g+1)) || echo "run fail glsl $tag"
else echo "glslang fail $tag: $(grep -m2 ERROR $t.g.log | tr '\n' ' ' | cut -c1-200)"; fi
done
echo "ran: hlsl->dxc->lavapipe $ok_h, committed glsl->glslang->lavapipe $ok_g, dxc compile failures $fail"
echo "dxc-compute Fma total: $(for s in fam/*.h.spv; do spirv-dis $s | grep -c ' Fma '; done | python3 -c "import sys;print(sum(int(l) for l in sys.stdin))")"
python3 $P/fam.py cmp /src/packages/cultmath/tests/CultMath.Tests/fixtures/glsl-parity.json fam hlsl
python3 $P/fam.py cmp /src/packages/cultmath/tests/CultMath.Tests/fixtures/glsl-parity.json fam glsl
python3 - <<'PY2'
import os,filecmp
fs=sorted(f[:-9] for f in os.listdir('/tmp/w/fam') if f.endswith('.hlsl.out'))
same=sum(filecmp.cmp('/tmp/w/fam/%s.hlsl.out'%f,'/tmp/w/fam/%s.glsl.out'%f,shallow=False) for f in fs if os.path.exists('/tmp/w/fam/%s.glsl.out'%f))
print('lavapipe: HLSL-path and GLSL-path outputs bit-identical in',same,'of',len(fs),'families')
PY2
t0=$(date +%s); for h in fam/*.h.spv; do t=${h%.h.spv}; n=$(python3 -c "import json;m={x['tag']:x for x in json.load(open('fam/manifest.json'))}['$(basename $t)'];print(m['n'],m['nout'])"); ./vkrun $h $t.in $t.rerun.out $n; cmp -s $t.rerun.out $t.hlsl.out || echo "NONDETERMINISTIC $(basename $t)"; done; echo "rerun of 105 SPIR-V modules on lavapipe: $(( $(date +%s) - t0 ))s, determinism check done"
step "Q2b SPIRV-Cross ES 3.10 library through glslang on lavapipe (only if Q1 produced one)"
if [ -s lib310.glsl ]; then sed -E '/^#version/d; /^void main\(\)/,/^\}/d' lib310.glsl > lib.body.glsl; okl=0
for b in fam/*.body.glsl; do t=${b%.body.glsl}; tag=$(basename $t); n=$(python3 -c "import json;m={x['tag']:x for x in json.load(open('fam/manifest.json'))}['$tag'];print(m['n'],m['nout'])")
{ printf '#version 310 es\nprecision highp float;\nprecision highp int;\n'; cat lib.body.glsl $b; } > $t.l.comp
glslangValidator -V --target-env vulkan1.2 -o $t.l.spv $t.l.comp >$t.l.log 2>&1 && ./vkrun $t.l.spv $t.in $t.lib.out $n && okl=$((okl+1))
done; echo "lib path ran $okl"; grep -h -m1 ERROR fam/*.l.log | sort | uniq -c | sort -rn | head -5
python3 $P/fam.py cmp /src/packages/cultmath/tests/CultMath.Tests/fixtures/glsl-parity.json fam lib | head -15
fi
echo PROBE-DONE

11
docs/research/native-shader-probes/strict-form/nc.py Normal file
View file

@ -0,0 +1,11 @@
import re,sys
src,dst=sys.argv[1:3]
L=open(src).read().split('\n')
ids=[]
for l in L:
m=re.match(r'\s*(%\w+) = Op(FMul|FAdd|FSub|VectorTimesScalar|Dot)\b',l)
if m: ids.append(m.group(1))
i=next(k for k,l in enumerate(L) if re.search(r'= OpType',l))
L[i:i]=[' OpDecorate %s NoContraction'%x for x in ids]
open(dst,'w').write('\n'.join(L))
print(src,len(ids))

19
docs/research/native-shader-probes/strict-form/probe1.sh Normal file
View file

@ -0,0 +1,19 @@
set -e
apt-get update -qq >/dev/null; apt-get install -y -qq --no-install-recommends python3 spirv-tools >/dev/null 2>&1
P=packages/cultmath/tools
python3 - <<'PY'
p='packages/cultmath/tools/device/run-lavapipe.sh'
s=open(p).read()
k='python3 "$DEV/gen.py" "$W/work" "$CM"\n'
s=s.replace(k,k+'sed -i -E "s/[.](cos|sin)([^A-Za-z0-9_])/._\1\2/g" "$W"/work/*.comp\ngrep -E "^f10[34]\t" "$W/work/manifest.txt" >"$W/work/m2"; mv "$W/work/m2" "$W/work/manifest.txt"\n')
open(p,'w').write(s)
PY
bash $P/glsl/build-glsl.sh /tmp/glslout >/dev/null
GLSL_DIR=/tmp/glslout bash $P/device/run-lavapipe.sh 2>&1 | tail -4 || true
W=/tmp/device/work
for t in f103 f104; do for r in h g; do
spirv-dis --raw-id $W/$t.$r.spv -o /tmp/$t.$r.dis
echo "== $t $r: NoContraction $(grep -c NoContraction /tmp/$t.$r.dis) Fma $(grep -c ' Fma ' /tmp/$t.$r.dis) FMul $(grep -c OpFMul /tmp/$t.$r.dis) FAdd $(grep -c OpFAdd /tmp/$t.$r.dis) FSub $(grep -c OpFSub /tmp/$t.$r.dis) lines $(wc -l </tmp/$t.$r.dis)"
done; done
mkdir -p /tmp/dump; cp /tmp/f10?.?.dis /tmp/dump/; cp $W/f103.hlsl $W/f103.comp /tmp/dump/; cp /tmp/glslout/*.glsl /tmp/dump/
echo DUMP-BEGIN; tar czf - -C /tmp dump | base64 -w0; echo; echo DUMP-END

46
docs/research/native-shader-probes/strict-form/probe2.sh Normal file
View file

@ -0,0 +1,46 @@
set -e
apt-get update -qq >/dev/null; apt-get install -y -qq --no-install-recommends python3 spirv-tools >/dev/null 2>&1
cat >/tmp/nc.py <<'PYEOF'
import re,sys
src,dst=sys.argv[1:3]
L=open(src).read().split('\n')
ids=[]
for l in L:
m=re.match(r'\s*(%\w+) = Op(FMul|FAdd|FSub|VectorTimesScalar|Dot)\b',l)
if m: ids.append(m.group(1))
i=next(k for k,l in enumerate(L) if re.search(r'= OpType',l))
L[i:i]=[' OpDecorate %s NoContraction'%x for x in ids]
open(dst,'w').write('\n'.join(L))
print(src,len(ids))
PYEOF
P=packages/cultmath/tools
python3 - <<'PY'
p='packages/cultmath/tools/device/run-lavapipe.sh'
s=open(p).read()
k='python3 "$DEV/gen.py" "$W/work" "$CM"\n'
s=s.replace(k,k+'sed -i -E "s/[.](cos|sin)([^A-Za-z0-9_])/._\1\2/g" "$W"/work/*.comp\ngrep -E "^f10[34]\t" "$W/work/manifest.txt" >"$W/work/m2"; mv "$W/work/m2" "$W/work/manifest.txt"\n')
open(p,'w').write(s)
PY
bash $P/glsl/build-glsl.sh /tmp/glslout >/dev/null
GLSL_DIR=/tmp/glslout bash $P/device/run-lavapipe.sh 2>&1 | tail -4 || true
W=/tmp/device/work; R=/tmp/device
mkdir -p $R/nc/lavapipe-hlsl $R/nc/lavapipe-glsl
while IFS=$'\t' read -r tag _ _ n _ wout; do
for r in h g; do
spirv-dis --raw-id $W/$tag.$r.spv -o /tmp/$tag.$r.dis
python3 /tmp/nc.py /tmp/$tag.$r.dis /tmp/$tag.$r.nc.dis
spirv-as --target-env vulkan1.2 /tmp/$tag.$r.nc.dis -o $W/$tag.$r.nc.spv
spirv-val --target-env vulkan1.2 $W/$tag.$r.nc.spv
d=hlsl; [ $r = g ] && d=glsl
$R/vkrun $W/$tag.$r.nc.spv $W/$tag.in $R/nc/lavapipe-$d/$tag.out $n $wout >/dev/null
done
echo "$tag nc: hlsl==glsl $(cmp -s $R/nc/lavapipe-hlsl/$tag.out $R/nc/lavapipe-glsl/$tag.out && echo YES || echo NO); nc-hlsl==plain-hlsl $(cmp -s $R/nc/lavapipe-hlsl/$tag.out $R/results/lavapipe-hlsl/$tag.out && echo YES || echo NO); nc-glsl==plain-glsl $(cmp -s $R/nc/lavapipe-glsl/$tag.out $R/results/lavapipe-glsl/$tag.out && echo YES || echo NO)"
done <$W/manifest.txt
cp /tmp/device/vkrun /tmp/ 2>/dev/null || true
export CULTMATH_DEVICE_NAME=$(sort -u $R/device.txt | head -1 | sed 's/^device=//') CULTMATH_DEVICE_IDENTITY=x CULTMATH_WRITE_DEVICE_BOUNDS=1
for v in results nc; do
export CULTMATH_DEVICE_RESULTS=$R/$v
dotnet test packages/cultmath/tests/CultMath.Tests --no-build --filter "FullyQualifiedName~GlslMirrorTests.DeviceResultsHoldFixture" >/tmp/judge.$v.log 2>&1 || true
echo "== judge $v"; grep -E "^(lavapipe-)" packages/cultmath/tests/CultMath.Tests/fixtures/device-bounds-lavapipe.txt | grep -E " 10[34] " | cut -f1,2,5,6 || tail -5 /tmp/judge.$v.log
tail -3 /tmp/judge.$v.log
done

53
docs/research/native-shader-probes/strict-form/probe3.sh Normal file
View file

@ -0,0 +1,53 @@
set -e
apt-get update -qq >/dev/null; apt-get install -y -qq --no-install-recommends python3 spirv-tools >/dev/null 2>&1
cat >/tmp/nc.py <<'PYEOF'
import re,sys
src,dst=sys.argv[1:3]
L=open(src).read().split('\n')
ids=[]
for l in L:
m=re.match(r'\s*(%\w+) = Op(FMul|FAdd|FSub|VectorTimesScalar|Dot)\b',l)
if m: ids.append(m.group(1))
i=next(k for k,l in enumerate(L) if re.search(r'= OpType',l))
L[i:i]=[' OpDecorate %s NoContraction'%x for x in ids]
open(dst,'w').write('\n'.join(L))
print(src,len(ids))
PYEOF
P=packages/cultmath/tools
python3 - <<'PY'
p='packages/cultmath/tools/device/run-lavapipe.sh'
s=open(p).read()
k='python3 "$DEV/gen.py" "$W/work" "$CM"\n'
s=s.replace(k,k+'sed -i -E "s/[.](cos|sin)([^A-Za-z0-9_])/._\1\2/g" "$W"/work/*.comp\ngrep -E "^f10[34]\t" "$W/work/manifest.txt" >"$W/work/m2"; mv "$W/work/m2" "$W/work/manifest.txt"\n')
open(p,'w').write(s)
PY
bash $P/glsl/build-glsl.sh /tmp/glslout >/dev/null
GLSL_DIR=/tmp/glslout bash $P/device/run-lavapipe.sh 2>&1 | tail -4 || true
W=/tmp/device/work; R=/tmp/device
DXC=$(find $R/dxc -name dxc -type f | head -1); export LD_LIBRARY_PATH=$(dirname $DXC)/../lib:$(dirname $DXC)
CM=packages/cultmath
cat >/tmp/cnt.py <<'PYEOF'
import sys,struct
a=open(sys.argv[1],'rb').read();b=open(sys.argv[2],'rb').read()
n=len(a)//4;x=struct.unpack('<%dI'%n,a);y=struct.unpack('<%dI'%n,b)
d=[i for i in range(n) if x[i]!=y[i]]
print(len(d),'of',n,'cols',sorted(set(i%6 for i in d)))
PYEOF
run() { # name spv tag n wout
$R/vkrun $2 $W/$3.in /tmp/o.$1.$3 $4 $5 >/dev/null; }
OPT="--target-env=spv1.5 --eliminate-local-single-store --eliminate-dead-code-aggressive"
while IFS=$' ' read -r tag _ _ n _ wout; do
for r in h g; do spirv-dis --raw-id $W/$tag.$r.spv -o /tmp/$tag.$r.dis; python3 /tmp/nc.py /tmp/$tag.$r.dis /tmp/$tag.$r.nc.dis >/dev/null; spirv-as --target-env vulkan1.2 /tmp/$tag.$r.nc.dis -o $W/$tag.$r.nc.spv; done
# h variants
$DXC -T cs_6_0 -E main -HV 2021 -spirv -fspv-target-env=vulkan1.2 -Od -I $CM/shaders $W/$tag.hlsl -Fo $W/$tag.hOd.spv >/dev/null 2>&1
spirv-opt $OPT $W/$tag.hOd.spv -o $W/$tag.hOdO.spv
for v in hOd hOdO; do spirv-dis --raw-id $W/$tag.$v.spv -o /tmp/$tag.$v.dis; python3 /tmp/nc.py /tmp/$tag.$v.dis /tmp/$tag.$v.nc.dis >/dev/null; spirv-as --target-env vulkan1.2 /tmp/$tag.$v.nc.dis -o $W/$tag.$v.nc.spv; done
for v in h g hOd hOdO; do run $v $W/$tag.$v.spv $tag $n $wout; run $v.nc $W/$tag.$v.nc.spv $tag $n $wout; done
for v in h hOd hOdO; do
echo "$tag $v vs g: $(python3 /tmp/cnt.py /tmp/o.$v.$tag /tmp/o.g.$tag)"
echo "$tag $v.nc vs g.nc: $(python3 /tmp/cnt.py /tmp/o.$v.nc.$tag /tmp/o.g.nc.$tag)"
done
echo "$tag h.nc vs h: $(python3 /tmp/cnt.py /tmp/o.h.nc.$tag /tmp/o.h.$tag)"
echo "$tag g.nc vs g: $(python3 /tmp/cnt.py /tmp/o.g.nc.$tag /tmp/o.g.$tag)"
echo "$tag hOd.nc vs hOd: $(python3 /tmp/cnt.py /tmp/o.hOd.nc.$tag /tmp/o.hOd.$tag)"
done <$W/manifest.txt

53
docs/research/native-shader-probes/strict-form/probe4.sh Normal file
View file

@ -0,0 +1,53 @@
set -e
apt-get update -qq >/dev/null; apt-get install -y -qq --no-install-recommends python3 spirv-tools >/dev/null 2>&1
cat >/tmp/nc.py <<'PYEOF'
import re,sys
src,dst=sys.argv[1:3]
L=open(src).read().split('\n')
ids=[]
for l in L:
m=re.match(r'\s*(%\w+) = Op(FMul|FAdd|FSub|VectorTimesScalar|Dot)\b',l)
if m: ids.append(m.group(1))
i=next(k for k,l in enumerate(L) if re.search(r'= OpType',l))
L[i:i]=[' OpDecorate %s NoContraction'%x for x in ids]
open(dst,'w').write('\n'.join(L))
print(src,len(ids))
PYEOF
P=packages/cultmath/tools
python3 - <<'PY'
p='packages/cultmath/tools/device/run-lavapipe.sh'
s=open(p).read()
k='python3 "$DEV/gen.py" "$W/work" "$CM"\n'
s=s.replace(k,k+'sed -i -E "s/[.](cos|sin)([^A-Za-z0-9_])/._\\1\\2/g" "$W"/work/*.comp\n')
open(p,'w').write(s)
PY
bash $P/glsl/build-glsl.sh /tmp/glslout >/dev/null
GLSL_DIR=/tmp/glslout bash $P/device/run-lavapipe.sh 2>&1 | tail -4 || true
set +e
W=/tmp/device/work; R=/tmp/device
DXC=$(find $R/dxc -name dxc -type f | head -1); export LD_LIBRARY_PATH=$(dirname $DXC)/../lib:$(dirname $DXC)
CM=packages/cultmath
for d in plain ncod gis; do mkdir -p $R/$d/lavapipe-hlsl $R/$d/lavapipe-glsl; done
while IFS=$'\t' read -r tag _ _ n _ wout; do
spirv-dis --raw-id $W/$tag.g.spv -o /tmp/g.dis; python3 /tmp/nc.py /tmp/g.dis /tmp/g.nc.dis >/dev/null; spirv-as --target-env vulkan1.2 /tmp/g.nc.dis -o /tmp/g.nc.spv
$DXC -T cs_6_0 -E main -HV 2021 -spirv -fspv-target-env=vulkan1.2 -Od -I $CM/shaders $W/$tag.hlsl -Fo /tmp/hOd.spv >/dev/null 2>&1
spirv-dis --raw-id /tmp/hOd.spv -o /tmp/hOd.dis; python3 /tmp/nc.py /tmp/hOd.dis /tmp/hOd.nc.dis >/dev/null; spirv-as --target-env vulkan1.2 /tmp/hOd.nc.dis -o /tmp/hOd.nc.spv
$DXC -T cs_6_0 -E main -HV 2021 -spirv -fspv-target-env=vulkan1.2 -Gis -I $CM/shaders $W/$tag.hlsl -Fo /tmp/hGis.spv >/dev/null 2>&1
spirv-dis --raw-id /tmp/hGis.spv -o /tmp/hGis.dis; python3 /tmp/nc.py /tmp/hGis.dis /tmp/hGis.nc.dis >/dev/null; spirv-as --target-env vulkan1.2 /tmp/hGis.nc.dis -o /tmp/hGis.nc.spv
cp $R/results/lavapipe-hlsl/$tag.out $R/plain/lavapipe-hlsl/; cp $R/results/lavapipe-glsl/$tag.out $R/plain/lavapipe-glsl/
$R/vkrun /tmp/hOd.nc.spv $W/$tag.in $R/ncod/lavapipe-hlsl/$tag.out $n $wout >/dev/null
$R/vkrun /tmp/g.nc.spv $W/$tag.in $R/ncod/lavapipe-glsl/$tag.out $n $wout >/dev/null
$R/vkrun /tmp/hGis.nc.spv $W/$tag.in $R/gis/lavapipe-hlsl/$tag.out $n $wout >/dev/null
cp $R/ncod/lavapipe-glsl/$tag.out $R/gis/lavapipe-glsl/
for d in plain ncod gis; do cmp -s $R/$d/lavapipe-hlsl/$tag.out $R/$d/lavapipe-glsl/$tag.out || echo "EQ-FAIL $d $tag"; done
done <$W/manifest.txt
export CULTMATH_DEVICE_WORK=$W CULTMATH_DEVICE_NAME=$(sort -u $R/device.txt | head -1 | sed 's/^device=//') CULTMATH_DEVICE_IDENTITY=x CULTMATH_WRITE_DEVICE_BOUNDS=1
B=packages/cultmath/tests/CultMath.Tests/fixtures/device-bounds-lavapipe.txt
for v in plain ncod gis; do
export CULTMATH_DEVICE_RESULTS=$R/$v
dotnet test packages/cultmath/tests/CultMath.Tests --no-build --filter "FullyQualifiedName~GlslMirrorTests.DeviceResultsHoldFixture" >/tmp/judge.$v.log 2>&1 || true
grep -aE "Error Message" -A8 /tmp/judge.$v.log | head -14; ls -la $B; cp $B /tmp/bounds.$v; echo "judge $v: $(tail -3 /tmp/judge.$v.log | tr '\n' ' ')"
done
for v in plain ncod gis; do awk -F'\t' -v v=$v '{k=$1; u[k]+=$5; if($5+0>m[k]+0)m[k]=$5; c[k]++} END{for(k in c) print "BOUNDS",v,k,"families",c[k],"maxulp",m[k],"sumulp",u[k]}' /tmp/bounds.$v; done
for v in plain ncod; do awk -F'\t' '{print $1"/"$2"\t"$5}' /tmp/bounds.$v | sort > /tmp/b.$v; done
join -t$'\t' /tmp/b.plain /tmp/b.ncod | awk -F'\t' '$2!=$3{print "ULPDIFF",$1,"plain",$2,"ncod",$3}' | head -60

87
docs/research/native-shader-probes/vkrun.c Normal file
View file

@ -0,0 +1,87 @@
// Probe runner: one compute dispatch, binding 0 = input uints, binding 1 = output uints.
// usage: vkrun <spv> <in.bin> <out.bin> <ncases> <out_words_per_case> [entry]
#include <vulkan/vulkan.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#define CK(x) do { VkResult r_ = (x); if (r_ != VK_SUCCESS) { fprintf(stderr, "FAIL %s = %d\n", #x, r_); exit(2); } } while (0)
static void *slurp(const char *p, size_t *n) {
FILE *f = fopen(p, "rb"); if (!f) { perror(p); exit(2); }
fseek(f, 0, SEEK_END); *n = ftell(f); fseek(f, 0, SEEK_SET);
void *b = malloc(*n ? *n : 4); fread(b, 1, *n, f); fclose(f); return b;
}
int main(int argc, char **argv) {
if (argc < 6) { fprintf(stderr, "usage\n"); return 2; }
size_t spvn, inn; void *spv = slurp(argv[1], &spvn); void *in = slurp(argv[2], &inn);
unsigned ncases = atoi(argv[4]), ow = atoi(argv[5]); const char *entry = argc > 6 ? argv[6] : "main";
size_t outn = (size_t)ncases * ow * 4;
VkApplicationInfo ai = { VK_STRUCTURE_TYPE_APPLICATION_INFO, 0, "probe", 1, "probe", 1, VK_API_VERSION_1_2 };
VkInstanceCreateInfo ici = { VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO, 0, 0, &ai };
VkInstance inst; CK(vkCreateInstance(&ici, 0, &inst));
uint32_t np = 1; VkPhysicalDevice pd; vkEnumeratePhysicalDevices(inst, &np, &pd); if (!np) { fprintf(stderr, "no device\n"); return 2; }
VkPhysicalDeviceFloatControlsProperties fc = { VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FLOAT_CONTROLS_PROPERTIES };
VkPhysicalDeviceProperties2 pp = { VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2, &fc };
vkGetPhysicalDeviceProperties2(pd, &pp);
if (getenv("VKRUN_INFO"))
printf("device=%s api=%u.%u driver=%u f32: denormPreserve=%u ftz=%u rte=%u rtz=%u signedZeroInfNan=%u\n",
pp.properties.deviceName, VK_VERSION_MAJOR(pp.properties.apiVersion), VK_VERSION_MINOR(pp.properties.apiVersion),
pp.properties.driverVersion, fc.shaderDenormPreserveFloat32, fc.shaderDenormFlushToZeroFloat32,
fc.shaderRoundingModeRTEFloat32, fc.shaderRoundingModeRTZFloat32, fc.shaderSignedZeroInfNanPreserveFloat32);
float prio = 1; VkDeviceQueueCreateInfo qci = { VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO, 0, 0, 0, 1, &prio };
VkDeviceCreateInfo dci = { VK_STRUCTURE_TYPE_DEVICE_CREATE_INFO, 0, 0, 1, &qci };
VkDevice dev; CK(vkCreateDevice(pd, &dci, 0, &dev)); VkQueue q; vkGetDeviceQueue(dev, 0, 0, &q);
VkPhysicalDeviceMemoryProperties mp; vkGetPhysicalDeviceMemoryProperties(pd, &mp);
VkBuffer buf[2]; VkDeviceMemory mem[2]; size_t sz[2] = { inn, outn }; void *map[2];
for (int i = 0; i < 2; i++) {
VkBufferCreateInfo bci = { VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, 0, 0, sz[i], VK_BUFFER_USAGE_STORAGE_BUFFER_BIT };
CK(vkCreateBuffer(dev, &bci, 0, &buf[i])); VkMemoryRequirements mr; vkGetBufferMemoryRequirements(dev, buf[i], &mr);
uint32_t t = 0; VkMemoryPropertyFlags want = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT;
for (; t < mp.memoryTypeCount; t++) if ((mr.memoryTypeBits & (1u << t)) && (mp.memoryTypes[t].propertyFlags & want) == want) break;
VkMemoryAllocateInfo mai = { VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO, 0, mr.size, t };
CK(vkAllocateMemory(dev, &mai, 0, &mem[i])); CK(vkBindBufferMemory(dev, buf[i], mem[i], 0));
CK(vkMapMemory(dev, mem[i], 0, sz[i], 0, &map[i]));
}
memcpy(map[0], in, inn); memset(map[1], 0xEE, outn);
VkDescriptorSetLayoutBinding b[2] = { { 0, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 1, VK_SHADER_STAGE_COMPUTE_BIT },
{ 1, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 1, VK_SHADER_STAGE_COMPUTE_BIT } };
VkDescriptorSetLayoutCreateInfo lci = { VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO, 0, 0, 2, b };
VkDescriptorSetLayout dsl; CK(vkCreateDescriptorSetLayout(dev, &lci, 0, &dsl));
VkPipelineLayoutCreateInfo plci = { VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO, 0, 0, 1, &dsl };
VkPipelineLayout pl; CK(vkCreatePipelineLayout(dev, &plci, 0, &pl));
VkShaderModuleCreateInfo smci = { VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO, 0, 0, spvn, spv };
VkShaderModule sm; CK(vkCreateShaderModule(dev, &smci, 0, &sm));
VkComputePipelineCreateInfo cpci = { VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO, 0, 0,
{ VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, 0, 0, VK_SHADER_STAGE_COMPUTE_BIT, sm, entry }, pl };
VkPipeline pipe; CK(vkCreateComputePipelines(dev, 0, 1, &cpci, 0, &pipe));
VkDescriptorPoolSize ps = { VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 2 };
VkDescriptorPoolCreateInfo dpci = { VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO, 0, 0, 1, 1, &ps };
VkDescriptorPool dp; CK(vkCreateDescriptorPool(dev, &dpci, 0, &dp));
VkDescriptorSetAllocateInfo dsai = { VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO, 0, dp, 1, &dsl };
VkDescriptorSet ds; CK(vkAllocateDescriptorSets(dev, &dsai, &ds));
VkDescriptorBufferInfo dbi[2] = { { buf[0], 0, VK_WHOLE_SIZE }, { buf[1], 0, VK_WHOLE_SIZE } };
VkWriteDescriptorSet w[2];
for (int i = 0; i < 2; i++) { VkWriteDescriptorSet x = { VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, 0, ds, i, 0, 1, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 0, &dbi[i] }; w[i] = x; }
vkUpdateDescriptorSets(dev, 2, w, 0, 0);
VkCommandPoolCreateInfo cpi = { VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO, 0, 0, 0 };
VkCommandPool cp; CK(vkCreateCommandPool(dev, &cpi, 0, &cp));
VkCommandBufferAllocateInfo cbai = { VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO, 0, cp, VK_COMMAND_BUFFER_LEVEL_PRIMARY, 1 };
VkCommandBuffer cb; CK(vkAllocateCommandBuffers(dev, &cbai, &cb));
VkCommandBufferBeginInfo cbbi = { VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO };
CK(vkBeginCommandBuffer(cb, &cbbi));
vkCmdBindPipeline(cb, VK_PIPELINE_BIND_POINT_COMPUTE, pipe);
vkCmdBindDescriptorSets(cb, VK_PIPELINE_BIND_POINT_COMPUTE, pl, 0, 1, &ds, 0, 0);
vkCmdDispatch(cb, (ncases + 63) / 64, 1, 1);
CK(vkEndCommandBuffer(cb));
VkSubmitInfo si = { VK_STRUCTURE_TYPE_SUBMIT_INFO, 0, 0, 0, 0, 1, &cb };
CK(vkQueueSubmit(q, 1, &si, 0)); CK(vkQueueWaitIdle(q));
FILE *f = fopen(argv[3], "wb"); fwrite(map[1], 1, outn, f); fclose(f);
return 0;
}

178
docs/research/native-shader-testing.md Normal file
View file

@ -0,0 +1,178 @@
# Native shader testing: how established projects check shader numerics without a browser, and what each path proves
For `cultmath-tapes:question:device-run-driver` (campaign `cultmath-tapes`). Companion to `docs/research/device-capture-runners.md` (CultLib c9d3e672), which covered browser drivers only; not repeated here.
Also serves the operator's question of 2026-10-16 on why CultMath lowers HLSL to GLSL with its own text lowering (section on HLSL to GLSL ES / SPIR-V toolchains).
Gathered by an Eyes pass for Eureka Self, session self-2026-10-16-ag4, 2026-10-16. Facts only; no recommendation.
Strength marks: [src] read from a primary source text this pass (raw file fetched and read or grepped locally, or a spec PDF converted to text); [doc] vendor or project documentation page read this pass (via a fetch summariser, so check before quoting); [web] search-result summary only, not read in full; [abs] searched, not found.
Nothing was built or run. Dates matter: several repos below are moving; read at HEAD on 2026-10-16.
## 0. The record
- [src] Operator, 2026-10-16, verbatim: "There has to be a better way to test shader code than getting a browser involved". An observation that sets research direction, not a design ruling.
- [src] Question `device-run-driver` (viewed in the mind): options committed-captures / playwright-live / both; the runner is a static page that compiles each family's call against `CultMath.glsl`, reads back via RGBA32UI, emits raw bits; C# judges.
- [src] CultMath shader files: `packages/cultmath/shaders/` has `CultMath.hlsl` (673 lines), `CultMath.Affine.hlsl`, `.Cycles.hlsl`, `.Interval.hlsl`, `.Phacelle.hlsl`, and `CultMath.glsl` (900), `.Cycles.glsl`, `.Phacelle.glsl`. HLSL intrinsics seen by grep across the `.hlsl` files: `asuint` (13), `asfloat` (6), `saturate` (2); no `mad`, `fma`, `frexp`, `ldexp`, `precise`, `sincos`, `rcp`, `groupshared`, `#include` hits in the grep (the `GlslLowering.cs` doc comment says the HLSL has "includes" that are inlined).
- [src] The `.glsl` files contain no `precision` statement and no `#version` line (grep found none); a consumer supplies them.
- [src] `GlslLowering.cs` header: the GLSL ES 3.00 files are a regex text transformation of the HLSL; a dialect gap is fixed in the HLSL "in the subset both languages share".
## 1. Google Amber
- [doc] README (github.com/google/amber): runs on Vulkan (fully supported); Dawn listed "WIP". A command-line `amber` tool runs script files, reports pass/fail by exit code, can write output buffers and images to files. Says "not an officially supported Google product". AmberScript is the format getting new features; VkScript (the VkRunner syntax) is kept for history, no new features. Build bots check that Amber compiles on several targets; "integration tests don't run on the bots because of bot limitations, so only unit tests run in CI".
- [src] Maintenance: GitHub API `google/amber`: not archived, created 2018-11-13, pushed 2026-09-24, 76 open issues. Last five commits 2026-05-28 to 2026-09-24 (ray query support, acceleration structures, alpha-to-coverage, barrier fixes).
- [doc] AmberScript (docs/amber_script.md): starts `#!amber`. `SHADER compute <name> <format>` where format is GLSL, HLSL, SPIRV-ASM, SPIRV-HEX, SPIRV-BIN or OPENCL-C, inline, `FILE`, or `VIRTUAL_FILE`; optional `TARGET_ENV`. `PIPELINE compute <name> ... END`. `BUFFER <name> DATA_TYPE <type> [STD140|STD430] DATA ... END | FILL v | SERIES_FROM s INC_BY i`, types include int8..int64, uint8..uint64, float16, float, double, vectors, matrices. `COPY a TO b`. `RUN <pipeline> x y z`.
- [doc] Expectations: `EXPECT <buf> IDX <n> EQ|NE|LT|LE|GT|GE <values>`; with tolerance `EXPECT <buf> IDX <n> TOLERANCE <t1..t4> EQ <values>` (1 to 4 floats, a trailing `%` makes one relative; components without one use a default); whole-buffer `EQ_BUFFER`, `RMSE_BUFFER ... TOLERANCE <x>`, `EQ_HISTOGRAM_EMD_BUFFER ... TOLERANCE <x>`. Pixel forms `EQ_RGB`, `EQ_RGBA` use 0-255. Comparison is on typed values, not raw bit patterns; a uint32 buffer compared with `EQ` is an exact integer compare. [abs] A ULP-based tolerance form was not found in the AmberScript doc.
- [doc] VkScript (docs/vk_script.md): `probe ssbo` has a `~=` fuzzy-equal comparator and a `tolerance t0 [t1 t2 t3]` command.
- [web] Amber with SwiftShader: a build option `AMBER_ENABLE_SWIFTSHADER`/`AMBER_USE_SWIFTSHADER` builds SwiftShader as a Vulkan ICD; run with `VK_ICD_FILENAMES` pointing at its manifest; `amber -d` style device listing confirms it. [abs] Amber's own CI workflow files were not found at the paths tried (404); whether Amber CI uses lavapipe or SwiftShader was not established.
- [web] Amber lineage: shader test files mirror VkRunner's format; Amber is shipped inside Android's `external/deqp-deps` and Vulkan CTS.
## 2. Software Vulkan: Mesa lavapipe and SwiftShader
### 2.1 Conformance and float behaviour
- [web] Lavapipe announced formally Vulkan 1.2 conformant (Phoronix, May 2022) and Vulkan 1.3 conformant (July 2022; Khronos submission 696, AMD Ryzen 7 1800X, Mesa 22.1). A Mesa issue (#5472, older) complained lavapipe reported a conformance version while not yet conformant; resolution not checked.
- [web] Mesa MR 11739 "WIP: lavapipe: float_controls" (author's words as summarised): passes about 97% of CTS, fails `float_controls...denorm_*_preserve` cases, "llvmpipe/gallivm have no concept of setting a specific denorm mode"; the author said they probably would not continue. Current merged state of float controls in lavapipe: [abs] (the Mesa GitLab pages and `lvp_device.c` were behind bot protection or not at the tried path).
- [web] A separate llvmpipe MR disables denorms for compute shaders to align with fragment shaders, because OpenCL CTS tests "don't expect working denorms".
- [web] SwiftShader commit "Implement VK_KHR_shader_float_controls": at that time SwiftShader exposed no preserve, denorm or rounding controls and set all denorm-preserve and flush-to-zero flags false. Current values: [abs].
- [doc] Vulkan float controls (core in 1.2): a flush-to-zero flag set means denormals may be flushed, not that they are; preservation needs `shaderDenormPreserveFloat32` and a `DenormPreserve` execution mode on the entry point. Rounding-mode controls do not apply to float to int conversions (always toward zero).
- [web] Mesa lavapipe docs page (docs.mesa3d.org/drivers/lavapipe.html) 404s at the tried URL; the `llvmpipe.html` page was read and says nothing of lavapipe floats.
- [src] D3D11 floating-point rules (learn.microsoft.com, doc dated 2025-02/04): 32-bit ops are within 1 ULP (add, subtract, multiply: 0.5 ULP "applies only to addition, subtraction, and multiplication", hardware may truncate); "Denorms are flushed to sign-preserved zero on input and output of any floating-point mathematical operation" (data movement exempt); min/max flush for comparison but the result may or may not be flushed; fused operations (mad, dp3) are no less accurate than the worst serial ordering with each step allowed 1 ULP; sqrt and rcp 1 ULP, rcp/rsq have their own relaxed requirement; x*1.0, x/1.0, x+-0.0 return x "except denorm flushed"; NaN payload bits not preserved; no exceptions or traps; double precision preserves denorms and follows IEEE 754R; float16 preserves denorms.
- [src] GLSL ES 3.00 spec section 4.5.1 (Khronos PDF, converted locally): highp float is "defined by the IEEE 754 standard for 32-bit" in storage; "Any subnormal (denormalized) value input into a shader or potentially generated by any operation in a shader can be flushed to 0. The rounding mode cannot be set and is undefined." Required precisions: `a+b, a-b, a*b` correctly rounded; `a/b` and `1.0/b` 2.5 ULP for b in [2^-126, 2^126]; `a*b+c` "correctly rounded single operation or sequence of two correctly rounded operations"; `exp`, `exp2` (3 + 2|x|) ULP; `log`, `log2` 3 ULP outside [0.5, 2.0] and absolute error < 2^-21 inside; `inversesqrt` 2 ULP; `sqrt` inherits from `1.0/inversesqrt`; trig and `determinant` have undefined precision; explicit conversions correctly rounded; "rounding mode ... must not affect the result by more than 1 ULP". NaNs and Infs "are not required to be generated".
- [src] GLSL ES 3.10 spec (NVIDIA-hosted copy of the Khronos PDF) repeats the same flush-to-zero sentence (line 3061 of the converted text).
- [src] Table of required ranges in ES 3.00: highp relative precision 2^-24 (as IEEE); mediump minimum range (-2^14, 2^14), relative 2^-10; lowp minimum range (-2, 2), absolute 2^-8. Integers: highp 32-bit; mediump 16 to 32 bits; lowp 9 to 32 bits. "Mediump and lowp ... have maximum range and precision as defined by IEEE 754", so an implementation may run them at 32 bits.
### 2.2 Who uses software Vulkan in CI
- [src] wgpu `.github/workflows/ci.yml` (trunk, read via fetch): job `gpu-test` on `windows-2022`, `macos-14`, `ubuntu-24.04`. Linux installs the Vulkan SDK and Mesa (driver named only via the composite action, not visible), sets `LVP_POISON_MEMORY: true`; Windows installs DXC and WARP (`install-warp` action), Agility SDK, Mesa, Vulkan SDK, sets `WGPU_DX12_COMPILER: dxc`. Tests run with `cargo xtask test` (nextest).
- [doc] wgpu `docs/testing.md`: names `LVP_POISON_MEMORY` ("fills all newly initialized memory with a non-zero value"); describes a parameter system that skips or expects failure per adapter; says nothing on float or denorm differences.
- [web] onnxruntime commit replaced Mesa lavapipe with SwiftShader in headless CI because an old lavapipe returned `VK_ERROR_INCOMPATIBLE_DRIVER` when Dawn asked for a Vulkan 1.3 instance; the SwiftShader ICD is built from source there and used by Dawn.
- [web] Dawn/Chromium use the SwiftShader Vulkan ICD for headless runs; ANGLE end2end has a `VULKAN_SWIFTSHADER` config and a `--deqp-egl-display-type=angle-swiftshader` mode (section 4).
- [web] Wine/vkd3d: tests marked broken on WARP; a November 2025 commit disabled 64-bit wave bitwise ops on WARP "known to be buggy"; a merge request says WARP "doesn't emulate faithfully a hardware device".
- [abs] No project was found that commits raw float bit patterns from lavapipe or SwiftShader as a reference. Projects found compare with tolerance, by pixel, or by exact integer.
## 3. GLSL/HLSL to SPIR-V
### 3.1 glslang
- [src] glslang README (read at main): ESSL is a supported front end; compute shaders use `.comp`; stage is inferred from the file extension; C++ and C interfaces exist; JS/WASM build via `-DENABLE_GLSLANG_JS=ON`. Tests: gtest under `gtests/` plus a `runtests` script, expected outputs in `Test/baseResults/`.
- [src] The HLSL front end of glslang is deprecated: README states deprecation, slated for removal in the next major version with at least 18 months' notice, no bug reports accepted. Commit "Deprecate the HLSL front-end." dated 2026-04-16 (GitHub API). This does not touch `dxc -spirv`, a separate compiler.
- [web] Compute shaders need desktop GLSL 4.30 or GLSL ES 3.10 (Mesa error text "Compute shaders require GLSL 4.30 or GLSL ES 3.10"). In ES 3.10 only the last member of a shader storage block may be a runtime-sized array; desktop is looser.
- [src] SPIRV-Cross (section 3.3) throws "At least ESSL 3.10 required for compute shaders." for ES versions under 310. So a `#version 300 es` compute shader does not exist: a GLSL ES 3.00 fragment function ported to compute becomes a 3.10 shader.
- [src] ES 3.10 section 4.7.4: vertex and compute languages have predeclared `precision highp float; precision highp int;`; the fragment language has "no default precision qualifier for floating point types" and `mediump int`. So an unqualified float in a compute shader is highp, while a fragment shader must declare a float precision.
- [src] ES 3.10 and 3.00 share the flush-to-zero sentence and the ULP table for highp (the 3.10 table was not diffed line by line; the denorm sentence was).
- [abs] A published comparison of ES compute against desktop 4.30 numerics (same function, fragment vs compute) was not found. The two specs' highp rules, as read, are the same sentence; what a given driver does per stage is not stated by either spec.
### 3.2 dxc -spirv
- [src] DXC `docs/SPIR-V.rst` (235 KB, read by grep): default target environment `vulkan1.0` via `-fspv-target-env`; `-fspv-entrypoint-name=<name>` exists; `min16float`, `min10float`, `min16int`, `min12int`, `min16uint` map to 32-bit types with `RelaxedPrecision` unless `-enable-16bit-types`, which gives real 16-bit types and the `Float16` capability. `lib_6_x` appears only in the raytracing section ("must be compiled as library using lib_6_3/lib_6_4"). [abs] No statement in the doc that a non-raytracing `lib_6_x` library compiles to SPIR-V with exported functions; no statement on denormal execution modes.
- [src] `SPV_KHR_float_controls` is in the supported-extension list; no description of when dxc emits denorm modes (not found by grep of the doc).
- [src] HLSL exposes `asuint`/`asfloat` as bitcasts; in SPIR-V these are `OpBitcast`. (CultMath's use is section 0.)
### 3.3 SPIRV-Cross (SPIR-V to GLSL ES/HLSL/MSL)
- [src] README (main): ES example `spirv-cross --version 310 --es test.spv`; sections on HLSL-derived SPIR-V; names glslangValidator's `-e` entry-point flag in an error hint. Active: commits 2026-09-18, 2026-09-22, 2026-10-14.
- [src] `spirv_glsl.cpp`: `#version` is written as `#version <n> es` when `options.es && version > 100`; ES is allowed from 100 up (the `300 es` form is the emitted string); buffer blocks need ES >= 300, `binding` needs >= 310.
- [src] Precision: `backend.allow_precision_qualifiers = options.vulkan_semantics || options.es;` `flags_to_qualifiers_glsl` emits `mediump ` for ids with `RelaxedPrecision`, else `highp `, and omits either when it equals the fragment default; structs and doubles get none. `Options::Fragment` defaults are `default_float_precision = Mediump`, `default_int_precision = Highp` (`spirv_glsl.hpp` lines 196-197). `emit_header()` writes `precision <p> float;` and `precision <p> int;` only for fragment shaders when `options.es`. So a function-only library run through `--es` with default options would, in a fragment context, assume mediump float defaults and tag non-relaxed values `highp`.
- [src] Library modules: `spirv_parser.cpp` (main) handles a module with no `OpEntryPoint` but with `OpDecorate ... LinkageAttributes ... Export`: marks `is_library_module`, uses the first exported function as a placeholder entry point with execution model GLCompute, and `spirv_glsl.cpp` emits "each exported function as a normal free function" plus the callees. A module with neither throws "There is no entry point in the SPIR-V module." The code paths exist at HEAD; which release first had them, and whether the output is valid for ES, were not checked.
- [web] A Unity issue reports min16float hints (`min16f as def32` modifier in FXC bytecode) not honoured in SPIR-V/ESSL output (mediump produced where highp was expected). One blog summary says SPIRV-Cross may pick a low default precision unless told otherwise (not authoritative).
- [src] Readability: [abs] no source characterising readability of the emitted GLSL was found; SPIRV-Cross output is generated text with temporaries named `_<id>` unless debug names survive (names come from OpName; not verified for dxc output).
## 4. Native ANGLE without a browser
- [doc] ANGLE README: implements EGL 1.5 and GLES 2.0/3.0/3.1 over Vulkan, desktop GL, GLES, D3D9, D3D11, Metal; "Chrome uses ANGLE for all graphics rendering on Windows" and it is the default WebGL backend for Chrome and Firefox on Windows; its shader compiler can emit Desktop GLSL, Vulkan GLSL, HLSL, ESSL.
- [doc] ANGLE `doc/DevSetup.md`: build standalone with `fetch angle`, `gn gen out/Debug`, `autoninja -C out/Debug`; tests, libraries and samples land in `out/Debug`; on Windows default backend is D3D11 (D3D9 otherwise); backend chosen at runtime through `EGL_ANGLE_platform_angle` via `eglGetPlatformDisplayEXT` (`util/EGLWindow.cpp` as example); backends removable via GN args `angle_enable_d3d11` etc. [abs] prebuilt binaries not covered in the doc.
- [src] ANGLE `doc/dEQP.md`: targets `angle_deqp_gles2_tests`, `angle_deqp_gles3_tests`, `angle_deqp_gles31_tests`, `angle_deqp_egl_tests`; renderer chosen with `--deqp-egl-display-type=angle-d3d11 | angle-d3d9 | angle-d3d11-fl93 | angle-gl | angle-gles | angle-metal | angle-swiftshader | angle-vulkan` (same as `--use-angle=X`); default is Vulkan except Apple (OpenGL); test lists come from `third_party/VK-GL-CTS/.../gles3-main.txt`; GoogleTest builds of the same targets run on the bots with expectation files in `src/tests/deqp_support`.
- [src] ANGLE `doc/TestingAndProcesses.md`: angle_end2end_tests expectations live in `src/tests/angle_end2end_tests_expectations.txt` (`SKIP`, `TIMEOUT` only); the WebGL conformance expectations live in the Chromium tree, not the ANGLE tree.
- [web] `angle_end2end_tests` configs include `ES2_D3D11_WARP`, `ES2_D3D11_REFERENCE`, feature-level variants and `VULKAN_SWIFTSHADER`; end2end tests subsume the older angle_tests and standalone tests.
- [doc] ANGLE `EGL_ANGLE_create_context_webgl_compatibility` / `ANGLE_webgl_compatibility` (spec text, Draft): `eglCreateContext` attribute `EGL_CONTEXT_WEBGL_COMPATIBILITY_ANGLE` adds WebGL validation; shaders with `#version 100` get WebGL 1 rules and `#version 300 es` get WebGL 2 rules.
- [src] Path in D3D11 (`src/libANGLE/renderer/d3d/d3d11/Renderer11.cpp`, main): ANGLE translator emits HLSL, then `D3DCompile` with profile `vs_5_0`/`ps_5_0` style (`getMajorShaderModel`), flags `D3DCOMPILE_OPTIMIZATION_LEVEL2`, plus `D3DCOMPILE_IEEE_STRICTNESS` only `if (workarounds.enableIEEEStrictness)`; on failure retries with `SKIP_VALIDATION`, then `SKIP_OPTIMIZATION`. The HLSL is generated at link time (`DynamicShaderCompilation.md`) because signatures between stages must be known.
- [src] ANGLE `doc/CompilingTranslatorWithEmscripten.md`: the translator alone can be compiled for use from a page (preliminary notes).
- [abs] The `enableIEEEStrictness` workaround's default value in ANGLE and in Chrome was not found. [abs] Whether Chrome's WebGL2 context and a standalone ANGLE EGL context share all translator options and workaround flags was not established (same repository; Chrome applies its own feature flags and the WebGL-compatibility context).
- [abs] No source found for a project that uses a standalone ANGLE `libEGL`/`libGLESv2` as a compute-readback harness; ANGLE's own tests use it as the system under test.
- [src] Context: GLES 3.0 cannot run compute (3.1 adds it); ANGLE advertises 3.1 (README). A WebGL2 fragment readback and an ES 3.1 compute readback are different ANGLE shader paths (pixel-shader HLSL vs compute-shader HLSL).
## 5. D3D11/D3D12 compute harnesses for HLSL
- [src] DXC repo `tools/clang/unittests/HLSLExec/` files: `ExecutionTest.cpp`, `HlslExecTestUtils.cpp`, `LongVectors.cpp`, `ShaderOpArithTable.xml`, `ShaderOpTest.cpp` and others; the test project uses TAEF (WEX) test methods.
- [src] `HlslExecTestUtils.cpp`: `useWarpByDefault()` returns true unless `_HLK_CONF` is defined; when the `UseWARP` parameter is set it creates a device from `EnumWarpAdapter` at feature level 11_0, with an optional `WARP_DLL` path to load a specific WARP build.
- [src] `ExecutionTest.cpp`: `DenormBinaryFloatOpTest`/`DenormTertiaryFloatOpTest` read tables from `ShaderOpArithTable.xml#DenormBinaryFloatOpTable`/`DenormTertiaryFloatOpTable` (rows like `FDivDenormFTZ`; columns `ShaderOp.Target`, `ShaderOp.Arguments`, `Validation.Type`, `Validation.Tolerance`, inputs and expected values); a helper `ifdenorm_flushf_eq` accepts either flushed or unflushed results; derivative tests use `CompareFloatULP(..., 1)` with alternative acceptable values.
- [src] `LongVectors.cpp`: validation config `Epsilon(tol)` or `Ulp(tol)`; default is ULP 1.0 for floating-point types, exact for others; failures print the validation type and tolerance.
- [src] DXC's `HLSLExec` tests thus run on WARP by default and on hardware via a flag; tolerances are per-row in data tables.
- [web] Shader Test Framework (KStocky, D3D12, Catch2, HLSL 202x): README says "very experimental"; tests are HLSL functions run through a `ShaderTestFixture`; a device can be hardware or WARP (README read by fetch did not state WARP; a search summary and the Skyrim Community Shaders DeepWiki page did); README does not state its float comparison. Used by Skyrim Community Shaders per a DeepWiki page (AI-generated, unverified).
- [web] An LLVM 2025 dev meeting talk lists "accuracy tolerances" and "NaN/INF/denorm handling" among hard areas for HLSL test writing.
- [web] WARP is Microsoft's recommended D3D11 device for service/server environments; Session 0 services cannot use hardware D3D9 HAL on WDDM (also in the earlier file).
- [abs] WARP's own float conformance statement (denorm flushing, mad fusion) was not found in a primary document; the D3D11 rules in 2.1 are the contract WARP implements.
- [web] vkd3d-proton and Wine's `d3d12` tests mark WARP-specific broken cases (see 2.2).
## 6. Other paths
### 6.1 Slang
- [doc] Targets page (shader-slang.org/slang/user-guide/targets.html): D3D11 (DXBC), D3D12 (DXIL), Vulkan (SPIR-V), OpenGL (GLSL source, SPIR-V, vendor binaries; "only limited support for compiling code for OpenGL"), Metal, CUDA and OptiX, CPU compute ("compute only", "functional, but not feature- or performance-complete"), WebGPU (WGSL, "work in progress"). GLSL ES: not mentioned.
- [doc] `a2-04-glsl-target-specific.md`: version selected by `-profile glsl_<version>` (example `glsl_450`); the GLSL target "is currently less mature than the SPIR-V target and has several known limitations"; known issue with 3-element vector constant-buffer packing. GLSL ES / `300 es`: not mentioned. [abs] A search for Slang emitting `#version 300 es` found nothing.
- [doc] `docs/cpu-target.md`: outputs C++ (`-target cpp`) or executables, shared libraries, host-callable code, object code; in-process execution through `getTargetHostCallable`; default host-callable backend is `slang-llvm` (no file system, no C compiler needed, shader style only), alternatives are system Clang/GCC/Visual Studio via temp files; limitations: no barriers, no user atomics (except compiler-generated 32/64-bit int add), entry points cannot be named `main`, `float16_t` unsupported, limited bounds-check handling; "no content on floating-point semantics".
- [web] Slang can compile to SPIR-V directly or via GLSL (`-emit-spirv-via-glsl`, glslang downstream).
### 6.2 wgpu / naga
- [src] naga README table: inputs SPIR-V (primary), WGSL (primary, "fully validated"), GLSL (secondary, "GLSL 440+ and Vulkan semantics only"); outputs SPIR-V, Metal, HLSL ("Shader Model 5.0+"), WGSL (secondary), GLSL (secondary, "GLSL 330+ and GLSL ES 300+"). HLSL is not an input language. naga is a different translator from ANGLE's; wgpu's DX12 path uses naga HLSL output compiled with DXC or FXC (`WGPU_DX12_COMPILER`, ci.yml).
- [src] wgpu CI deletes `naga/tests/out` snapshots before test runs (ci.yml), i.e. naga keeps snapshot-compared output files.
### 6.3 Tint (Dawn's WGSL compiler)
- [web] Search snapshots (2020-2021, a CMake-only Dawn fork): Tint reads WGSL (default on) and SPIR-V (`TINT_BUILD_SPV_READER`, off by default); writers for SPIR-V, WGSL, HLSL, MSL, GLSL (GLSL writer added October 2021, validated with glslang). [abs] The current upstream README was not retrievable (404 at tried paths), so current readers and writers are unconfirmed.
### 6.4 Shader-function unit-test frameworks and GPU-free paths
- [web] `shadertest` (Kupoman, Python): load a GLSL file, call its functions from pytest; supports bool, int, uint, float and vectors; implementation (GPU context) not read.
- [web] `shdr` (cdave1): experimental CPU emulator for GLSL fragment shaders, translates GLSL to JavaScript, "does not quite match the glsl". `rpu` (Rust crate): a GLSL-like language compiled to WebAssembly for CPU rendering, not full GLSL. HlslUnit (.NET): an interpreter that runs HLSL on the CPU (search summary).
- [web] Blender added infrastructure to run GLSL shader unit tests and shares C++/GLSL source in the GPU module, compiled through the real GPU backend.
- [web] Khronos forum "GLSL Unit" (Google MapsGL team) unit-testing framework for shaders, old, likely dormant.
- [doc] Vulkan Validation Layers `debug_printf`: shaders print through `debugPrintfEXT`; output goes to the validation layer message stream; [web] each a runtime Vulkan layer, not a numeric assertion framework.
- [abs] No GPU-free GLSL ES interpreter with IEEE-faithful float semantics, maintained, and used by a notable project was found.
## 7. Cross-cutting: tools that produce SPIR-V for sections 1-3 (item 8)
| Tool | Input | Output | Library with no entry point | Maintenance |
|---|---|---|---|---|
| glslang | GLSL, ESSL, HLSL (deprecated 2026-04-16) | SPIR-V | not covered in README | active (commits 2026-04) |
| dxc `-spirv` | HLSL | SPIR-V | `lib_6_x` only documented for raytracing; `-E` entry otherwise [abs] export behaviour | active repo, wiki and docs |
| SPIRV-Cross | SPIR-V | GLSL (incl. `300 es`, `310 es`), HLSL, MSL | HEAD has library-module path keyed on `LinkageAttributes Export` | active (2026-10-14) |
| Slang | Slang (accepts HLSL-like syntax) | SPIR-V, HLSL, GLSL desktop (`-profile glsl_N`), WGSL (WIP), CPU, others | not examined | active |
| Unity HLSLcc | DX bytecode (DXBC) | GLSL 3.2+, GLSL ES 2.0+ ("OpenGL ES 3.0+" used for Unity), Metal, Vulkan GLSL | n/a (bytecode) | `pushed_at` 2024-07-16, not archived, 30 open issues, README "used to generate all Unity shaders for OpenGL, OpenGL ES 3.0+, Metal, Vulkan"; supports `min16float`, infers sampler precision |
| naga | WGSL, SPIR-V, GLSL 440+ | SPIR-V, WGSL, GLSL ES 300+, HLSL SM5+, MSL | n/a | active (wgpu) |
| Tint | WGSL (SPIR-V reader optional) | SPIR-V, HLSL, MSL, GLSL (per 2021 snapshot) | n/a | unconfirmed |
- [src] Which can also feed Amber/Vulkan runs: glslang and dxc produce SPIR-V from GLSL/HLSL; Amber itself accepts HLSL and GLSL source in `SHADER` blocks directly (section 1), choosing compilers internally [web: Amber builds against glslang/shaderc; which is used for HLSL not read].
- [src] Which can emit GLSL ES 3.00 for the web from one source: SPIRV-Cross (`--es --version 300`, with the fragment precision options above), naga (GLSL ES 300+), HLSLcc (GLSL ES from DXBC; `ES 3.0+` per README), Slang (not documented).
- [web] WGSL for the web: naga and Tint write WGSL (naga: "secondary"); Slang WGSL target "work in progress".
## 8. What each path exercises (stated as facts about the scope of each check)
| Path | Executes | Compiler/translator in the chain | Float rules in force | What a pass establishes | What it does not |
|---|---|---|---|---|---|
| Amber on lavapipe | SPIR-V on Mesa llvmpipe JIT (CPU) | glslang or dxc/other to SPIR-V, then Mesa NIR/LLVM | Vulkan spec; GLSL-derived SPIR-V denorm handling per Mesa; FMA contraction per Mesa/LLVM | The shader source, as compiled to SPIR-V, computes within the stated tolerance on a Vulkan 1.3 conformant implementation | Behaviour of ANGLE's HLSL, fxc/D3DCompile, NVIDIA D3D11 |
| Amber on SwiftShader | SPIR-V on SwiftShader's JIT (Reactor/Subzero) | same front end | Vulkan spec; float controls reported off (older commit) | same, on a second conforming implementation | same |
| Amber on a real Vulkan GPU | SPIR-V on the driver | same | vendor | The SPIR-V on that driver | WebGL path |
| glslang/SPIRV-Cross text snapshot tests | nothing executed | translators only | none | translation is stable between versions | numerics |
| ANGLE `angle_end2end`/dEQP on D3D11 | GLSL ES via ANGLE translator, HLSL, `D3DCompile` (fxc), NVIDIA D3D11 driver, as in Chrome on Windows | ANGLE as built in the harness (the same codebase Chrome vendors) | D3D11 rules (flush to zero, 1 ULP) over ESSL rules | The ES shader behaviour of that ANGLE build on that driver | Chrome's own flags/feature overrides unless reproduced |
| ANGLE on SwiftShader (`angle-swiftshader`) | GLSL ES via ANGLE to SPIR-V, SwiftShader | ANGLE Vulkan backend | Vulkan | ANGLE's Vulkan translation path, software | ANGLE's D3D11 path |
| D3D11/12 compute on WARP | HLSL via dxc/fxc, WARP CPU rasterizer | Microsoft compiler | D3D rules as implemented by WARP | The HLSL source on a D3D conformant software device | NVIDIA hardware or the WebGL route |
| D3D11/12 compute on GTX 1070 | HLSL via dxc/fxc, NVIDIA driver | | D3D rules, vendor | The HLSL source on that GPU | other vendors |
| wgpu/naga on lavapipe or WARP | WGSL (or SPIR-V in) via naga | naga | WGSL/Vulkan or D3D12 | naga's translation of that source on that device | ANGLE or glslang/dxc paths |
| Slang CPU | Slang to C++ or LLVM, run on the host CPU | Slang | host CPU IEEE with the host compiler's flags (page silent) | Slang source semantics on a CPU | HLSL or GLSL as consumers compile them |
| GLSL-to-CPU emulators/interpreters | the interpreter | the interpreter | the interpreter's host floats | the interpreted source | any real driver |
| Browser WebGL2 (earlier file) | GLSL ES via browser's ANGLE | | | the exact consumer path on that browser/GPU | other browsers/drivers |
- [src] Spec positions that bound what a conforming-implementation pass can claim: GLSL ES 3.00/3.10 allow flush of subnormals to zero at any time, undefined rounding, `a*b+c` either fused or two roundings, and undefined trig precision; D3D11 requires flushing subnormals on math input/output and allows fused ops up to the unfused tolerance; Vulkan lets an implementation flush unless a preserve mode is requested and supported.
- [src] Where the same GLSL ES source goes through ANGLE→HLSL→`D3DCompile`→driver, the compiler flags in `Renderer11.cpp` (OPTIMIZATION_LEVEL2, retry configurations, conditional IEEE_STRICTNESS) are part of the path; a SPIR-V route (glslang or dxc, then a Vulkan implementation) has none of those stages.
- [abs] No source found that quantifies disagreement between ANGLE→D3D11 and a Vulkan software implementation for the same GLSL ES function.
## 9. Gaps (searched, not resolved)
- Lavapipe and SwiftShader current `VkPhysicalDeviceFloatControlsProperties` values; FMA contraction policy in either; the lavapipe docs page.
- Amber's own CI jobs and which compilers it uses for HLSL and GLSL.
- Whether `dxc -spirv` can emit an exported-function library that SPIRV-Cross's library path accepts; whether `dxc` flush/preserve execution modes are emitted by default.
- Tint's current readers and writers.
- Slang GLSL ES output; Slang CPU float flags.
- ANGLE: default of `enableIEEEStrictness`; prebuilt standalone `libEGL`/`libGLESv2` availability for Windows outside Chrome/Electron; Chrome-specific translator options.
- WARP numerics statement; DXC test tolerance values per intrinsic row (the XML table structure was read, not every row).

31
docs/research/serializer-unowned-holders.md Normal file
View file

@ -0,0 +1,31 @@
# Prior art: refuse or walk a container the identity walk does not own
For `variants:question:c2a-unowned-holders` (campaign `variants`, cut `c2a-element-id-holes` r5).
These are from Imagination's reading. No web fetch was made in this pass, so check the
pointers before quoting them outside the mind.
| System | Unknown or unsupported type in a payload | Extension point | Pointer |
|---|---|---|---|
| Protocol Buffers | Not expressible. A field is a scalar, a message, `repeated`, or `map<scalar, V>`. No other container exists. | None. You model the data as messages. | protobuf.dev, Language Guide (proto3), "Repeated fields" and "Maps" |
| Microsoft Bond | Fixed container set: `list`, `vector`, `set`, `map`, `nullable`, `blob`, `bonded`. Any other type is a codegen error. | Type aliases with explicit converters, declared per type. | microsoft.github.io/bond/manual/bond_cs.html, "Type aliases" and "Custom type mappings" |
| Orleans 7+ serializer | Throws `CodecNotFoundException` when it serializes a type with no codec (not `[GenerateSerializer]`, not registered). | Surrogates via `[RegisterConverter]`, or codecs registered per type. Built-in codecs cover the BCL immutable collections. | learn.microsoft.com/dotnet/orleans/host/configuration-guide/serialization |
| MessagePack-CSharp 3.1.7 | `DynamicObjectResolver` refuses an unannotated type (formatter null, the serializer throws) unless the caller opts into contractless. | Resolvers, `[MessagePackFormatter]`. | Probed: `scratchpad/imag-r5/probe1.log` (NkHold: std=null, contractless builds one) |
| System.Text.Json (source gen) | `NotSupportedException` for types outside the generated metadata. Polymorphism needs `[JsonDerivedType]` registration. | `JsonConverter<T>`, `[JsonDerivedType]`. | learn.microsoft.com/dotnet/standard/serialization/system-text-json/polymorphism |
## What the table says for CultCache's id walk
1. Every system that guarantees something about a payload's structure fails closed on a
shape it does not own. The failure is loud and comes at build or first use. It never
drops data silently. Protobuf and Bond refuse when the schema is written, and Orleans,
MessagePack and STJ refuse when the type is first serialized. None of them walks an
arbitrary wrapper on the chance that it is a container.
2. Each system supports a fixed set of containers and walks those natively. Extension is
explicit and per type: Bond aliases, Orleans surrogates, STJ converters. The walker
never infers it from a type's shape.
3. Orleans covers the BCL immutable collections with built-in codecs. Each added container
is a deliberate, tested entry. A reasonable consumer therefore expects ImmutableArray to
be supported when the library says so, and refused loudly when it does not. Nobody
expects silent nil ids.
CultCache's id walk today is open-world: a shape it does not recognise passes as a leaf.
That is the opposite of every row above, and it is the common cause of findings s1-s4.

111
docs/research/shader-variable-inputs.md Normal file
View file

@ -0,0 +1,111 @@
# Shader variable-length inputs: how shader libraries take a span, and what CultMath's build route allows
For `cultmath-tapes:follow_up:shader-weighted-lattice-loop` and `cultmath-tapes:follow_up:lattice-shader-pins` (campaign `cultmath-tapes`).
Imagination pass, agent `imagination-ag4-shader-loop`, session `self-2026-10-16-ag4`, 2026-10-17, against CultLib `origin/main` `e2051c5a`
and the build branch `hands/cultmath-glsl-build-dxc` `23ac4e89`. Probes are in `docs/research/shader-variable-inputs/`.
Tags: [src] primary source text read this pass; [web] search summary only; [probe] run this pass, output quoted below.
## 1. The question
C#'s `weighted_lattice_cells` reads the body span once per lattice cell (the weight is the exact power border of the
cell centre), and the ring search around it is the part a consumer cannot safely write by hand: the exit bound,
the two passes and the box-independent fold order. The shader port has the four steps only. A shader function cannot
take `ReadOnlySpan`, the library targets GLSL ES 3.00 fragment shaders (WebGL2: no storage buffers), and the GLSL is
built from the HLSL by dxc `lib_6_3` SPIR-V plus SPIRV-Cross (ruling `glsl-es-via-dxc-spirv-cross`).
So how do bodies reach a shader `weighted_lattice_cells`?
## 2. Prior art: how shader code that ships as a library takes a variable number of inputs
- [src] Unity URP (`ShaderLibrary/Input.hlsl`, `RealtimeLights.hlsl`, Graphics `master`): a fixed capacity
`MAX_VISIBLE_LIGHTS` chosen per platform (`MAX_VISIBLE_LIGHT_COUNT_LOW_END_MOBILE` under `SHADER_API_GLES30`,
`_MOBILE` for GLES3/WebGPU, `_DESKTOP` otherwise), constant-buffer arrays `float4 _AdditionalLightsPosition[MAX_VISIBLE_LIGHTS]`
and the like, with `StructuredBuffer<LightData>` only behind `USE_STRUCTURED_BUFFER_FOR_LIGHT_DATA` (defined 0). The
count is a uniform read through `GetAdditionalLightsCount()`, and `GetAdditionalLight(uint i, float3 positionWS)` is
the accessor; `LIGHT_LOOP_BEGIN(lightCount)` / `LIGHT_LOOP_END` macros hide the cluster versus plain loop. The
pipeline owns the storage and the accessor; user shaders call them. Capacity plus count, storage owned by the include.
- [src] Filament (`shaders/src/surface_light_punctual.fs`, `main`): the loop is the engine's, over a froxel record
(`froxelsUniforms.records`, then `froxelRecordUniforms.records`) giving `recordOffset` and `count`; each light is
`lightsUniforms.lights[lightIndex]`, a uniform-block array. Material code never sees the loop. The engine owns both
the storage and the loop because it also owns the uniform layout.
- [src] three.js (`ShaderChunk/lights_fragment_begin.glsl.js`, `dev`): `for ( int i = 0; i < NUM_POINT_LIGHTS; i ++ )`
under `#pragma unroll_loop_start`, reading `pointLights[ i ]`; `NUM_POINT_LIGHTS` is a define the renderer writes,
so the shader is regenerated per light count. Count as a compile-time constant; the library is source text.
- [web] Bevy: clustered forward needs storage buffers; on WebGL2 it falls back to a uniform buffer with a capped
object count (`MAX_UNIFORM_BUFFER_CLUSTERABLE_OBJECTS` 204 in an older `bevy_pbr`, asserted to fit 16384 bytes; the
Many Lights example says 256 lights under `WGPU_SETTINGS_PRIO=webgl2`). Capacity plus count where storage is absent.
- [web] lygia (README conventions, via search): reuse by `#ifndef` / `#define` options the caller sets before the
include (`SAMPLER_FNC(TEX, UV)`, `MYFUNC_TYPE`, per-function sampler overrides). Caller-supplied accessor macros; the
library ships as source text, which is what lets a macro reach inside a function body.
The pattern that splits them: a caller-supplied accessor (lygia, URP's accessor functions, three.js's define) works
where the library is source text the caller's preprocessor sees. Where the library is compiled or its storage layout is
fixed (URP's GLES path, Bevy's WebGL2 fallback, Filament's uniform blocks), the input is a fixed-capacity array plus a
count, and the capacity is a named library constant.
## 3. What the build route allows (probes)
| Candidate | HLSL (dxc) | Library SPIR-V and SPIRV-Cross GLSL | C# mirror (`TransformHlslToCSharp`) | Verdict |
|---|---|---|---|---|
| Fixed-size array parameter plus count | compiles, `lib_6_3` and `cs_6_0` [probe P1, P3] | see P2 | needs one transform (`T x[N]` parameter to `T[] x`) | see section 4 |
| Caller accessor macro (`CULTMATH_BODY(i)`) | the library build defines no accessor, so the function is absent [probe P4] | absent from the built GLSL: SPIRV-Cross output carries no consumer macro | the mirror would need the macro too | refused by the route |
| Bodies packed into library-owned constants | a library cbuffer or uniform | a uniform block in a function-only library; a binding the consumer does not own | a static field | refused (P5, by construction, not run): a library may not own a consumer's binding |
Probes (files in `shader-variable-inputs/`):
- [probe] P1, local, dxc 2026_07_29 (`.tools/dxc`): `lattice-array.hlsl` (CultMath.hlsl, `prototype.hlsl`, one export)
compiles with `-T lib_6_3 -HV 2021 -spirv -fspv-target-env=universal1.5 -Od`. The SPIR-V types the array as
`OpTypeArray %v4float %uint_16` behind a Function pointer, and `-Od` copies it at each call (`param_var_bodies`).
- [probe] P3, local: `entry-cbuffer.hlsl` passes a cbuffer array directly and a `StructuredBuffer` copied into a local
array; both compile to SPIR-V (`cs_6_0`) and to DXIL.
- [probe] P4, local: `macro-accessor.hlsl` with no `CULTMATH_BODY` defined fails, `use of undeclared identifier`: a
macro-parameterised function does not exist in a library compiled without the caller's macros, so the built GLSL
cannot carry it. Shipping it would need a hand-kept GLSL text beside the build, a second owner the ruling removed.
- [probe] P2, Yggdrasil job (`probe.sh` at `23ac4e89`, sdk:10.0, 2026-10-17): with the prototype appended to
`CultMath.hlsl` and two build-tool edits (`wrappers.py` strips `[N]` from an argument name; `partition.py` accepts
`static const int`), `build-glsl.sh` is green ("build shape ok", "suffixed ids: 0; inout: 0") and emits
`const int CULTMATH_MAX_SITES = 16;`, `vec4 cultmath_power_border(vec3 p, vec4 sites[16], int count)` and
`vec4 cultmath_weighted_lattice_cells(vec2 p, float time, vec4 bodies[16], int count, ...)`. A `#version 300 es`
fragment consumer passing `uniform vec4 zone[CULTMATH_MAX_SITES]` compiles with glslang.
- [probe] Same job, lavapipe (`llvmpipe (LLVM 20.1.2, 256 bits)`), 257 cases from `ref/Program.cs` (the ring-17 point
of `TheCapIsReachedAndMatchesTheFullBox` and 7 neighbours, four looks, five normals including (3e-30, -4e-30) and
(1e30, 2e30), a zero normal at stretch 1, counts 0, 1, 3, 6, 16), judged against C#'s bits at 2.2e-6 max(|v|, 1):
| Variant (HLSL route unless named) | bit-exact | over bound | worst |
|---|---|---|---|
| built GLSL | 237 | 0 | 1.19e-6 |
| HLSL (equal to built GLSL bit for bit) | 237 | 0 | 1.19e-6 |
| M1 exit at `0.5 k` | 236 | 1 | 4.28e-3 |
| M2 exit ignores the smooth band (`>= f2`) | 226 | 13 | 8.33e-2 |
| M3 normaliser drops the divide by the largest component | 148 | 96 | 1e30 |
| M4 normaliser drops the divide by length | 1 | 208 | 0.259 |
| M5 smooth step loses its `min` cap | 153 | 85 | 2.26e-3 |
| M6 box pass skips its first row | 228 | 9 | 0.807 |
| M7 exit one ring late (`0.5 (k - 2)`) | 237 | 0 | 1.19e-6 |
M7 is equivalent: the result does not depend on the box (the proof above `weighted_lattice_cells`), so a late exit
costs rings and changes nothing. M1 is killed by one case of 257, the ring-17 point: an early exit only matters where
a far ring wins, so the device family must carry several such points.
## 4. Decision
The fixed-size array plus count is the only candidate the route admits, and it is the shape the compiled-library
prior art uses (URP's GLES path, Bevy's WebGL2 fallback, Filament's uniform blocks). The capacity is a named library
constant, `CULTMATH_MAX_SITES`, shared by every span-taking shader function. Its value is
`cultmath-tapes:question:shader-site-capacity` (Aetheria zones: p50 17, p99 57, max 80 bodies, gravity map B28).
The shader recomputes each distance in the box pass instead of caching it: C#'s 75 x 75 stack cache is not a shader
option, and recomputing gives the same value, as the probe's bit-exact HLSL and built-GLSL agreement shows.
Reflection cannot pass a `ReadOnlySpan`, so the mirror and the fixture reach the C# span functions through one
explicit adapter table rather than `MethodInfo.Invoke`.
## 5. lattice-shader-pins
- The smooth-step family draws m in [f1, f1 + smoothing), so the cap never binds. The cap binds when m = f1 + smoothing,
d = f1 + x smoothing with x around 1.3 to 1.9, and smoothing above about 0.25: next - m = (d - m) h - h (1 - h)
smoothing / (1 + 3 smoothing) is positive there (for smoothing 1 and d - m = 0.5, h = 0.156 and next - m = +0.045).
M5 above is killed by the whole function already; the step family should kill it on its own.
- Platform values: `8b31e47e` rewrote 7 platform-family entries of `glsl-parity.json` from "Ubuntu 24.04.5 LTS" to
"Microsoft Windows 10.0.26200", while `7ddb1040` recorded the lavapipe bounds on Yggdrasil against that fixture.
The bounds, the lavapipe job and the build route all run in Yggdrasil's sdk:10.0 Ubuntu image, so that image
owns the fixture's platform values too. A write on another OS is refused rather than reviewed.

15
docs/research/shader-variable-inputs/entry-cbuffer.hlsl Normal file
View file

@ -0,0 +1,15 @@
// Probe C: a compute entry passing a constant-buffer array, and one passing a StructuredBuffer copied to a local array.
#include "lattice-array.hlsl"
cbuffer Zone : register(b0) { float4 zoneBodies[CULTMATH_MAX_SITES]; int zoneCount; };
StructuredBuffer<float4> bufferBodies : register(t1);
RWStructuredBuffer<float4> result : register(u2);
[numthreads(64, 1, 1)]
void main(uint3 id : SV_DispatchThreadID)
{
float2 p = float2(id.x * 7.0, id.y * 11.0);
float4 local[CULTMATH_MAX_SITES];
for (int i = 0; i < CULTMATH_MAX_SITES; i++)
local[i] = i < zoneCount ? bufferBodies[i] : float4(0.0, 0.0, 0.0, 1.0);
result[id.x] = cultmath_weighted_lattice_cells(p, 1.0, zoneBodies, zoneCount, 260.0, 0.3, 1.6, 120.0, 0.35, float2(1.0, 2.0), 1.5)
+ cultmath_weighted_lattice_cells(p, 1.0, local, zoneCount, 260.0, 0.3, 1.6, 120.0, 0.35, float2(1.0, 2.0), 1.5);
}

16
docs/research/shader-variable-inputs/entry.comp.tail Normal file
View file

@ -0,0 +1,16 @@
layout(local_size_x = 64) in;
layout(std430, binding = 0) readonly buffer IB { uint a[]; } In;
layout(std430, binding = 1) buffer OB { uint a[]; } Out;
void main()
{
uint i = gl_GlobalInvocationID.x; if (i >= NCASES) return; uint b = i * 76u;
vec4 bodies[CULTMATH_MAX_SITES];
for (int k = 0; k < CULTMATH_MAX_SITES; k++)
bodies[k] = uintBitsToFloat(uvec4(In.a[b + 2u + 4u * uint(k)], In.a[b + 3u + 4u * uint(k)], In.a[b + 4u + 4u * uint(k)], In.a[b + 5u + 4u * uint(k)]));
uint c = b + 67u;
vec4 r = cultmath_weighted_lattice_cells(uintBitsToFloat(uvec2(In.a[b], In.a[b + 1u])), uintBitsToFloat(In.a[c]), bodies, int(In.a[b + 66u]),
uintBitsToFloat(In.a[c + 1u]), uintBitsToFloat(In.a[c + 2u]), uintBitsToFloat(In.a[c + 3u]), uintBitsToFloat(In.a[c + 4u]),
uintBitsToFloat(In.a[c + 5u]), uintBitsToFloat(uvec2(In.a[c + 6u], In.a[c + 7u])), uintBitsToFloat(In.a[c + 8u]));
uvec4 w = floatBitsToUint(r);
Out.a[i * 4u] = w.x; Out.a[i * 4u + 1u] = w.y; Out.a[i * 4u + 2u] = w.z; Out.a[i * 4u + 3u] = w.w;
}

16
docs/research/shader-variable-inputs/entry.hlsl Normal file
View file

@ -0,0 +1,16 @@
// Probe compute entry (HLSL route): 76 words in per case (p, 16 bodies, count, 9 parameters), 4 words out.
#include "CultMath.hlsl"
[[vk::binding(0)]] StructuredBuffer<uint> In;
[[vk::binding(1)]] RWStructuredBuffer<uint> Out;
[numthreads(64, 1, 1)] void main(uint3 id : SV_DispatchThreadID)
{
uint i = id.x; if (i >= NCASES) return; uint b = i * 76u;
float4 bodies[CULTMATH_MAX_SITES];
for (int k = 0; k < CULTMATH_MAX_SITES; k++)
bodies[k] = float4(asfloat(In[b + 2 + 4 * k]), asfloat(In[b + 3 + 4 * k]), asfloat(In[b + 4 + 4 * k]), asfloat(In[b + 5 + 4 * k]));
uint c = b + 67u;
float4 r = cultmath_weighted_lattice_cells(float2(asfloat(In[b]), asfloat(In[b + 1])), asfloat(In[c]), bodies, asint(In[b + 66]),
asfloat(In[c + 1]), asfloat(In[c + 2]), asfloat(In[c + 3]), asfloat(In[c + 4]), asfloat(In[c + 5]),
float2(asfloat(In[c + 6]), asfloat(In[c + 7])), asfloat(In[c + 8]));
Out[i * 4] = asuint(r.x); Out[i * 4 + 1] = asuint(r.y); Out[i * 4 + 2] = asuint(r.z); Out[i * 4 + 3] = asuint(r.w);
}

14
docs/research/shader-variable-inputs/judge.py Normal file
View file

@ -0,0 +1,14 @@
#!/usr/bin/env python3
"""Compare a device output with C#'s reference bits: cases over the lavapipe bound 2.2e-6 max(|v|, 1), and the largest.
usage: judge.py <ref.bin> <out.bin> <label>"""
import math, struct, sys
ref, out = open(sys.argv[1], 'rb').read(), open(sys.argv[2], 'rb').read()
n = len(ref) // 16
r, o = struct.unpack('<%df' % (4 * n), ref), struct.unpack('<%df' % (4 * n), out)
over, exact, worst = 0, 0, 0.0
for i in range(n):
s = max((abs(o[4*i+j] - r[4*i+j]) / max(abs(r[4*i+j]), 1.0) if math.isfinite(o[4*i+j]) else math.inf) for j in range(4))
exact += ref[16*i:16*i+16] == out[16*i:16*i+16]
over += s > 2.2e-6
worst = max(worst, s)
print('%s: cases %d, bit-exact %d, over 2.2e-6 %d, worst %.3g' % (sys.argv[3], n, exact, over, worst))

7
docs/research/shader-variable-inputs/lattice-array.hlsl Normal file
View file

@ -0,0 +1,7 @@
// Probe A (local dxc): the prototype as a lib_6_3 SPIR-V library export, bodies as a fixed-size array parameter plus a count.
#include "CultMath.hlsl"
#include "prototype.hlsl"
export float4 probe_lattice(float2 p, float4 bodies[CULTMATH_MAX_SITES], int count)
{
return cultmath_weighted_lattice_cells(p, 1.0, bodies, count, 260.0, 0.3, 1.6, 120.0, 0.35, float2(1.0, 2.0), 1.5);
}

12
docs/research/shader-variable-inputs/macro-accessor.hlsl Normal file
View file

@ -0,0 +1,12 @@
// Probe B: bodies through a caller-supplied accessor macro. The library build defines no accessor.
#include "CultMath.hlsl"
#ifdef CULTMATH_BODY
float4 cultmath_power_border_m(float3 p)
{
float4 border = float4(0.0, 0.0, 0.0, 1.0e30);
for (int k = 1; k < CULTMATH_BODY_COUNT; k++)
border = cultmath_power_border_step(p, CULTMATH_BODY(0), CULTMATH_BODY(k), border);
return border;
}
#endif
export float4 probe_macro(float3 p) { return cultmath_power_border_m(p); }

55
docs/research/shader-variable-inputs/probe.sh Normal file
View file

@ -0,0 +1,55 @@
#!/usr/bin/env bash
# Probe for shader-weighted-lattice-loop: the prototype through the dxc + SPIRV-Cross build and on lavapipe, against C#.
# Run from a CultLib clone at a revision with packages/cultmath/tools/glsl/build-glsl.sh (hands/cultmath-glsl-build-dxc
# 23ac4e89), inside mcr.microsoft.com/dotnet/sdk:10.0, with this directory at $P. The clone is disposable: the
# prototype is appended to its CultMath.hlsl, and two build-tool lines are patched (array arguments, int constants).
set -euo pipefail
P=${P:-/tmp/probe}; ROOT=$(pwd); CM=$ROOT/packages/cultmath; W=/tmp/pw; mkdir -p $W
apt-get update -qq >/dev/null && apt-get install -y -qq --no-install-recommends python3 >/dev/null
cat $P/prototype.hlsl >> $CM/shaders/CultMath.hlsl
python3 - $CM/tools/glsl/wrappers.py <<'PY'
import sys; f = sys.argv[1]; t = open(f).read()
t = t.replace(r"args = [re.split(r'\s+', p)[-1] for p in params]", r"args = [re.split(r'\s+', re.sub(r'\[.*\]', '', p))[-1] for p in params]")
open(f, 'w').write(t)
PY
grep -c "re.sub(r'" $CM/tools/glsl/wrappers.py
python3 - $CM/tools/glsl/partition.py <<'PY'
import sys; f = sys.argv[1]; t = open(f).read()
t = t.replace(r"r'^static const float (CULTMATH_\w+) = (-?\d+(?:\.\d*)?(?:[eE][+-]?\d+)?);$'", r"r'^static const (float|int) (CULTMATH_\w+) = (-?\d+(?:\.\d*)?(?:[eE][+-]?\d+)?);$'")
t = t.replace("consts[o].append('const float %s = %s;' % m.groups())", "consts[o].append('const %s %s = %s;' % m.groups())")
open(f, 'w').write(t)
PY
grep -c "(float|int)" $CM/tools/glsl/partition.py
echo "== build"
bash $CM/tools/glsl/build-glsl.sh $W/built 2>&1 | tail -12
echo "== array functions in the built core.glsl"
grep -nE "^(const int CULTMATH_MAX_SITES|vec4 cultmath_power_border\(|float cultmath_lattice_cell_distance\(|vec4 cultmath_weighted_lattice_cells\()" $W/built/core.glsl
echo "== ES 3.00 fragment consumer passing a uniform array"
{ printf '#version 300 es\nprecision highp float;\nprecision highp int;\n'; cat $W/built/core.glsl
printf 'uniform vec4 zone[CULTMATH_MAX_SITES];\nuniform int zoneCount;\nout highp vec4 c;\nvoid main() { c = cultmath_weighted_lattice_cells(gl_FragCoord.xy, 1.0, zone, zoneCount, 260.0, 0.7, 1.6, 120.0, 0.35, vec2(0.6, 0.8), 1.5); }\n'; } > $W/consumer.frag
glslangValidator $W/consumer.frag && echo "consumer ES 3.00 ok"
echo "== lavapipe"
apt-get install -y -qq --no-install-recommends mesa-vulkan-drivers libvulkan-dev gcc libc6-dev >/dev/null
gcc -O2 -o $W/vkrun $CM/tools/device/vkrun.c -lvulkan
cp -r $P/ref $W/ref && sed -i 's#../../../../packages#'"$CM"'/..#' $W/ref/Ref.csproj
N=$(dotnet run --project $W/ref -c Release -- $W/in.bin $W/ref.bin | tail -1); echo "cases $N"
DXC=$(find $CM/.tools/glsl/dxc -name dxc -type f | head -1); export LD_LIBRARY_PATH=$(dirname $DXC)/../lib:$(dirname $DXC)
{ printf '#version 310 es\nprecision highp float;\nprecision highp int;\n#define NCASES %su\n' $N; cat $W/built/core.glsl $P/entry.comp.tail; } > $W/e.comp
glslangValidator -V --target-env vulkan1.2 -o $W/g.spv $W/e.comp >$W/g.log || { head $W/g.log; exit 1; }
$W/vkrun $W/g.spv $W/in.bin $W/g.out $N 4
python3 $P/judge.py $W/ref.bin $W/g.out built-glsl
run_hlsl() { # $1 label, $2 sed expression applied to a copy of the shaders
rm -rf $W/s && cp -r $CM/shaders $W/s && sed -i "$2" $W/s/CultMath.hlsl
$DXC -T cs_6_0 -E main -HV 2021 -spirv -fspv-target-env=vulkan1.2 -D NCASES=${N}u -I $W/s $P/entry.hlsl -Fo $W/h.spv >$W/h.log 2>&1 || { head -5 $W/h.log; exit 1; }
$W/vkrun $W/h.spv $W/in.bin $W/h.out $N 4 >/dev/null
python3 $P/judge.py $W/ref.bin $W/h.out "$1"
}
run_hlsl hlsl 's/^$//'
cmp $W/h.out $W/g.out && echo "hlsl equals built glsl bit for bit" || echo "hlsl and built glsl differ (plain form; the gate is strict form)"
run_hlsl "M1 exit at 0.5k" 's/0\.5 \* (k - 1) >= max/0.5 * k >= max/'
run_hlsl "M2 exit ignores the smooth band" 's/>= max(f2, f1 + 2\.0 \* smoothing))$/>= f2)/'
run_hlsl "M3 normaliser drops the pre-scale" 's/normal = stretchNormal \/ scale;/normal = stretchNormal;/'
run_hlsl "M4 normaliser drops the length" 's/normal = normal \/ length(normal);//'
run_hlsl "M5 smooth step loses its cap" 's/return min(next, f1 + smoothing);/return next;/'
run_hlsl "M6 box pass skips its first row" 's/for (int dz = -reach; dz <= reach; dz++)/for (int dz = -reach + 1; dz <= reach; dz++)/'
run_hlsl "M7 late exit (equivalent: window-independent)" 's/0\.5 \* (k - 1) >= max/0.5 * (k - 2) >= max/'

75
docs/research/shader-variable-inputs/prototype.hlsl Normal file
View file

@ -0,0 +1,75 @@
// The prototype, appended to CultMath.hlsl by probe.sh: the whole weighted_lattice_cells with bodies as a fixed-size array and a count.
static const int CULTMATH_MAX_SITES = 16;
float4 cultmath_power_border(float3 p, float4 sites[CULTMATH_MAX_SITES], int count)
{
CultSiteCells cells = cultmath_site_cells_empty();
for (int i = 0; i < count; i++)
cells = cultmath_site_cells_insert(cells, cultmath_power_score(p, sites[i]), i);
float4 border = float4(0.0, 0.0, 0.0, 1.0e30);
int owner = (int)cells.nearestSite;
if (owner < 0)
return border;
for (int k = 0; k < count; k++)
if (k != owner)
border = cultmath_power_border_step(p, sites[owner], sites[k], border);
return border;
}
float cultmath_lattice_cell_distance(float2 q, float2 cell, float time, float drift, float4 bodies[CULTMATH_MAX_SITES], int count,
float cellSize, float boost, float falloff, float2 normal, float stretch)
{
float3 site = cultmath_lattice_site(cell, time, drift);
float border = cultmath_power_border(float3((cell + 0.5) * cellSize, 0.0), bodies, count).w;
return cultmath_lattice_distance(q, site.xy, cultmath_lattice_weight(site.z, border, boost, falloff), normal, stretch);
}
float4 cultmath_weighted_lattice_cells(float2 p, float time, float4 bodies[CULTMATH_MAX_SITES], int count, float cellSize, float drift,
float boost, float falloff, float smoothing, float2 stretchNormal, float stretch)
{
float scale = max(abs(stretchNormal.x), abs(stretchNormal.y));
float2 normal = float2(1.0, 0.0);
if (scale > 0.0)
{
normal = stretchNormal / scale;
normal = normal / length(normal);
}
float2 q = p / cellSize;
float2 origin = floor(q);
int cx0 = (int)origin.x;
int cz0 = (int)origin.y;
float f1 = 1.0e30;
float f2 = 1.0e30;
int reach = 0;
for (int k = 0; k <= 37; k++)
{
if (k > 1 && 0.5 * (k - 1) >= max(f2, f1 + 2.0 * smoothing))
break;
for (int dz = -k; dz <= k; dz++)
{
int step = dz == -k || dz == k || k == 0 ? 1 : 2 * k;
for (int dx = -k; dx <= k; dx += step)
{
float d = cultmath_lattice_cell_distance(q, float2(cx0 + dx, cz0 + dz), time, drift, bodies, count, cellSize, boost, falloff, normal, stretch);
if (d < f1) { f2 = f1; f1 = d; }
else if (d < f2) f2 = d;
}
}
reach = k;
}
float m = f1 + smoothing;
float best = 1.0e30;
float second = 1.0e30;
int winDx = 0;
int winDz = 0;
for (int dz = -reach; dz <= reach; dz++)
for (int dx = -reach; dx <= reach; dx++)
{
float d = cultmath_lattice_cell_distance(q, float2(cx0 + dx, cz0 + dz), time, drift, bodies, count, cellSize, boost, falloff, normal, stretch);
if (d < best) { second = best; best = d; winDx = dx; winDz = dz; }
else if (d < second) second = d;
m = cultmath_lattice_smooth_step(m, d, f1, smoothing);
}
int4 id = cultmath_pcg4d(int4(cx0 + winDx, cz0 + winDz, 0, 0));
return float4(best, second, m, (float)((uint)id.w >> 8) * (1.0 / 16777216.0));
}

50
docs/research/shader-variable-inputs/ref/Program.cs Normal file
View file

@ -0,0 +1,50 @@
// Probe reference: writes the lattice cases (76 words each) and C#'s weighted_lattice_cells bits (4 words each).
// usage: Ref <in.bin> <ref.bin>; prints the case count.
using CultMath;
using static CultMath.math;
const int Max = 16;
var cases = new List<(float2 p, float t, float4[] b, float cell, float drift, float boost, float falloff, float s, float2 n, float st)>();
float4[] Bodies(float t, int count)
{
var b = new float4[count];
for (var i = 0; i < count; i++)
{
var a = 0.001f * (i + 1) * t + i; var r = 300.0f + 220.0f * i;
b[i] = new float4(r * MathF.Cos(a), r * MathF.Sin(a), 0.0f, 40.0f + 40.0f * i);
}
return b;
}
// The ring-17 point of LatticeCellsTests.TheCapIsReachedAndMatchesTheFullBox, and its neighbours.
for (var j = 0; j < 8; j++)
cases.Add((new float2(1446.7345f + 3.0f * j, -680.7637f), 97.69269f, Bodies(97.69269f, 6), 260.0f, 0.7f, 4.0f, 1.0e6f, 1.0f, new float2(0.6f, 0.8f), 2.0f));
var looks = new[] { (1.6f, 1.0f, 0.35f), (1.6f, 2.0f, 0.35f), (4.0f, 2.0f, 1.0f), (0.0f, 1.0f, 0.05f) };
var normals = new[] { new float2(0.6f, 0.8f), new float2(3.0f, 4.0f), new float2(3.0e-30f, -4.0e-30f), new float2(1.0e30f, 2.0e30f), new float2(-1.0f, 1.0e-3f) };
var rng = new System.Random(20261017);
float U(float a, float b) => a + (b - a) * (float)rng.NextDouble();
for (var i = 0; i < 248; i++)
{
var (boost, stretch, s) = looks[i % 4];
var n = normals[(i / 4) % normals.Length];
var count = new[] { 6, 0, 1, 16, 6, 3 }[(i / 20) % 6];
var t = U(0.0f, 200.0f);
cases.Add((new float2(U(-3000, 3000), U(-3000, 3000)), t, Bodies(t, count), 260.0f, 0.7f, boost, 120.0f, s, n, stretch));
}
// Zero normal at stretch 1 is (1, 0).
cases.Add((new float2(10.0f, 20.0f), 1.0f, Bodies(1.0f, 6), 260.0f, 0.7f, 1.6f, 120.0f, 0.35f, new float2(0.0f, 0.0f), 1.0f));
using var input = new BinaryWriter(File.Create(args[0]));
using var reference = new BinaryWriter(File.Create(args[1]));
foreach (var c in cases)
{
input.Write(c.p.x); input.Write(c.p.y);
for (var k = 0; k < Max; k++)
{
var v = k < c.b.Length ? c.b[k] : new float4(0.0f, 0.0f, 0.0f, 0.0f);
input.Write(v.x); input.Write(v.y); input.Write(v.z); input.Write(v.w);
}
input.Write(c.b.Length);
foreach (var f in new[] { c.t, c.cell, c.drift, c.boost, c.falloff, c.s, c.n.x, c.n.y, c.st }) input.Write(f);
var r = weighted_lattice_cells(c.p, c.t, c.b, c.cell, c.drift, c.boost, c.falloff, c.s, c.n, c.st);
reference.Write(r.x); reference.Write(r.y); reference.Write(r.z); reference.Write(r.w);
}
Console.WriteLine(cases.Count);

4
docs/research/shader-variable-inputs/ref/Ref.csproj Normal file
View file

@ -0,0 +1,4 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup><OutputType>Exe</OutputType><TargetFramework>net10.0</TargetFramework><Nullable>enable</Nullable><ImplicitUsings>enable</ImplicitUsings></PropertyGroup>
<ItemGroup><ProjectReference Include="../../../../packages/cultmath/src/CultMath/CultMath.csproj" /></ItemGroup>
</Project>

2
docs/semver-policy.md
View file

@ -111,7 +111,7 @@ swap under "Changed" rather than "Breaking".
## Pre-1.0 packages
A package at `0.y.z` (currently `org.gamecult.cultmath`, at `0.4.x`) uses the
A package at `0.y.z` (currently `org.gamecult.cultmath`, at `0.5.x`) uses the
standard pre-1.0 convention: `MAJOR` stays `0` during initial development, so
`MINOR` carries what `MAJOR` means once the package reaches `1.0.0`. Concretely:

24
package-lock.json generated
View file

@ -1184,10 +1184,11 @@
},
"packages/cultcache-ts": {
"name": "@gamecult/cultcache-ts",
"version": "0.14.0",
"version": "0.14.2",
"license": "MIT",
"dependencies": {
"@msgpack/msgpack": "^3.1.3",
"koffi": "3.3.0",
"zod": "^3.25.76"
},
"devDependencies": {
@ -1195,30 +1196,31 @@
"typescript": "^5.9.0"
},
"engines": {
"node": ">=18"
"node": ">=20"
}
},
"packages/cultmesh-browser": {
"version": "0.3.0",
"version": "0.4.0",
"license": "MIT",
"dependencies": {
"@gamecult/cultcache-ts": "^0.14.0",
"cultnet-ts": "^0.2.0"
"cultnet-ts": "^0.3.0"
},
"devDependencies": {
"@types/node": "^24.0.0",
"cultmesh-ts": "^0.2.0",
"typescript": "^5.9.0"
},
"engines": {
"node": ">=18"
"node": ">=20"
}
},
"packages/cultmesh-ts": {
"version": "0.1.1",
"version": "0.2.0",
"license": "MIT",
"dependencies": {
"@gamecult/cultcache-ts": "^0.14.0",
"cultnet-ts": "^0.2.0",
"cultnet-ts": "^0.3.0",
"koffi": "3.3.0"
},
"devDependencies": {
@ -1227,14 +1229,14 @@
"zod": "^3.25.76"
},
"engines": {
"node": ">=18"
"node": ">=20"
}
},
"packages/cultnet-ts": {
"version": "0.2.0",
"version": "0.3.0",
"license": "MIT",
"dependencies": {
"@gamecult/cultcache-ts": "^0.14.0",
"@gamecult/cultcache-ts": "^0.14.1",
"@msgpack/msgpack": "^3.1.3",
"ajv": "^8.17.1"
},
@ -1245,7 +1247,7 @@
"zod": "^3.25.76"
},
"engines": {
"node": ">=18"
"node": ">=20"
}
}
}

4
packages/cultcache-rs/Cargo.toml
View file

@ -1,6 +1,6 @@
[package]
name = "cultcache-rs"
version = "0.2.0"
version = "0.3.0"
edition = "2024"
license = "MIT"
publish = false
@ -22,6 +22,8 @@ redb = "4.1.0"
serde = { version = "1", features = ["derive"] }
serde_bytes = "0.11"
serde_json = "1"
serde_stacker = "0.1.14"
stacker = "0.1.25"
uuid = { version = "1", features = ["v4"] }
[dev-dependencies]

33
packages/cultcache-rs/README.md
View file

@ -456,6 +456,39 @@ That is the Rust version of the original trick:
The derive/registry work should preserve that shape. Its job is to remove
manual registration ceremony, not to loosen the closed-world boundary.
Payload nesting is bounded at 256 MessagePack containers, checked before any
recursion; past the bound the refusal is a typed `PayloadTooDeep`. Typed decode
of stored bytes then grows its stack: the decoder checks the remaining stack
at entry and `serde_stacker` checks it at every container, moving
onto a fresh segment whenever less than 512 KiB remains, so a payload inside the
bound decodes on a small thread instead of aborting the process. Envelope and
store reads (single-file, redb, v1 snapshots) use the same grown decoder and
keep accepting and refusing what they did, without the payload walk.
The guarantee holds for records the derives produce. `MAX_RECORD_INLINE_BYTES`
(16 KiB) is the largest `size_of` a `DatabaseEntry`, `CultSlots` or `CultUnion`
record may have, and the derives refuse a larger one at compile time: put its
large data behind a `Vec` or a `Box`. A field of a type the derives did not
produce that is larger than that, behind a `Box` or `Vec`, is outside the
guarantee. The decoder's own frames have a fixed floor that no payload changes: at the bound a thread decodes a record of
this size from about 56 KiB in a release build and 72 KiB in a debug build for a caller that only tests the result
(`is_ok`), plus the caller's own frames. The figure depends on how the caller holds the result: one that moves the
record out holds a second copy and needs more (about 104 KiB in debug). Stack growth was verified on
x86_64-pc-windows-msvc by hand (`decode_stack`, revision 10a39dac, 13 tests, debug and release); a `stacker` bump or a change
to the stack-size tests repeats it.
The crate does not
build for wasm32 (`fs2`), and no wasm behaviour is claimed.
### Build dependencies
Stack growth adds these crates to every consumer's build, and a C toolchain
(`psm`, `stacker`). On all targets: `serde_stacker` 0.1.14, `stacker` 0.1.25,
`psm` 0.1.32, `cc` 1.8.0, `find-msvc-tools` 0.1.14, `shlex` 2.0.1 (MIT OR
Apache-2.0), `object` 0.39.1 (Apache-2.0 OR MIT) and `ar_archive_writer` 0.5.3
(Apache-2.0 WITH LLVM-exception, build-only). On `x86_64-pc-windows-msvc` also
`libc` 0.2.190 and `windows-sys` 0.61.2 (MIT OR Apache-2.0), beside the 0.59 the
crate uses itself.
## License
Private GameCult infrastructure for now. Public packaging can wait until the

2
packages/cultcache-rs/cultcache-rs-derive/Cargo.toml
View file

@ -1,6 +1,6 @@
[package]
name = "cultcache-rs-derive"
version = "0.1.0"
version = "0.2.0"
edition = "2024"
publish = false

10
packages/cultcache-rs/cultcache-rs-derive/src/lib.rs
View file

@ -275,6 +275,11 @@ fn slot_codec(ident: &syn::Ident, slots: &[CultCacheSlot]) -> proc_macro2::Token
});
quote! {
const _: () = ::core::assert!(
::core::mem::size_of::<#ident>() <= ::cultcache_rs::MAX_RECORD_INLINE_BYTES,
"a CultCache record is at most cultcache_rs::MAX_RECORD_INLINE_BYTES bytes inline; box its largest slot"
);
impl ::cultcache_rs::SlotRecord for #ident {
fn __slots_array() {}
}
@ -509,6 +514,11 @@ fn union_codec(input: DeriveInput) -> syn::Result<proc_macro2::TokenStream> {
});
Ok(quote! {
const _: () = ::core::assert!(
::core::mem::size_of::<#ident>() <= ::cultcache_rs::MAX_RECORD_INLINE_BYTES,
"a CultCache record is at most cultcache_rs::MAX_RECORD_INLINE_BYTES bytes inline; box its largest slot"
);
// An arm is a slots array: a newtype over anything else is refused here, by the
// trait the CultSlots and DatabaseEntry derives implement.
const _: fn() = || {

19
packages/cultcache-rs/examples/cultcache_interop.rs
View file

@ -62,7 +62,7 @@ fn main() -> Result<()> {
let mut args = std::env::args().skip(1);
let mode = args
.next()
.ok_or_else(|| anyhow!("expected mode: write | read"))?;
.ok_or_else(|| anyhow!("expected mode: write | read | hold-lock"))?;
let options = parse_args(args.collect());
let file = require_arg(&options, "file")?;
@ -70,11 +70,28 @@ fn main() -> Result<()> {
"write" => write_note(file, require_arg(&options, "runtime-id")?)?,
"read" => read_note(file)?,
"read-deck" => read_deck(file)?,
"hold-lock" => hold_lock(file, require_arg(&options, "ms")?.parse()?)?,
_ => return Err(anyhow!("unknown mode {mode}")),
}
Ok(())
}
/// Holds the store lock as the library opens it (`<file>.lock`, fs2 exclusive), for `ms`.
fn hold_lock(file: &str, ms: u64) -> Result<()> {
let lock = std::fs::OpenOptions::new()
.create(true)
.read(true)
.truncate(false)
.write(true)
.open(format!("{file}.lock"))?;
fs2::FileExt::lock_exclusive(&lock)?;
println!("{}", json!({ "held": true }));
std::io::Write::flush(&mut std::io::stdout())?;
std::thread::sleep(std::time::Duration::from_millis(ms));
fs2::FileExt::unlock(&lock)?;
Ok(())
}
fn write_note(file: &str, runtime_id: &str) -> Result<()> {
let mut cache = build_cache(file)?;
cache.pull_all_backing_stores()?;

155
packages/cultcache-rs/src/lib.rs
View file

@ -600,12 +600,124 @@ fn element_header(bytes: &[u8], position: usize) -> Option<(usize, Option<usize>
})
}
/// The one decode of a document payload: the nesting bound first, then the deserializer, whose own
/// depth counter (1024) is only a backstop. A failure keeps its kind and never quotes the bytes.
/// Readers of stored envelope payloads outside this crate decode through it too.
/// The stack left below which a decode moves to a fresh segment. Measured (map D10, D12): the
/// distance between two stack checks is 14 times the inline size of the record entered plus 3 KiB in
/// a debug build and 6 times in release, so at [`MAX_RECORD_INLINE_BYTES`] it is 220 KiB and 93 KiB;
/// 512 KiB is 2.3 times the larger. A record the derives produced can never exceed that bound.
const DECODE_RED_ZONE: usize = 512 * 1024;
/// The size of each segment a decode grows onto, four red zones. It does not change the cost of a
/// decode (map D6); it only has to exceed the red zone plus one gap.
const DECODE_STACK_SEGMENT: usize = 2 * 1024 * 1024;
/// The largest `size_of` a record the derives produce may have. `DatabaseEntry`, `CultSlots` and
/// `CultUnion` refuse a larger one at compile time: put its large data behind a `Vec` or a `Box`.
///
/// Decoding builds a record in stack frames that hold it, and the distance between two stack checks
/// grows with its size, so the decode's red zone is set against a record of exactly this size. A
/// field of a type the derives did not produce (a plain serde struct, a large tuple) that exceeds
/// this size behind a `Box` or `Vec` is outside the guarantee.
///
/// ```compile_fail,E0080
/// use cultcache_rs::CultSlots;
///
/// #[derive(CultSlots)]
/// struct TooLarge {
/// #[cultcache(key = 0)]
/// a: [[u64; 32]; 32],
/// #[cultcache(key = 1)]
/// b: [[u64; 32]; 32],
/// #[cultcache(key = 2)]
/// one_more: u8,
/// }
/// ```
///
/// A union is refused by its own size, though every arm is within the limit:
///
/// ```compile_fail,E0080
/// use cultcache_rs::{CultSlots, CultUnion};
///
/// #[derive(CultSlots)]
/// struct Arm {
/// #[cultcache(key = 0)]
/// a: [[u64; 32]; 32],
/// #[cultcache(key = 1)]
/// b: [[u64; 32]; 32],
/// }
///
/// #[derive(CultUnion)]
/// enum TooLarge {
/// #[cultcache(key = 0)]
/// Arm(Arm),
/// #[cultcache(key = 1)]
/// Leaf,
/// }
/// ```
///
/// A record of exactly the limit, a union over a record of exactly the limit's half, and a record
/// under it with a `Vec` for its large data compile:
///
/// ```
/// use cultcache_rs::{CultSlots, CultUnion};
///
/// #[derive(CultSlots)]
/// struct AtTheLimit {
/// #[cultcache(key = 0)]
/// a: [[u64; 32]; 32],
/// #[cultcache(key = 1)]
/// b: [[u64; 32]; 32],
/// }
///
/// #[derive(CultSlots)]
/// struct Half {
/// #[cultcache(key = 0)]
/// a: [[u64; 32]; 32],
/// }
///
/// #[derive(CultUnion)]
/// enum Fits {
/// #[cultcache(key = 0)]
/// Half(Half),
/// #[cultcache(key = 1)]
/// Leaf,
/// }
///
/// #[derive(CultSlots)]
/// struct Boxed {
/// #[cultcache(key = 0)]
/// large: Vec<u8>,
/// #[cultcache(key = 1)]
/// header: [[u64; 32]; 31],
/// }
/// ```
pub const MAX_RECORD_INLINE_BYTES: usize = 16 * 1024;
/// The one grown decoder of stored bytes: the outer stack check, then `serde_stacker` re-checks at
/// every container serde's impls descend through, skipped values included. No nesting bound is applied here; [`decode_payload`] walks first.
fn decode_grown<T: serde::de::DeserializeOwned>(bytes: &[u8]) -> std::result::Result<T, rmp_serde::decode::Error> {
stacker::maybe_grow(DECODE_RED_ZONE, DECODE_STACK_SEGMENT, || {
let mut reader = rmp_serde::Deserializer::from_read_ref(bytes);
let mut grown = serde_stacker::Deserializer::new(&mut reader);
grown.red_zone = DECODE_RED_ZONE;
grown.stack_size = DECODE_STACK_SEGMENT;
T::deserialize(grown)
})
}
/// The one decode of a document payload: the nesting bound first, then [`decode_grown`], whose
/// rmp-serde depth counter (1024) is only a backstop. A failure keeps its kind and never quotes the
/// bytes. Readers of stored envelope payloads outside this crate decode through it too.
///
/// A payload inside the bound decodes on any thread that has the decoder's fixed floor, which no payload
/// changes: at the bound, about 56 KiB in release and 72 KiB in debug for a caller that only tests the
/// result (`is_ok`), plus the caller's own frames. The figure depends on how the caller holds the result:
/// one that moves the record out holds a second copy and needs more (about 104 KiB in debug). That holds
/// for every record the derives produce, which [`MAX_RECORD_INLINE_BYTES`] bounds. Stack growth was
/// verified on x86_64-pc-windows-msvc by hand (`decode_stack`, revision 10a39dac, 13 tests, debug and release); a
/// `stacker` bump or a change to the stack-size tests repeats it.
pub fn decode_payload<T: serde::de::DeserializeOwned>(payload: &[u8]) -> Result<T> {
check_payload_depth(payload)?;
rmp_serde::from_slice(payload).map_err(opaque_decode_error)
decode_grown(payload).map_err(opaque_decode_error)
}
/// The nesting bound for an envelope admitted without a typed decode, from the same walk that
@ -625,6 +737,7 @@ fn opaque_decode_error(error: rmp_serde::decode::Error) -> anyhow::Error {
anyhow!(describe_decode_error(&error))
}
/// The text of a decode failure. Field names in it come from the type, never from the bytes.
fn describe_decode_error(error: &rmp_serde::decode::Error) -> String {
use rmp_serde::decode::Error;
match error {
@ -637,11 +750,35 @@ fn describe_decode_error(error: &rmp_serde::decode::Error) -> String {
Error::Syntax(message) | Error::Uncategorized(message) if message.starts_with(RECORD_REFUSAL) => {
return message.clone();
}
Error::Syntax(_) | Error::Uncategorized(_) => "a value does not fit the store format",
Error::Syntax(message) | Error::Uncategorized(message) => {
return describe_serde_message(message);
}
}
.to_string()
}
/// serde's own texts, kept only where they cannot carry a received value: a missing or duplicated
/// field is named by the type's static field name (checked to look like one), an unknown tag and a
/// mistyped value are described by kind. Everything else, `unknown field` included, is opaque.
fn describe_serde_message(message: &str) -> String {
for lead in ["missing field `", "duplicate field `"] {
if let Some(name) = message.strip_prefix(lead).and_then(|rest| rest.strip_suffix('`')) {
let is_name = (1..=64).contains(&name.len())
&& name.bytes().all(|byte| byte.is_ascii_alphanumeric() || byte == b'_');
if is_name {
return message.to_string();
}
}
}
if message.starts_with("unknown variant ") {
"a tag names no variant this type knows".to_string()
} else if message.starts_with("invalid type: ") {
"a value has a MessagePack type the store format does not allow".to_string()
} else {
"a value does not fit the store format".to_string()
}
}
/// What one pass over a file's bytes learned about its header. `first` is the leading slot when it is a MessagePack
/// string (`str`, never `bin`, as in every other runtime), and `None` when the bytes do not open with an array whose first
/// slot is a string (a legacy envelope array candidate). `whole` is the proof that the bytes are exactly one complete
@ -2400,7 +2537,7 @@ fn read_redb_entry(
table
.get(key.as_slice())?
.map(|value| {
rmp_serde::from_slice(value.value())
decode_grown(value.value())
.map_err(opaque_decode_error)
.context("failed to decode redb CultCache envelope")
})
@ -2424,7 +2561,7 @@ fn read_all_redb(
.iter()?
.map(|row| {
let (_, value) = row?;
rmp_serde::from_slice(value.value())
decode_grown(value.value())
.map_err(opaque_decode_error)
.context("failed to decode redb CultCache envelope")
})
@ -3200,13 +3337,13 @@ impl DiskStore {
/// The legacy envelope array: an array of envelope maps. Its acceptance is the one verdict on an array whose first slot
/// is not a store header, for reading it and for replacing it.
fn decode_legacy_envelopes(bytes: &[u8]) -> Result<Vec<CultCacheEnvelope>> {
rmp_serde::from_slice(bytes).map_err(anyhow::Error::from)
decode_grown(bytes).context("failed to decode legacy CultCache envelope array")
}
/// Decodes a store whose header the caller has already read as a readable one: its envelopes and its catalog.
fn decode_store_snapshot(bytes: &[u8]) -> Result<(Vec<CultCacheEnvelope>, Vec<PersistedSchemaCatalogEntry>)> {
let snapshot: PersistedStoreSnapshot =
rmp_serde::from_slice(bytes).context("failed to decode CultCache v1 snapshot")?;
decode_grown(bytes).context("failed to decode CultCache v1 snapshot")?;
// A schema id names the entry that has it as its own id; only an id no entry owns names the entry that lists it as compatible.
let mut catalog = BTreeMap::<String, String>::new();

478
packages/cultcache-rs/tests/decode_stack.rs Normal file
View file

@ -0,0 +1,478 @@
//! Typed decode never aborts a payload inside the nesting bound, whatever the caller's stack, for
//! every record the derives accept (up to `MAX_RECORD_INLINE_BYTES` inline); envelope reads of a
//! store go through the same grown decoder.
//! Each test runs its decode in a child test process on a thread of a stated size, so an abort
//! fails one named test. Values are built, encoded, compared and dropped on a large thread; only
//! the decode runs on the small one.
use cultcache_rs::CacheBackingStore;
use cultcache_rs::CultCacheEnvelope;
use cultcache_rs::CultCacheStoreUnreadable;
use cultcache_rs::CultCacheStoreUnreadableKind;
use cultcache_rs::CultSlots;
use cultcache_rs::CultUnion;
use cultcache_rs::MAX_PAYLOAD_DEPTH;
use cultcache_rs::MAX_RECORD_INLINE_BYTES;
use cultcache_rs::PayloadTooDeep;
use cultcache_rs::RedbMessagePackBackingStore;
use cultcache_rs::SingleFileMessagePackBackingStore;
use cultcache_rs::check_payload_depth;
use cultcache_rs::decode_payload;
use serde::de::DeserializeOwned;
const CHILD_MARKER: &str = "CULTCACHE_DECODE_STACK_CHILD";
const KIB: usize = 1024;
#[derive(Clone, Debug, PartialEq, CultSlots)]
struct Wide32 {
#[cultcache(key = 0)]
a0: [u64; 32],
#[cultcache(key = 1)]
a1: [u64; 32],
#[cultcache(key = 2)]
a2: [u64; 32],
#[cultcache(key = 3)]
a3: [u64; 32],
#[cultcache(key = 4)]
a4: [u64; 32],
#[cultcache(key = 5)]
a5: [u64; 32],
#[cultcache(key = 6)]
a6: [u64; 32],
#[cultcache(key = 7)]
a7: [u64; 32],
#[cultcache(key = 8)]
a8: [u64; 32],
#[cultcache(key = 9)]
a9: [u64; 32],
#[cultcache(key = 10)]
a10: [u64; 32],
#[cultcache(key = 11)]
a11: [u64; 32],
#[cultcache(key = 12)]
a12: [u64; 32],
#[cultcache(key = 13)]
a13: [u64; 32],
#[cultcache(key = 14)]
a14: [u64; 32],
#[cultcache(key = 15)]
a15: [u64; 32],
#[cultcache(key = 16)]
a16: [u64; 32],
#[cultcache(key = 17)]
a17: [u64; 32],
#[cultcache(key = 18)]
a18: [u64; 32],
#[cultcache(key = 19)]
a19: [u64; 32],
#[cultcache(key = 20)]
a20: [u64; 32],
#[cultcache(key = 21)]
a21: [u64; 32],
#[cultcache(key = 22)]
a22: [u64; 32],
#[cultcache(key = 23)]
a23: [u64; 32],
#[cultcache(key = 24)]
a24: [u64; 32],
#[cultcache(key = 25)]
a25: [u64; 32],
#[cultcache(key = 26)]
a26: [u64; 32],
#[cultcache(key = 27)]
a27: [u64; 32],
#[cultcache(key = 28)]
a28: [u64; 32],
#[cultcache(key = 29)]
a29: [u64; 32],
#[cultcache(key = 30)]
a30: [u64; 32],
#[cultcache(key = 31)]
a31: [u64; 32],
#[cultcache(key = 32)]
next: Option<Box<Wide32>>,
}
#[derive(Clone, Debug, PartialEq, CultSlots)]
struct One {
#[cultcache(key = 0)]
a: u8,
}
#[derive(Clone, Debug, PartialEq, CultSlots)]
struct DynamicSlot {
#[cultcache(key = 0)]
v: serde_json::Value,
}
/// Runs `body` on a thread with a stack of `bytes`, returning what it returns.
fn on_stack<R: Send + 'static>(bytes: usize, body: impl FnOnce() -> R + Send + 'static) -> R {
std::thread::Builder::new().stack_size(bytes).spawn(body).unwrap().join().unwrap()
}
/// Runs `body` in a child process that runs only the test `name`; the parent passes when the
/// child exits cleanly having run exactly that one test. An abort in the child fails the parent.
fn in_child(name: &str, body: impl FnOnce()) {
if std::env::var_os(CHILD_MARKER).is_some() {
body();
return;
}
let output = std::process::Command::new(std::env::current_exe().unwrap())
.args([name, "--exact", "--nocapture", "--test-threads=1"])
.env(CHILD_MARKER, "1")
.output()
.unwrap();
let stdout = String::from_utf8_lossy(&output.stdout);
let stderr = String::from_utf8_lossy(&output.stderr);
assert!(
output.status.success() && stdout.contains("1 passed"),
"child {name} failed: {:?}
{stdout}
{stderr}",
output.status
);
}
/// A chain of `levels` Wide32 records: `levels` containers deep, plus one for its last arrays.
fn wide_chain(levels: usize) -> Wide32 {
let mut chain: Option<Box<Wide32>> = None;
for level in (0..levels).rev() {
let mut node = Wide32 { next: chain, ..empty_wide() };
node.a0[0] = level as u64;
chain = Some(Box::new(node));
}
*chain.unwrap()
}
fn empty_wide() -> Wide32 {
Wide32 {
a0: [0; 32],
a1: [0; 32],
a2: [0; 32],
a3: [0; 32],
a4: [0; 32],
a5: [0; 32],
a6: [0; 32],
a7: [0; 32],
a8: [0; 32],
a9: [0; 32],
a10: [0; 32],
a11: [0; 32],
a12: [0; 32],
a13: [0; 32],
a14: [0; 32],
a15: [0; 32],
a16: [0; 32],
a17: [0; 32],
a18: [0; 32],
a19: [0; 32],
a20: [0; 32],
a21: [0; 32],
a22: [0; 32],
a23: [0; 32],
a24: [0; 32],
a25: [0; 32],
a26: [0; 32],
a27: [0; 32],
a28: [0; 32],
a29: [0; 32],
a30: [0; 32],
a31: [0; 32],
next: None,
}
}
/// The decode alone runs on the small thread. Its result is boxed there so that the thread's own
/// return machinery, which would hold several more copies of a large record in a debug build, does
/// not count against the stack under test; what the caller's frames hold is the caller's to size.
fn decode_on<T: DeserializeOwned + Send + 'static>(stack: usize, bytes: Vec<u8>) -> anyhow::Result<T> {
*on_stack(stack, move || Box::new(decode_payload::<T>(&bytes)))
}
#[test]
fn a_wide_record_at_the_limit_decodes_on_a_1mib_thread() {
in_child("a_wide_record_at_the_limit_decodes_on_a_1mib_thread", || {
let bytes = on_stack(64 * KIB * KIB, || {
rmp_serde::to_vec(&wide_chain(MAX_PAYLOAD_DEPTH - 1)).unwrap()
});
let decoded = decode_on::<Wide32>(1024 * KIB, bytes).unwrap();
on_stack(64 * KIB * KIB, move || {
assert!(decoded == wide_chain(MAX_PAYLOAD_DEPTH - 1));
});
});
}
#[test]
fn one_past_the_limit_is_refused_typed_on_a_1mib_thread() {
in_child("one_past_the_limit_is_refused_typed_on_a_1mib_thread", || {
let bytes = on_stack(64 * KIB * KIB, || {
rmp_serde::to_vec(&wide_chain(MAX_PAYLOAD_DEPTH)).unwrap()
});
let error = decode_on::<Wide32>(1024 * KIB, bytes).unwrap_err();
assert_eq!(
error.downcast_ref::<PayloadTooDeep>(),
Some(&PayloadTooDeep { depth: MAX_PAYLOAD_DEPTH + 1, limit: MAX_PAYLOAD_DEPTH })
);
});
}
#[test]
fn a_foreign_value_at_the_limit_decodes_on_a_64kib_thread() {
in_child("a_foreign_value_at_the_limit_decodes_on_a_64kib_thread", || {
// The slot's array is depth 1, so 255 nested arrays under it reach depth 256.
let value = on_stack(64 * KIB * KIB, || {
let mut value = serde_json::Value::Null;
for _ in 0..MAX_PAYLOAD_DEPTH - 1 {
value = serde_json::Value::Array(vec![value]);
}
value
});
let expected = DynamicSlot { v: value };
let bytes = on_stack(64 * KIB * KIB, {
let expected = expected.clone();
move || rmp_serde::to_vec(&expected).unwrap()
});
let decoded = decode_on::<DynamicSlot>(64 * KIB, bytes).unwrap();
on_stack(64 * KIB * KIB, move || {
assert!(decoded == expected);
drop(expected);
});
});
}
#[test]
fn a_skipped_slot_at_the_limit_decodes_on_a_64kib_thread() {
in_child("a_skipped_slot_at_the_limit_decodes_on_a_64kib_thread", || {
// [1, [[...]]]: slot 1 is unknown to One and is skipped; the whole payload is 256 deep.
let mut bytes = vec![0x92u8, 0x01];
bytes.extend(std::iter::repeat_n(0x91u8, MAX_PAYLOAD_DEPTH - 2));
bytes.push(0x90);
let decoded = decode_on::<One>(64 * KIB, bytes).unwrap();
assert_eq!(decoded, One { a: 1 });
});
}
// A record the derives accept at the bound: two slots of 31 arrays of 32 words, 15,880 bytes inline.
#[derive(Clone, Debug, PartialEq, CultSlots)]
struct Largest {
#[cultcache(key = 0)]
a: [[u64; 32]; 31],
#[cultcache(key = 1)]
b: [[u64; 32]; 31],
#[cultcache(key = 2)]
next: Option<Box<Largest>>,
}
#[derive(Clone, Debug, PartialEq, CultSlots)]
struct LargestUnionArm {
#[cultcache(key = 0)]
a: [[u64; 32]; 31],
#[cultcache(key = 1)]
b: [[u64; 32]; 31],
#[cultcache(key = 2)]
next: Option<Box<LargestUnion>>,
}
// The fixture is the largest arm the derives accept next to a unit variant, which is the point.
#[allow(clippy::large_enum_variant)]
#[derive(Clone, Debug, PartialEq, CultUnion)]
enum LargestUnion {
#[cultcache(key = 0)]
Arm(LargestUnionArm),
#[cultcache(key = 1)]
Leaf,
}
// The fixtures sit within a KiB under the bound, so a decode that survives them survives any record.
const _: () = assert!(
std::mem::size_of::<Largest>() <= MAX_RECORD_INLINE_BYTES
&& std::mem::size_of::<Largest>() + KIB >= MAX_RECORD_INLINE_BYTES
&& std::mem::size_of::<LargestUnion>() <= MAX_RECORD_INLINE_BYTES
&& std::mem::size_of::<LargestUnion>() + KIB >= MAX_RECORD_INLINE_BYTES
);
fn largest_chain(levels: usize) -> Largest {
let mut chain: Option<Box<Largest>> = None;
for level in (0..levels).rev() {
let mut a = [[0u64; 32]; 31];
a[0][0] = level as u64;
chain = Some(Box::new(Largest { a, b: [[0; 32]; 31], next: chain }));
}
*chain.unwrap()
}
fn union_chain(levels: usize) -> LargestUnion {
let mut chain = LargestUnion::Leaf;
for level in (0..levels).rev() {
let mut a = [[0u64; 32]; 31];
a[0][0] = level as u64;
chain = LargestUnion::Arm(LargestUnionArm { a, b: [[0; 32]; 31], next: Some(Box::new(chain)) });
}
chain
}
#[test]
fn a_wide_record_20_levels_deep_decodes_on_a_1mib_thread() {
in_child("a_wide_record_20_levels_deep_decodes_on_a_1mib_thread", || decode_wide_chain(20, 1024 * KIB));
}
#[test]
fn a_wide_record_100_levels_deep_decodes_on_a_1mib_thread() {
in_child("a_wide_record_100_levels_deep_decodes_on_a_1mib_thread", || decode_wide_chain(100, 1024 * KIB));
}
#[test]
fn a_wide_record_decodes_on_a_96kib_thread_at_one_level_and_at_the_limit() {
in_child("a_wide_record_decodes_on_a_96kib_thread_at_one_level_and_at_the_limit", || {
decode_wide_chain(1, 96 * KIB);
decode_wide_chain(MAX_PAYLOAD_DEPTH - 1, 96 * KIB);
});
}
#[test]
fn the_largest_record_decodes_on_an_88kib_thread_at_one_level() {
in_child("the_largest_record_decodes_on_an_88kib_thread_at_one_level", || {
// The door's own floor, not the payload, decides this size: one Largest level, boxed as decode_on
// does, needs 72 KiB in debug and 56 in release with the outer stack check, and 104 in release
// without it.
let bytes = on_stack(64 * KIB * KIB, || rmp_serde::to_vec(&largest_chain(1)).unwrap());
let decoded = decode_on::<Largest>(88 * KIB, bytes).unwrap();
on_stack(64 * KIB * KIB, move || assert!(decoded == largest_chain(1)));
});
}
fn decode_wide_chain(levels: usize, stack: usize) {
let bytes = on_stack(64 * KIB * KIB, move || rmp_serde::to_vec(&wide_chain(levels)).unwrap());
let decoded = decode_on::<Wide32>(stack, bytes).unwrap();
on_stack(64 * KIB * KIB, move || assert!(decoded == wide_chain(levels)));
}
#[test]
fn the_largest_record_at_the_limit_decodes_on_every_thread_size() {
in_child("the_largest_record_at_the_limit_decodes_on_every_thread_size", || {
// 254 records of Largest are 256 containers deep (each is one, its slots' arrays two more);
// 127 union levels are the same (a union and its arm are two each, the leaf's arrays two).
let (records, unions) = on_stack(64 * KIB * KIB, || {
let records = rmp_serde::to_vec(&largest_chain(254)).unwrap();
let unions = rmp_serde::to_vec(&union_chain(127)).unwrap();
assert!(check_payload_depth(&records).is_ok());
assert!(check_payload_depth(&unions).is_ok());
assert!(check_payload_depth(&rmp_serde::to_vec(&largest_chain(255)).unwrap()).is_err());
assert!(check_payload_depth(&rmp_serde::to_vec(&union_chain(128)).unwrap()).is_err());
(records, unions)
});
let sizes = (160..=1024).step_by(32).map(|kib| kib * KIB).chain([1536, 2048, 3072, 4096, 8192].map(|kib| kib * KIB));
for size in sizes {
println!("thread of {} KiB", size / KIB);
let decoded = decode_on::<Largest>(size, records.clone()).unwrap();
let decoded_union = decode_on::<LargestUnion>(size, unions.clone()).unwrap();
on_stack(64 * KIB * KIB, move || {
assert!(decoded == largest_chain(254));
assert!(decoded_union == union_chain(127));
});
}
});
}
/// One stored envelope, `{key: "k", type: "t", payload: bin[1], storedAt: "s", extra: [[[...]]]}`,
/// whose unknown key nests 1000 containers deep and is skipped without a payload walk.
fn deep_envelope() -> Vec<u8> {
let mut bytes = vec![0x85];
for (name, value) in [("key", &[0xa1, b'k'][..]), ("type", &[0xa1, b't']), ("payload", &[0xc4, 1, 1]), ("storedAt", &[0xa1, b's'])] {
bytes.push(0xa0 | name.len() as u8);
bytes.extend(name.as_bytes());
bytes.extend(value);
}
bytes.extend([0xa5, b'e', b'x', b't', b'r', b'a']);
bytes.extend(std::iter::repeat_n(0x91u8, 999));
bytes.push(0x90);
bytes
}
fn fixstr(text: &str) -> Vec<u8> {
let mut bytes = vec![0xa0 | text.len() as u8];
bytes.extend(text.as_bytes());
bytes
}
#[test]
fn a_legacy_store_with_a_deep_unknown_envelope_key_reads_on_a_64kib_thread() {
in_child("a_legacy_store_with_a_deep_unknown_envelope_key_reads_on_a_64kib_thread", || {
let directory = tempfile::tempdir().unwrap();
let path = directory.path().join("legacy.cc");
std::fs::write(&path, [vec![0x91], deep_envelope()].concat()).unwrap();
let entries = on_stack(64 * KIB, move || SingleFileMessagePackBackingStore::new(path).pull_all_read_only_snapshot()).unwrap();
assert_eq!(entries.len(), 1);
assert_eq!(entries[0].key, "k");
});
}
#[test]
fn a_v1_store_with_a_deep_fifth_record_slot_is_refused_typed_on_a_64kib_thread() {
in_child("a_v1_store_with_a_deep_fifth_record_slot_is_refused_typed_on_a_64kib_thread", || {
// [header, [], [[key, schema, storedAt, payload, [[[...]]]]]]: the record's unknown fifth slot is
// skipped 1000 containers deep, then the record is refused.
let mut bytes = vec![0x93];
bytes.extend(fixstr("cultcache.store.v1"));
bytes.extend([0x90, 0x91, 0x95]);
for text in ["k", "s", "t"] {
bytes.extend(fixstr(text));
}
bytes.extend([0xc4, 1, 1]);
bytes.extend(std::iter::repeat_n(0x91u8, 999));
bytes.push(0x90);
let directory = tempfile::tempdir().unwrap();
let path = directory.path().join("v1.cc");
std::fs::write(&path, bytes).unwrap();
let error = on_stack(64 * KIB, move || SingleFileMessagePackBackingStore::new(path).pull_all_read_only_snapshot()).unwrap_err();
let unreadable = error.downcast_ref::<CultCacheStoreUnreadable>().expect("a typed refusal");
assert!(matches!(unreadable.kind, CultCacheStoreUnreadableKind::Undecodable));
});
}
/// A redb store holding the one deep envelope under its identity, `["t", "k"]`.
fn redb_with_deep_row(path: &std::path::Path) {
let database = redb::Database::create(path).unwrap();
let write = database.begin_write().unwrap();
{
let mut table = write
.open_table(redb::TableDefinition::<&[u8], &[u8]>::new("cultcache_envelopes"))
.unwrap();
table.insert([0x92, 0xa1, b't', 0xa1, b'k'].as_slice(), deep_envelope().as_slice()).unwrap();
}
write.commit().unwrap();
}
#[test]
fn a_redb_store_with_a_deep_unknown_envelope_key_reads_on_a_104kib_thread() {
in_child("a_redb_store_with_a_deep_unknown_envelope_key_reads_on_a_104kib_thread", || {
let directory = tempfile::tempdir().unwrap();
let path = directory.path().join("rows.redb");
redb_with_deep_row(&path);
let entries = on_stack(104 * KIB, move || RedbMessagePackBackingStore::new(path).unwrap().pull_all()).unwrap();
assert_eq!(entries.len(), 1);
assert_eq!(entries[0].key, "k");
});
}
#[test]
fn a_redb_compare_and_swap_reads_a_deep_unknown_envelope_key_on_a_104kib_thread() {
in_child("a_redb_compare_and_swap_reads_a_deep_unknown_envelope_key_on_a_104kib_thread", || {
let directory = tempfile::tempdir().unwrap();
let path = directory.path().join("rows.redb");
redb_with_deep_row(&path);
let expected = CultCacheEnvelope {
key: "k".into(),
r#type: "t".into(),
payload: vec![1],
stored_at: "s".into(),
schema_id: None,
};
let replacement = CultCacheEnvelope { payload: vec![2], ..expected.clone() };
let swapped = on_stack(104 * KIB, move || {
RedbMessagePackBackingStore::new(path).unwrap().compare_and_swap_entry(&expected, replacement)
})
.unwrap();
assert!(swapped);
});
}

75
packages/cultcache-rs/tests/union_parity.rs
View file

@ -338,6 +338,73 @@ fn a_union_that_is_not_a_key_and_slots_pair_is_refused_naming_the_enum() {
}
}
// A plain derive struct: the door names a missing or duplicated field (a static name from the type)
// and describes a mistyped value by kind, never quoting it.
#[derive(Debug, serde::Deserialize)]
#[allow(dead_code)]
struct Probe {
name: String,
flag: bool,
}
#[test]
fn a_missing_or_duplicate_field_is_named_and_a_mistyped_value_is_not() {
let name = |value: &str| [&[0xa4][..], b"name", &[0xa0 | value.len() as u8][..], value.as_bytes()].concat();
let flag_true = [&[0xa4][..], b"flag", &[0xc3][..]].concat();
let missing = [&[0x81][..], &name("a")].concat();
let message = cultcache_rs::decode_payload::<Probe>(&missing).unwrap_err().to_string();
assert!(message.contains("missing field `flag`"), "{message}");
let duplicated = [&[0x83][..], &name("a"), &name("b"), &flag_true].concat();
let message = cultcache_rs::decode_payload::<Probe>(&duplicated).unwrap_err().to_string();
assert!(message.contains("duplicate field `name`"), "{message}");
let canary = "CANARY-3d9e";
let flag_text = [&[0xa4][..], b"flag", &[0xa0 | canary.len() as u8][..], canary.as_bytes()].concat();
let mistyped = [&[0x82][..], &name("a"), &flag_text].concat();
let message = cultcache_rs::decode_payload::<Probe>(&mistyped).unwrap_err().to_string();
assert!(!message.contains("CANARY"), "{message}");
assert!(message.contains("MessagePack type"), "{message}");
}
// A decoder whose error text is the string it reads, standing for any serde message.
struct Raises;
impl<'de> serde::Deserialize<'de> for Raises {
fn deserialize<D: serde::Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
let message = <String as serde::Deserialize>::deserialize(deserializer)?;
Err(serde::de::Error::custom(message))
}
}
#[test]
fn a_field_name_is_kept_only_when_it_looks_like_a_field_name() {
let refusal = |message: &str| {
let bytes = rmp_serde::to_vec(message).unwrap();
match cultcache_rs::decode_payload::<Raises>(&bytes) {
Ok(_) => panic!("Raises never decodes"),
Err(error) => error.to_string(),
}
};
assert_eq!(refusal("missing field `a_1`"), "missing field `a_1`");
assert_eq!(refusal("duplicate field `Z9`"), "duplicate field `Z9`");
let long = format!("missing field `{}`", "a".repeat(64));
assert_eq!(refusal(&long), long);
for message in [
"missing field `has space CANARY`".to_string(),
"missing field `x-CANARY`".to_string(),
"missing field ``".to_string(),
format!("duplicate field `{}`", "a".repeat(65)),
"unknown field `CANARY`, expected `a`".to_string(),
] {
let text = refusal(&message);
assert_eq!(text, "a value does not fit the store format", "{message}");
}
assert_eq!(refusal("unknown variant `CANARY`, expected `a`"), "a tag names no variant this type knows");
assert!(!refusal("invalid type: string \"CANARY\", expected a bool").contains("CANARY"));
}
#[test]
fn the_key_decides_the_arm() {
// Tool and Part carry different slots under different keys in the two unions.
@ -826,12 +893,12 @@ fn the_raw_doors_hold_the_bound_for_arms_maps_and_arrays() {
}
}
// A debug build on a 2 MiB thread carries 405 containers of this document; the bound keeps every
// accepted payload below that, and every refusal is an error, not an abort.
// The bound counts containers, not stack: decode grows its own stack, so the document at the limit
// decodes on a 1 MiB thread in both profiles, and every refusal is an error, not an abort.
#[test]
fn stack_budget_debug_2mib() {
fn the_union_chain_at_the_limit_decodes_on_a_1mib_thread() {
std::thread::Builder::new()
.stack_size(2 * 1024 * 1024)
.stack_size(1024 * 1024)
.spawn(|| {
assert_eq!(
through_cache(&vector("depth-at-limit.msgpack")).unwrap(),

33
packages/cultcache-ts/CHANGELOG.md
View file

@ -1,5 +1,8 @@
# Changelog
All notable changes to `@gamecult/cultcache-ts` are recorded here, following
[Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
## Unreleased
### Breaking
@ -12,3 +15,33 @@
- `readSingleFileStore(path)`, the one synchronous reader of a single-file store, for consumers that must read a store file rather than decode it themselves. `pullAll`, `push`, `delete` and `pushAll` ask the same reader.
- `StoreUnreadableError` for a store file this runtime cannot read, `SchemaConflictError` for a write the catalog cannot describe, and `schemaIdentityOf`.
- A soft `pushAll` writes only where nothing is at the path, leaves a store it can read byte for byte, and refuses what the reader refuses.
## [0.14.2]
### Added
- `SingleFileMessagePackBackingStore` takes a cross-process write lock on `<file>.lock`,
interoperable with the C# and Rust runtimes (`flock` on Unix, `LockFileEx` on Windows).
`push` and `delete` read the current file, merge and replace it under the lock, so concurrent
TypeScript and Rust writers no longer lose each other's records across those calls. A full
`pushAll` still replaces the whole store, so records a concurrent writer added are lost to it.
C# writes are not all alike: a conditional commit on a clean store lands onto the file as it
is, but C# writers can still overwrite records a concurrent TypeScript writer added.
- The staged file is fsynced before the replace, and on Unix the directory after it. A
directory sync failure after the replace is reported as replaced but not durable.
### Changed
- The Windows replace is a POSIX-semantics rename first (it lands while C# and Rust readers
hold the store open), falling back to a write-through `MoveFileExW`, with 25 attempts 20 ms
apart on errors 5, 32 and 33. Writers wait up to 30 s for the lock.
- Opening a legacy store that needs healing rewrites it under the lock, from a read taken
under the lock.
### Dependencies
- `koffi` 3.3.0, loaded only on the first write.
## [0.14.1]
### Added
- `checkPayloadDepth` is re-exported from `@gamecult/cultcache-ts/document`, so dependents
(`cultnet-ts/content`) bound payload nesting with the same check the cache
applies on decode.

5
packages/cultcache-ts/README.md
View file

@ -358,7 +358,8 @@ That is the TypeScript version of the original CultCache trick:
- each record is `[key, schemaId, storedAt, payload]`
- each catalog entry is `[schemaId, schemaName, schemaVersion, contentHash, canonicalSchemaJson, compatibleSchemaIds, members]`
- it still reads the older TS/Rust-style envelope array and rewrites repaired legacy payloads as v1 snapshots
- it queues local writes so one process does not step on its own shoelaces
- it queues local writes in order, then takes the shared `<file>.lock` (`flock` on Unix, `LockFileEx` on Windows, via koffi, loaded on the first write) for each one, so C#, Rust and TypeScript writers exclude each other; the read a write merges from is taken under that lock
- it fsyncs the staged file and, on Unix, the directory after the replace; on Windows it replaces with a POSIX-semantics rename and falls back to a write-through `MoveFileExW`, retrying briefly on sharing violations
- it persists raw MessagePack payload bytes inside schema-addressed records
It is a control-plane store, not a tiny god. If multiple processes hammer the same file, you still need a coordinator or an external lock instead of prayer.
It is a control-plane store, not a tiny god: every write rewrites the whole file.

3
packages/cultcache-ts/package.json
View file

@ -1,6 +1,6 @@
{
"name": "@gamecult/cultcache-ts",
"version": "0.14.0",
"version": "0.14.2",
"description": "CultCache-style typed persistence for TypeScript with schema-backed polymorphic MessagePack storage.",
"main": "dist/index.js",
"types": "dist/index.d.ts",
@ -61,6 +61,7 @@
},
"dependencies": {
"@msgpack/msgpack": "^3.1.3",
"koffi": "3.3.0",
"zod": "^3.25.76"
},
"devDependencies": {

2
packages/cultcache-ts/src/document.ts
View file

@ -105,7 +105,7 @@ export function documentPayloadBytes(payload: DocumentPayload): Uint8Array {
return payload as unknown as Uint8Array;
}
export { CultCachePayloadDepthError, MAX_PAYLOAD_DEPTH } from "./payload-depth";
export { CultCachePayloadDepthError, MAX_PAYLOAD_DEPTH, checkPayloadDepth } from "./payload-depth";
// The payload nesting bound covers the MessagePack the library itself writes: the plain default formatter and a codec. A
// user formatter's bytes are opaque and in any format; its format and bounds are its own.

66
packages/cultcache-ts/src/single-file-messagepack-backing-store.ts
View file

@ -1,11 +1,12 @@
import { lstatSync, readFileSync } from "node:fs";
import { mkdir, rename, rm, writeFile } from "node:fs/promises";
import { mkdir } from "node:fs/promises";
import { dirname, resolve } from "node:path";
import { decode, encode } from "@msgpack/msgpack";
import { z } from "zod";
import { documentPayloadBytes, documentPayloadFromBytes } from "./document";
import { commitStoreFile, withStoreWriteLock } from "./store-file";
import { STORE_FORMAT_VERSION, SchemaConflictError, type StoreFormat, StoreUnreadableError, isStoreSnapshot, requireStoreSlots, requireV1RecordSlots } from "./store-format";
import type {
CacheBackingStore,
@ -51,7 +52,14 @@ export class SingleFileMessagePackBackingStore implements CacheBackingStore {
const disk = this.#readDisk();
this.#format = disk.format;
if (disk.repairedLegacyPayload) {
await this.#writeAll([], disk.envelopes);
// The heal decides from its own read, taken under the store lock.
await this.#enqueue(async () => {
const again = this.#readDisk();
this.#format = again.format;
if (again.repairedLegacyPayload) {
await this.#writeAll([], again.envelopes);
}
});
}
return disk.envelopes;
@ -59,7 +67,9 @@ export class SingleFileMessagePackBackingStore implements CacheBackingStore {
async push(entry: CultCacheEnvelope): Promise<void> {
await this.#enqueue(async () => {
const existing = await this.pullAll();
const disk = this.#readDisk();
this.#format = disk.format;
const existing = disk.envelopes;
const filtered = existing.filter(
(candidate) => !(candidate.type === entry.type && candidate.key === entry.key),
);
@ -69,7 +79,9 @@ export class SingleFileMessagePackBackingStore implements CacheBackingStore {
async delete(entry: CultCacheEnvelope): Promise<void> {
await this.#enqueue(async () => {
const existing = await this.pullAll();
const disk = this.#readDisk();
this.#format = disk.format;
const existing = disk.envelopes;
const filtered = existing.filter(
(candidate) => !(candidate.type === entry.type && candidate.key === entry.key),
);
@ -113,9 +125,11 @@ export class SingleFileMessagePackBackingStore implements CacheBackingStore {
async #enqueue<T>(operation: () => Promise<T>): Promise<T> {
let result!: T;
const next = this.#writeQueue.then(async () => {
result = await operation();
});
const next = this.#writeQueue.then(() =>
withStoreWriteLock(this.filePath, async () => {
result = await operation();
}),
);
this.#writeQueue = next.then(
() => undefined,
@ -129,17 +143,7 @@ export class SingleFileMessagePackBackingStore implements CacheBackingStore {
async #writeAll(supplied: CultCacheEnvelope[], arrived: CultCacheEnvelope[]): Promise<void> {
await mkdir(dirname(this.filePath), { recursive: true });
const tempPath = `${this.filePath}.tmp-${process.pid}-${Date.now()}-${Math.random()
.toString(36)
.slice(2)}`;
try {
await writeFile(tempPath, encode(encodeSnapshot(supplied, arrived, this.#format)));
await renameWithRetry(tempPath, this.filePath);
} catch (error) {
await rm(tempPath, { force: true }).catch(() => undefined);
throw error;
}
await commitStoreFile(this.filePath, encode(encodeSnapshot(supplied, arrived, this.#format)));
}
}
@ -196,32 +200,6 @@ export function readStoreBytes(path: string, lstat: (path: string) => { isSymbol
throw Object.assign(new Error(`CultCache store ${path} ${what}.`), { code: "ENOENT" });
}
async function renameWithRetry(source: string, destination: string): Promise<void> {
let delayMs = 10;
let lastError: unknown;
for (let attempt = 0; attempt < 10; attempt++) {
try {
await rename(source, destination);
return;
} catch (error) {
const code = (error as NodeJS.ErrnoException).code;
if (code !== "EPERM" && code !== "EBUSY" && code !== "EACCES") {
throw error;
}
lastError = error;
await delay(delayMs);
delayMs *= 2;
}
}
throw lastError;
}
function delay(milliseconds: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, milliseconds));
}
type PersistedRecord = {
key: string;
schemaId: string;

391
packages/cultcache-ts/src/store-file.ts Normal file
View file

@ -0,0 +1,391 @@
// The TypeScript owner of a single-file store's write transaction: the cross-process lock on
// `<path>.lock`, the staged file, its fsync, the replace and the directory fsync.
// Node only, and not exported from the package index. The OS calls Node does not expose
// (flock, LockFileEx, a POSIX-semantics rename on Windows) go through koffi, which is required
// lazily inside `loadKernel`, so a consumer that never writes a store never loads it.
import { closeSync, openSync, constants as fsConstants } from "node:fs";
import { mkdir, open, rename, rm } from "node:fs/promises";
import type { FileHandle } from "node:fs/promises";
import { dirname } from "node:path";
const LOCK_POLL_MS = 10;
const LOCK_TIMEOUT_MS = 30_000;
/** Thrown when the staged file replaced the store but the directory entry may not be durable. */
export class StoreReplacedNotDurableError extends Error {
readonly replaced = true;
constructor(storePath: string, cause: unknown) {
super(
`CultCache store ${storePath} was replaced, but syncing its directory failed, so the replace may not survive a crash: ${
cause instanceof Error ? cause.message : String(cause)
}`,
);
this.name = "StoreReplacedNotDurableError";
}
}
/** The file operations of one commit. Tests record or fail them; production uses `nodeStoreFileOps`. */
export interface StoreFileOps {
createExclusive(path: string): Promise<FileHandle>;
write(handle: FileHandle, bytes: Uint8Array): Promise<void>;
sync(handle: FileHandle): Promise<void>;
close(handle: FileHandle): Promise<void>;
replace(staged: string, destination: string): Promise<void>;
syncDirectory(directory: string): Promise<void>;
remove(path: string): Promise<void>;
}
export const nodeStoreFileOps: StoreFileOps = {
createExclusive: (path) => open(path, "wx"),
write: async (handle, bytes) => {
await handle.writeFile(bytes);
},
sync: (handle) => handle.sync(),
close: (handle) => handle.close(),
replace: (staged, destination) => replaceStoreFile(staged, destination),
syncDirectory: async (directory) => {
const handle = await open(directory, "r");
try {
await handle.sync();
} finally {
await handle.close();
}
},
remove: (path) => rm(path, { force: true }),
};
/**
* Writes `bytes` to a unique staged file beside the store, fsyncs it, replaces the store with it
* and, on Unix, fsyncs the directory so the replace is durable. Windows relies on the replace
* itself, as the Rust runtime does. Any failure before the replace removes the staged file and
* leaves the store untouched; a directory sync failure after it throws
* `StoreReplacedNotDurableError`. The caller holds the store lock.
*/
export async function commitStoreFile(
storePath: string,
bytes: Uint8Array,
ops: StoreFileOps = nodeStoreFileOps,
): Promise<void> {
const staged = `${storePath}.tmp-${process.pid}-${Date.now()}-${Math.random().toString(36).slice(2)}`;
let handle: FileHandle | undefined;
try {
handle = await ops.createExclusive(staged);
await ops.write(handle, bytes);
await ops.sync(handle);
const closing = handle;
handle = undefined;
await ops.close(closing);
await ops.replace(staged, storePath);
} catch (error) {
if (handle) {
await ops.close(handle).catch(() => undefined);
}
await ops.remove(staged).catch(() => undefined);
throw error;
}
if (process.platform !== "win32") {
try {
await ops.syncDirectory(dirname(storePath));
} catch (error) {
throw new StoreReplacedNotDurableError(storePath, error);
}
}
}
/** Replaces `destination` with `staged`: POSIX-first on Windows, `rename` elsewhere. */
export async function replaceStoreFile(staged: string, destination: string): Promise<void> {
if (process.platform === "win32") {
await replaceStoreFileWindows(staged, destination, windowsKernel());
return;
}
await rename(staged, destination);
}
const ERROR_ACCESS_DENIED = 5;
const ERROR_SHARING_VIOLATION = 32;
const ERROR_LOCK_VIOLATION = 33;
const ERROR_NOT_SUPPORTED = 50;
const ERROR_INVALID_PARAMETER = 87;
const REPLACE_ATTEMPTS = 25;
const REPLACE_RETRY_MS = 20;
/** MOVEFILE_REPLACE_EXISTING | MOVEFILE_WRITE_THROUGH. */
export const MOVEFILE_REPLACE_WRITE_THROUGH = 0x9;
/** The kernel32 calls a Windows replace needs, each answering 0 or a Win32 error code. */
export interface ReplaceKernel {
/** CreateFileW(staged, DELETE|SYNCHRONIZE) then FileRenameInfoEx with REPLACE_IF_EXISTS|POSIX_SEMANTICS, then CloseHandle. */
renamePosix(staged: string, destination: string): number;
/** MoveFileExW(staged, destination, flags). */
moveFileEx(staged: string, destination: string, flags: number): number;
sleep(milliseconds: number): Promise<void>;
}
export async function replaceStoreFileWindows(
staged: string,
destination: string,
kernel: ReplaceKernel,
): Promise<void> {
let posix = true;
let lastCode = 0;
for (let attempt = 1; attempt <= REPLACE_ATTEMPTS; attempt++) {
let code = posix ? kernel.renamePosix(staged, destination) : kernel.moveFileEx(staged, destination, MOVEFILE_REPLACE_WRITE_THROUGH);
if (posix && (code === 1 || code === ERROR_NOT_SUPPORTED || code === ERROR_INVALID_PARAMETER)) {
// POSIX semantics are unsupported here (FAT, exFAT, before Windows 10 1607): the rest of this call is write-through.
posix = false;
code = kernel.moveFileEx(staged, destination, MOVEFILE_REPLACE_WRITE_THROUGH);
}
if (code === 0) {
return;
}
if (code !== ERROR_ACCESS_DENIED && code !== ERROR_SHARING_VIOLATION && code !== ERROR_LOCK_VIOLATION) {
throw new Error(`CultCache could not replace the store: Win32 error ${code}.`);
}
lastCode = code;
await kernel.sleep(REPLACE_RETRY_MS);
}
throw new Error(
`CultCache could not replace the store after ${REPLACE_ATTEMPTS} attempts: last Win32 error ${lastCode}.`,
);
}
// ---------------------------------------------------------------------------------------------
// The cross-process write lock.
export interface StoreLockOptions {
/** How long to wait for the lock; 30 s by default, as the C# runtime waits. */
timeoutMs?: number;
}
/**
* Runs `operation` while holding the exclusive store lock on `<storePath>.lock`. C# holds it
* by a share-none open, Rust by an exclusive file lock; this takes the same lock by LockFileEx
* on Windows and flock on Unix, so all of them exclude each other. Polls every 10 ms and holds
* no handle on the lock file between polls. The lock file is created if absent, never
* written and never deleted.
*/
export async function withStoreWriteLock<T>(
storePath: string,
operation: () => Promise<T>,
options: StoreLockOptions = {},
): Promise<T> {
const lockPath = `${storePath}.lock`;
const timeoutMs = options.timeoutMs ?? LOCK_TIMEOUT_MS;
await mkdir(dirname(lockPath), { recursive: true });
const started = Date.now();
let release: (() => void) | undefined;
while (release === undefined) {
release = tryLock(lockPath);
if (release === undefined) {
if (Date.now() - started >= timeoutMs) {
throw new Error(`CultCache timed out after ${timeoutMs} ms waiting for the store lock ${lockPath}.`);
}
await sleep(LOCK_POLL_MS);
}
}
try {
return await operation();
} finally {
release();
}
}
function tryLock(lockPath: string): (() => void) | undefined {
const platform = process.platform;
if (platform === "win32") {
return tryLockWindows(lockPath);
}
if (platform === "linux" || platform === "darwin") {
return tryLockUnix(lockPath, platform);
}
throw new Error(`CultCache cross-process store lock is not implemented on ${platform}.`);
}
function sleep(milliseconds: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, milliseconds));
}
// ---------------------------------------------------------------------------------------------
// koffi bindings, loaded on first use.
const GENERIC_READ_WRITE = 0xc0000000;
const GENERIC_ALL_DELETE_SYNC = 0x00110000; // DELETE | SYNCHRONIZE
const SHARE_READ_WRITE_DELETE = 0x7;
const OPEN_EXISTING = 3;
const OPEN_ALWAYS = 4;
const FILE_ATTRIBUTE_NORMAL = 0x80;
const LOCKFILE_EXCLUSIVE_FAIL_IMMEDIATELY = 0x3;
const FILE_RENAME_INFO_EX = 22;
const RENAME_REPLACE_POSIX = 0x3;
const INVALID_HANDLE_VALUE = -1;
const LOCK_EX_NB = 6;
const LOCK_UN = 8;
const EINTR = 4;
type Kernel32 = {
createFile(path: string, access: number, share: number, disposition: number): number;
lockFileEx(handle: number): boolean;
unlockFileEx(handle: number): void;
closeHandle(handle: number): void;
setRenameInfo(handle: number, buffer: Buffer): boolean;
moveFileEx(staged: string, destination: string, flags: number): boolean;
lastError(): number;
};
type Libc = {
flock(fd: number, operation: number): number;
errno(): number;
};
let kernel32Cache: Kernel32 | undefined;
let libcCache: Libc | undefined;
function loadKernel32(): Kernel32 {
if (kernel32Cache) {
return kernel32Cache;
}
// eslint-disable-next-line @typescript-eslint/no-var-requires
const koffi = require("koffi");
const lib = koffi.load("kernel32.dll");
const createFileW = lib.func(
"intptr_t CreateFileW(str16 name, uint32_t access, uint32_t share, void *security, uint32_t disposition, uint32_t flags, void *template)",
);
const lockFileEx = lib.func(
"int LockFileEx(intptr_t handle, uint32_t flags, uint32_t reserved, uint32_t low, uint32_t high, void *overlapped)",
);
const unlockFileEx = lib.func(
"int UnlockFileEx(intptr_t handle, uint32_t reserved, uint32_t low, uint32_t high, void *overlapped)",
);
const closeHandle = lib.func("int CloseHandle(intptr_t handle)");
const setFileInformationByHandle = lib.func(
"int SetFileInformationByHandle(intptr_t handle, int infoClass, void *info, uint32_t size)",
);
const moveFileExW = lib.func("int MoveFileExW(str16 existing, str16 destination, uint32_t flags)");
const getLastError = lib.func("uint32_t GetLastError()");
kernel32Cache = {
createFile: (path, access, share, disposition) =>
Number(createFileW(path, access, share, null, disposition, FILE_ATTRIBUTE_NORMAL, null)),
lockFileEx: (handle) => lockFileEx(handle, LOCKFILE_EXCLUSIVE_FAIL_IMMEDIATELY, 0, 0xffffffff, 0xffffffff, Buffer.alloc(32)) !== 0,
unlockFileEx: (handle) => {
unlockFileEx(handle, 0, 0xffffffff, 0xffffffff, Buffer.alloc(32));
},
closeHandle: (handle) => {
closeHandle(handle);
},
setRenameInfo: (handle, buffer) => setFileInformationByHandle(handle, FILE_RENAME_INFO_EX, buffer, buffer.length) !== 0,
moveFileEx: (staged, destination, flags) => moveFileExW(staged, destination, flags) !== 0,
lastError: () => Number(getLastError()),
};
return kernel32Cache;
}
function loadLibc(platform: "linux" | "darwin"): Libc {
if (libcCache) {
return libcCache;
}
// eslint-disable-next-line @typescript-eslint/no-var-requires
const koffi = require("koffi");
const lib = koffi.load(platform === "darwin" ? "libSystem.B.dylib" : "libc.so.6");
const flock = lib.func("int flock(int fd, int operation)");
libcCache = {
flock: (fd, operation) => flock(fd, operation),
errno: () => koffi.errno(),
};
return libcCache;
}
/** The bound kernel32 replace calls; exported for the tests that run them against the real kernel. */
export function windowsKernel(): ReplaceKernel {
const k = loadKernel32();
return {
renamePosix: (staged, destination) => {
const handle = k.createFile(staged, GENERIC_ALL_DELETE_SYNC, SHARE_READ_WRITE_DELETE, OPEN_EXISTING);
if (handle === INVALID_HANDLE_VALUE) {
return k.lastError();
}
try {
const name = Buffer.from(destination, "utf16le");
const info = Buffer.alloc(20 + name.length + 2);
info.writeUInt32LE(RENAME_REPLACE_POSIX, 0);
info.writeUInt32LE(name.length, 16);
name.copy(info, 20);
return k.setRenameInfo(handle, info) ? 0 : k.lastError();
} finally {
k.closeHandle(handle);
}
},
moveFileEx: (staged, destination, flags) => (k.moveFileEx(staged, destination, flags) ? 0 : k.lastError()),
sleep,
};
}
function tryLockWindows(lockPath: string): (() => void) | undefined {
const k = loadKernel32();
const handle = k.createFile(lockPath, GENERIC_READ_WRITE, SHARE_READ_WRITE_DELETE, OPEN_ALWAYS);
if (handle === INVALID_HANDLE_VALUE) {
const code = k.lastError();
if (code === ERROR_SHARING_VIOLATION) {
// A C# holder opened the lock file share-none.
return undefined;
}
throw new Error(`CultCache could not open the store lock ${lockPath}: Win32 error ${code}.`);
}
if (!k.lockFileEx(handle)) {
const code = k.lastError();
k.closeHandle(handle);
if (code === ERROR_LOCK_VIOLATION) {
// A Rust or TypeScript holder holds the exclusive byte-range lock.
return undefined;
}
throw new Error(`CultCache could not lock the store lock ${lockPath}: Win32 error ${code}.`);
}
return () => {
k.unlockFileEx(handle);
k.closeHandle(handle);
};
}
function tryLockUnix(lockPath: string, platform: "linux" | "darwin"): (() => void) | undefined {
const libc = loadLibc(platform);
const wouldBlock = platform === "darwin" ? 35 : 11; // EWOULDBLOCK == EAGAIN
const fd = openSync(lockPath, fsConstants.O_RDWR | fsConstants.O_CREAT, 0o666);
for (;;) {
if (libc.flock(fd, LOCK_EX_NB) === 0) {
return () => {
libc.flock(fd, LOCK_UN);
closeSync(fd);
};
}
const errno = libc.errno();
if (errno === EINTR) {
continue;
}
closeSync(fd);
if (errno === wouldBlock) {
return undefined;
}
throw new Error(`CultCache could not lock the store lock ${lockPath}: errno ${errno}.`);
}
}

102
packages/cultcache-ts/test/cult-cache.test.ts
View file

@ -2532,3 +2532,105 @@ test("every registration refusal is a SchemaConflictError naming the id and both
}
}
});
// ---------------------------------------------------------------------------------------------
// Cross-runtime store lock: each runtime's writer excludes the others on `<file>.lock`.
import { spawn } from "node:child_process";
import { withStoreWriteLock } from "../src/store-file";
type LockHolder = { exited: Promise<number | null>; stderr: () => string };
/** Starts a peer's `hold-lock` mode and resolves once it prints {"held":true}. */
async function holdLockWith(command: string, args: string[], cwd: string): Promise<LockHolder> {
const child = spawn(command, args, { cwd, stdio: ["ignore", "pipe", "pipe"] });
let errors = "";
child.stderr.on("data", (chunk) => {
errors += String(chunk);
});
const exited = new Promise<number | null>((resolveExit) => child.on("exit", (code) => resolveExit(code)));
await new Promise<void>((resolveHeld, rejectHeld) => {
let seen = "";
child.stdout.on("data", (chunk) => {
seen += String(chunk);
if (seen.includes('"held":true')) {
resolveHeld();
}
});
void exited.then((code) => rejectHeld(new Error(`hold-lock exited ${code} before holding: ${errors}`)));
});
return { exited, stderr: () => errors };
}
const lockHolders: Record<"csharp" | "rust", (file: string, ms: number) => Promise<LockHolder>> = {
csharp: (file, ms) =>
holdLockWith(dotnetCommand, [csharpInteropDll, "hold-lock", "--file", file, "--ms", String(ms)], cultLibRoot),
rust: (file, ms) =>
holdLockWith(rustInteropBinary, ["hold-lock", "--file", file, "--ms", String(ms)], cultcacheRsRoot),
};
const lockProbeEntry = (key: string): CultCacheEnvelope => ({
key,
type: "t",
payload: documentPayloadFromBytes(Uint8Array.of(0x90)),
storedAt: "2026-10-20T00:00:00Z",
});
for (const holder of ["csharp", "rust"] as const) {
test(`a TypeScript write waits while a ${holder === "csharp" ? "C#" : "Rust"} writer holds the store lock`, async () => {
await buildInteropPeers();
const dir = await mkdtemp(join(tmpdir(), "cultcache-lock-ts-waits-"));
const file = join(dir, "store.cc");
const held = await lockHolders[holder](file, 1500);
let finished = false;
const push = new SingleFileMessagePackBackingStore(file).push(lockProbeEntry("ts-record")).then(() => {
finished = true;
});
await new Promise((resolveWait) => setTimeout(resolveWait, 500));
assert.equal(finished, false, "the push must wait for the foreign lock");
assert.equal(existsSync(file), false, "nothing is written while the foreign writer holds the lock");
assert.equal(await held.exited, 0, held.stderr());
await push;
const keys = (await new SingleFileMessagePackBackingStore(file).pullAll()).map((entry) => entry.key);
assert.deepEqual(keys, ["ts-record"]);
});
}
for (const writer of ["csharp", "rust"] as const) {
test(`a ${writer === "csharp" ? "C#" : "Rust"} writer waits while a TypeScript writer holds the store lock`, async () => {
await buildInteropPeers();
const dir = await mkdtemp(join(tmpdir(), "cultcache-lock-peer-waits-"));
const file = join(dir, "store.cc");
const noteKey = `note:${writer}-writer`;
let peer: Promise<unknown> | undefined;
let peerDone = false;
await withStoreWriteLock(file, async () => {
peer =
writer === "csharp"
? execFileAsync(dotnetCommand, [csharpInteropDll, "write", "--file", file, "--runtime-id", "csharp-writer"], { cwd: cultLibRoot, timeout: 60_000 })
: execFileAsync(rustInteropBinary, ["write", "--file", file, "--runtime-id", "rust-writer"], { cwd: cultcacheRsRoot, timeout: 60_000 });
void peer.then(
() => (peerDone = true),
() => (peerDone = true),
);
await new Promise((resolveWait) => setTimeout(resolveWait, 1500));
assert.equal(peerDone, false, "the peer must still be waiting for the lock");
assert.equal(existsSync(file), false, `${noteKey} is absent while TypeScript holds the lock`);
});
await peer;
const keys = (await new SingleFileMessagePackBackingStore(file).pullAll()).map((entry) => entry.key);
assert.ok(keys.includes(noteKey), `${noteKey} is in the store after release`);
});
}
test("the store lock gives up on a Rust holder after its wait bound and names the lock path", async () => {
await buildInteropPeers();
const dir = await mkdtemp(join(tmpdir(), "cultcache-lock-timeout-"));
const file = join(dir, "store.cc");
const held = await lockHolders.rust(file, 2000);
await assert.rejects(
withStoreWriteLock(file, async () => undefined, { timeoutMs: 200 }),
(error: Error) => error.message.includes(`${file}.lock`),
);
assert.equal(await held.exited, 0, held.stderr());
});

716
packages/cultcache-ts/test/store-file.test.ts Normal file
View file

@ -0,0 +1,716 @@
import assert from "node:assert/strict";
import { execFile } from "node:child_process";
import { mkdirSync, readFileSync } from "node:fs";
import { mkdtemp, readFile, readdir, rm, writeFile } from "node:fs/promises";
import fsp = require("node:fs/promises");
import { tmpdir } from "node:os";
import { basename, join, resolve } from "node:path";
import test from "node:test";
import { promisify } from "node:util";
import { decode, encode } from "@msgpack/msgpack";
import { documentPayloadBytes, documentPayloadFromBytes } from "../src/document";
import { SingleFileMessagePackBackingStore } from "../src/single-file-messagepack-backing-store";
import type { CultCacheEnvelope } from "../src/types";
import {
MOVEFILE_REPLACE_WRITE_THROUGH,
StoreReplacedNotDurableError,
commitStoreFile,
nodeStoreFileOps,
replaceStoreFile,
replaceStoreFileWindows,
windowsKernel,
withStoreWriteLock,
} from "../src/store-file";
import type { ReplaceKernel, StoreFileOps } from "../src/store-file";
const execFileAsync = promisify(execFile);
async function scratch(): Promise<string> {
return mkdtemp(join(tmpdir(), "cultcache-store-file-"));
}
function recordingOps(calls: string[], overrides: Partial<StoreFileOps> = {}): StoreFileOps {
const real = nodeStoreFileOps;
return {
createExclusive: async (path) => {
calls.push("create");
return real.createExclusive(path);
},
write: async (handle, bytes) => {
calls.push("write");
await real.write(handle, bytes);
},
sync: async (handle) => {
calls.push("sync");
await real.sync(handle);
},
close: async (handle) => {
calls.push("close");
await real.close(handle);
},
replace: async (staged, destination) => {
calls.push("replace");
await real.replace(staged, destination);
},
syncDirectory: async (directory) => {
calls.push("syncDirectory");
await real.syncDirectory(directory);
},
remove: async (path) => {
calls.push("remove");
await real.remove(path);
},
...overrides,
};
}
test("a commit syncs the staged file before it replaces the store, and the directory after (Unix)", async () => {
const directory = await scratch();
try {
const store = join(directory, "store.cc");
const calls: string[] = [];
await commitStoreFile(store, new Uint8Array([1, 2, 3]), recordingOps(calls));
const expected = ["create", "write", "sync", "close", "replace"];
if (process.platform !== "win32") {
expected.push("syncDirectory");
}
assert.deepEqual(calls, expected);
assert.deepEqual([...(await readFile(store))], [1, 2, 3]);
assert.deepEqual(await readdir(directory), ["store.cc"]);
} finally {
await rm(directory, { recursive: true, force: true });
}
});
// The default write path, with nothing injected: every file handle it opens is observed through
// fs/promises.open (the module object, read at call time), recording what the handle did and what the
// store held at that moment. The replace is seen as the store's bytes changing, so the order holds on
// every platform without a seam in the replace itself.
test("the default write path syncs and closes the staged file before the store changes, and syncs the directory after (Unix)", async () => {
const directory = await scratch();
const realOpen = fsp.open;
try {
const store = join(directory, "store.cc");
await writeFile(store, "old");
const storeNow = () => readFileSync(store, "utf8");
const events: string[] = [];
(fsp as { open: typeof fsp.open }).open = (async (path: string, flags?: string) => {
const handle = await realOpen(path, flags as never);
const kind = basename(String(path)) === basename(directory) ? "dir" : String(path).includes(".tmp-") ? "staged" : "other";
const sync = handle.sync.bind(handle);
const close = handle.close.bind(handle);
handle.sync = () => {
events.push(`sync:${kind}:${storeNow()}`);
return sync();
};
handle.close = () => {
events.push(`close:${kind}:${storeNow()}`);
return close();
};
return handle;
}) as typeof fsp.open;
await commitStoreFile(store, new TextEncoder().encode("new"));
(fsp as { open: typeof fsp.open }).open = realOpen;
// The staged file is synced, then closed, while the store still holds its old bytes.
assert.deepEqual(events.filter((event) => event.includes(":staged:")), ["sync:staged:old", "close:staged:old"]);
if (process.platform === "win32") {
assert.deepEqual(events.filter((event) => event.includes(":dir:")), [], "Windows relies on the replace and opens no directory");
} else {
// The directory is synced and closed after the store changed.
assert.deepEqual(events.filter((event) => event.includes(":dir:")), ["sync:dir:new", "close:dir:new"]);
}
assert.equal(storeNow(), "new");
} finally {
(fsp as { open: typeof fsp.open }).open = realOpen;
await rm(directory, { recursive: true, force: true });
}
});
test("a commit that fails before the replace leaves the store byte for byte and no staged file behind", async () => {
const directory = await scratch();
try {
const store = join(directory, "store.cc");
await writeFile(store, Buffer.from([9, 9, 9]));
const calls: string[] = [];
const ops = recordingOps(calls, {
replace: async () => {
calls.push("replace");
throw new Error("replace refused");
},
});
await assert.rejects(commitStoreFile(store, new Uint8Array([1]), ops), /replace refused/);
assert.deepEqual([...(await readFile(store))], [9, 9, 9]);
assert.deepEqual(await readdir(directory), ["store.cc"]);
assert.ok(calls.includes("remove"));
assert.equal(calls.filter((call) => call === "close").length, 1, "the staged handle is closed exactly once");
assert.ok(!calls.includes("syncDirectory"));
} finally {
await rm(directory, { recursive: true, force: true });
}
});
test("a commit whose staged file cannot be synced never replaces the store", async () => {
const directory = await scratch();
try {
const store = join(directory, "store.cc");
await writeFile(store, Buffer.from([9]));
const calls: string[] = [];
const ops = recordingOps(calls, {
sync: async () => {
calls.push("sync");
throw new Error("disk full");
},
});
await assert.rejects(commitStoreFile(store, new Uint8Array([1]), ops), /disk full/);
assert.ok(!calls.includes("replace"));
assert.ok(calls.indexOf("close") !== -1 && calls.indexOf("close") < calls.indexOf("remove"), "the staged handle is closed, then the file removed");
assert.deepEqual([...(await readFile(store))], [9]);
assert.deepEqual(await readdir(directory), ["store.cc"]);
} finally {
await rm(directory, { recursive: true, force: true });
}
});
test(
"a directory sync failure after the replace reports the store replaced but not durable",
{ skip: process.platform === "win32" },
async () => {
const directory = await scratch();
try {
const store = join(directory, "store.cc");
await writeFile(store, Buffer.from([9]));
const ops = recordingOps([], {
syncDirectory: async () => {
throw new Error("EIO");
},
});
await assert.rejects(commitStoreFile(store, new Uint8Array([1]), ops), (error: unknown) => {
assert.ok(error instanceof StoreReplacedNotDurableError);
assert.equal(error.replaced, true);
assert.equal(error.name, "StoreReplacedNotDurableError");
assert.match(error.message, /was replaced/);
return true;
});
assert.deepEqual([...(await readFile(store))], [1]);
} finally {
await rm(directory, { recursive: true, force: true });
}
},
);
function fakeKernel(
posix: number[],
moveEx: number[],
): { kernel: ReplaceKernel; log: string[]; sleeps: number[] } {
const log: string[] = [];
const sleeps: number[] = [];
const kernel: ReplaceKernel = {
renamePosix: () => {
log.push("posix");
return posix.shift() ?? 0;
},
moveFileEx: (_staged, _destination, flags) => {
log.push(`moveex:${flags}`);
return moveEx.shift() ?? 0;
},
sleep: async (ms) => {
sleeps.push(ms);
},
};
return { kernel, log, sleeps };
}
test("the Windows replace falls back to write-through MoveFileExW where POSIX rename is unsupported, retries 5/32/33, and gives up after 25 attempts", async () => {
for (const unsupported of [1, 50, 87]) {
const { kernel, log } = fakeKernel([unsupported], [0]);
await replaceStoreFileWindows("a", "b", kernel);
assert.deepEqual(log, ["posix", `moveex:${MOVEFILE_REPLACE_WRITE_THROUGH}`]);
}
assert.equal(MOVEFILE_REPLACE_WRITE_THROUGH, 0x9);
// Once POSIX is unsupported the rest of the call stays on MoveFileExW.
{
const { kernel, log } = fakeKernel([87], [32, 0]);
await replaceStoreFileWindows("a", "b", kernel);
assert.deepEqual(log, ["posix", "moveex:9", "moveex:9"]);
}
for (const transient of [5, 32, 33]) {
const { kernel, log, sleeps } = fakeKernel([transient, transient, 0], []);
await replaceStoreFileWindows("a", "b", kernel);
assert.deepEqual(log, ["posix", "posix", "posix"], "a transient error retries the same primitive");
assert.deepEqual(sleeps, [20, 20]);
}
// A POSIX rename that works is the whole replace.
{
const { kernel, log, sleeps } = fakeKernel([0], []);
await replaceStoreFileWindows("a", "b", kernel);
assert.deepEqual(log, ["posix"]);
assert.deepEqual(sleeps, []);
}
// Once on MoveFileExW, an "unsupported" code is an error in any later attempt, never another fallback.
{
const { kernel, log } = fakeKernel([87], [32, 87]);
await assert.rejects(replaceStoreFileWindows("a", "b", kernel), /Win32 error 87/);
assert.deepEqual(log, ["posix", "moveex:9", "moveex:9"]);
}
// After the fallback, an "unsupported" code is an error, not another fallback.
for (const unsupported of [1, 50, 87]) {
const { kernel, log } = fakeKernel([87], [unsupported]);
await assert.rejects(replaceStoreFileWindows("a", "b", kernel), new RegExp(`Win32 error ${unsupported}`));
assert.deepEqual(log, ["posix", "moveex:9"]);
}
{
const { kernel, log } = fakeKernel(Array(30).fill(32), []);
await assert.rejects(replaceStoreFileWindows("a", "b", kernel), /after 25 attempts.*32/);
assert.equal(log.length, 25);
}
{
const { kernel, log } = fakeKernel([2], []);
await assert.rejects(replaceStoreFileWindows("a", "b", kernel), /Win32 error 2/);
assert.equal(log.length, 1);
}
{
const { kernel } = fakeKernel([87], [3]);
await assert.rejects(replaceStoreFileWindows("a", "b", kernel), /Win32 error 3/);
}
});
// The reader is a koffi CreateFileW holding the store sharing read, write and delete, the way
// the C# and Rust readers hold it.
function openSharingAll(path: string): { close: () => void } {
// eslint-disable-next-line @typescript-eslint/no-var-requires
const koffi = require("koffi");
const lib = koffi.load("kernel32.dll");
const createFileW = lib.func(
"intptr_t CreateFileW(str16 name, uint32_t access, uint32_t share, void *security, uint32_t disposition, uint32_t flags, void *template)",
);
const closeHandle = lib.func("int CloseHandle(intptr_t handle)");
const handle = Number(createFileW(path, 0x80000000, 0x7, null, 3, 0x80, null));
assert.notEqual(handle, -1, "the reader could not open the store");
let closed = false;
return { close: () => { if (!closed) { closed = true; closeHandle(handle); } } };
}
function openSharingReadWrite(path: string): { close: () => void } {
// eslint-disable-next-line @typescript-eslint/no-var-requires
const koffi = require("koffi");
const lib = koffi.load("kernel32.dll");
const createFileW = lib.func(
"intptr_t CreateFileW(str16 name, uint32_t access, uint32_t share, void *security, uint32_t disposition, uint32_t flags, void *template)",
);
const closeHandle = lib.func("int CloseHandle(intptr_t handle)");
const handle = Number(createFileW(path, 0x80000000, 0x3, null, 3, 0x80, null));
assert.notEqual(handle, -1, "the reader could not open the store");
let closed = false;
return { close: () => { if (!closed) { closed = true; closeHandle(handle); } } };
}
test(
"windows: a store replace lands while a reader holds the store sharing delete, as C# and Rust readers do",
{ skip: process.platform !== "win32" },
async () => {
const directory = await scratch();
try {
const store = join(directory, "store.cc");
const staged = join(directory, "staged.tmp");
await writeFile(store, Buffer.from([1, 1, 1]));
await writeFile(staged, Buffer.from([2, 2, 2]));
const reader = openSharingAll(store);
try {
await replaceStoreFile(staged, store);
assert.deepEqual([...(await readFile(store))], [2, 2, 2]);
} finally {
reader.close();
}
} finally {
await rm(directory, { recursive: true, force: true });
}
},
);
test(
"windows: a replace waits out a reader that holds the store without delete sharing",
{ skip: process.platform !== "win32" },
async () => {
const directory = await scratch();
try {
const store = join(directory, "store.cc");
const staged = join(directory, "staged.tmp");
await writeFile(store, Buffer.from([1, 1, 1]));
await writeFile(staged, Buffer.from([2, 2, 2]));
const reader = openSharingReadWrite(store);
const release = setTimeout(() => reader.close(), 120);
try {
await replaceStoreFile(staged, store);
assert.deepEqual([...(await readFile(store))], [2, 2, 2]);
} finally {
clearTimeout(release);
reader.close();
}
} finally {
await rm(directory, { recursive: true, force: true });
}
},
);
// ---------------------------------------------------------------------------------------------
// The cross-process write lock (TypeScript against TypeScript; C# and Rust are in cult-cache.test.ts).
const entryOf = (key: string, type = "t", marker = 0): CultCacheEnvelope => ({
key,
type,
payload: documentPayloadFromBytes(Uint8Array.of(0x91, marker)),
storedAt: "2026-10-20T00:00:00Z",
});
function sleepFor(milliseconds: number): Promise<void> {
return new Promise((resolveSleep) => setTimeout(resolveSleep, milliseconds));
}
function within<T>(promise: Promise<T>, milliseconds: number, what: string): Promise<T> {
let timer: NodeJS.Timeout | undefined;
const expired = new Promise<never>((_resolve, reject) => {
timer = setTimeout(() => reject(new Error(`${what} did not finish within ${milliseconds} ms (deadlock)`)), milliseconds);
});
return Promise.race([promise, expired]).finally(() => clearTimeout(timer));
}
test("a second writer in this process waits for the store lock and runs after release", async () => {
const directory = await scratch();
try {
const store = join(directory, "store.cc");
const order: string[] = [];
let second: Promise<void> | undefined;
await withStoreWriteLock(store, async () => {
second = withStoreWriteLock(store, async () => {
order.push("second");
});
await sleepFor(150);
order.push("first-done");
});
await second;
assert.deepEqual(order, ["first-done", "second"]);
} finally {
await rm(directory, { recursive: true, force: true });
}
});
test("the store lock gives up after its wait bound and names the lock path", async () => {
const directory = await scratch();
try {
const store = join(directory, "store.cc");
await withStoreWriteLock(store, async () => {
await assert.rejects(
withStoreWriteLock(store, async () => undefined, { timeoutMs: 200 }),
(error: Error) => error.message.includes(`${store}.lock`) && /200 ms/.test(error.message),
);
});
// The failed waiter released nothing it did not hold: the lock is free again.
assert.equal(await withStoreWriteLock(store, async () => "free", { timeoutMs: 200 }), "free");
} finally {
await rm(directory, { recursive: true, force: true });
}
});
test("a directory at the lock path fails at once with the lock path named, not after the 30 s wait", async () => {
const directory = await scratch();
try {
const store = join(directory, "store.cc");
mkdirSync(`${store}.lock`);
const started = Date.now();
let ran = false;
// The default wait bound (30 s) is in force: only a failure that does not wait can beat the race.
const outcome = await Promise.race([
withStoreWriteLock(store, async () => {
ran = true;
}).then(
() => ({ error: undefined as Error | undefined }),
(error: Error) => ({ error }),
),
new Promise<"waiting">((resolveWait) => setTimeout(() => resolveWait("waiting"), 3000).unref()),
]);
assert.notEqual(outcome, "waiting", "the lock attempt was still waiting after 3 s");
const { error } = outcome as { error: Error | undefined };
assert.ok(error, "locking a directory must fail");
assert.ok(error.message.includes(`${store}.lock`), error.message);
assert.ok(!/timed out/.test(error.message), error.message);
assert.ok(Date.now() - started < 3000);
assert.equal(ran, false);
} finally {
await rm(directory, { recursive: true, force: true });
}
});
test("the store lock is released when the operation throws", async () => {
const directory = await scratch();
try {
const store = join(directory, "store.cc");
await assert.rejects(withStoreWriteLock(store, async () => Promise.reject(new Error("boom"))), /boom/);
assert.equal(await withStoreWriteLock(store, async () => "free", { timeoutMs: 200 }), "free");
} finally {
await rm(directory, { recursive: true, force: true });
}
});
const legacyObjectPayload = (): Uint8Array =>
encode([{ key: "note:hello", type: "note", payload: { title: "Hello" }, storedAt: "2026-10-20T00:00:00Z" }]);
test("an open that heals a legacy store waits for the store lock and heals from its own read", async () => {
const directory = await scratch();
try {
const path = join(directory, "legacy.cc");
const legacy = legacyObjectPayload();
await writeFile(path, legacy);
const store = new SingleFileMessagePackBackingStore(path);
let healed: Promise<unknown> | undefined;
let settled = false;
await withStoreWriteLock(path, async () => {
healed = store.pullAll().then(() => {
settled = true;
});
await sleepFor(300);
assert.equal(settled, false, "the heal must wait for the lock");
assert.deepEqual([...(await readFile(path))], [...legacy], "the file keeps its legacy bytes while the lock is held elsewhere");
});
await within(healed!, 5000, "the heal");
assert.equal((decode(await readFile(path)) as unknown[])[0], "cultcache.store.v1");
} finally {
await rm(directory, { recursive: true, force: true });
}
});
test("push and delete over a legacy store that needs healing finish instead of waiting on their own lock", async () => {
const directory = await scratch();
try {
const path = join(directory, "legacy.cc");
await writeFile(path, legacyObjectPayload());
const store = new SingleFileMessagePackBackingStore(path);
await within(store.push(entryOf("a")), 5000, "push");
await writeFile(path, legacyObjectPayload());
await within(store.delete(entryOf("a")), 5000, "delete");
const keys = (await store.pullAll()).map((entry) => entry.key);
assert.deepEqual(keys, ["note:hello"]);
} finally {
await rm(directory, { recursive: true, force: true });
}
});
test("two TypeScript processes pushing distinct keys to one store lose none", async () => {
const directory = await scratch();
try {
const path = join(directory, "store.cc");
const dist = resolve(__dirname, "..", "..", "dist");
const script = `
const { SingleFileMessagePackBackingStore } = require(${JSON.stringify(join(dist, "single-file-messagepack-backing-store.js"))});
const { documentPayloadFromBytes } = require(${JSON.stringify(join(dist, "document.js"))});
(async () => {
const store = new SingleFileMessagePackBackingStore(${JSON.stringify(path)});
for (let i = 0; i < 40; i++) {
await store.push({ key: process.argv[1] + "-" + i, type: "t", payload: documentPayloadFromBytes(Uint8Array.of(0x90)), storedAt: "2026-10-20T00:00:00Z" });
}
})().catch((error) => { console.error(error); process.exit(1); });
`;
await Promise.all(
["p", "q"].map((name) => execFileAsync(process.execPath, ["-e", script, name], { timeout: 120_000 })),
);
const keys = (await new SingleFileMessagePackBackingStore(path).pullAll()).map((entry) => entry.key).sort();
assert.equal(keys.length, 80);
assert.equal(new Set(keys).size, 80);
} finally {
await rm(directory, { recursive: true, force: true });
}
});
const snapshotOf = async (store: SingleFileMessagePackBackingStore): Promise<string[]> =>
(await store.pullAll())
.map((entry) => `${entry.type}/${entry.key}:${[...documentPayloadBytes(entry.payload)].at(-1)}`)
.sort();
test("push replaces the record at the same type and key and keeps every other", async () => {
const directory = await scratch();
try {
const store = new SingleFileMessagePackBackingStore(join(directory, "store.cc"));
await store.pushAll([entryOf("a", "t", 1), entryOf("b", "t", 2), entryOf("a", "u", 3)]);
await store.push(entryOf("a", "t", 9));
assert.deepEqual(await snapshotOf(store), ["t/a:9", "t/b:2", "u/a:3"]);
await store.push(entryOf("c", "t", 4));
assert.deepEqual(await snapshotOf(store), ["t/a:9", "t/b:2", "t/c:4", "u/a:3"]);
} finally {
await rm(directory, { recursive: true, force: true });
}
});
test("delete removes only the record at that type and key", async () => {
const directory = await scratch();
try {
const store = new SingleFileMessagePackBackingStore(join(directory, "store.cc"));
await store.pushAll([entryOf("a", "t", 1), entryOf("b", "t", 2), entryOf("a", "u", 3)]);
await store.delete(entryOf("a", "t"));
assert.deepEqual(await snapshotOf(store), ["t/b:2", "u/a:3"]);
await store.delete(entryOf("zzz", "t"));
assert.deepEqual(await snapshotOf(store), ["t/b:2", "u/a:3"]);
} finally {
await rm(directory, { recursive: true, force: true });
}
});
test("pushAll replaces the whole store, and a soft pushAll leaves an existing store alone", async () => {
const directory = await scratch();
try {
const path = join(directory, "store.cc");
const store = new SingleFileMessagePackBackingStore(path);
await store.pushAll([entryOf("a", "t", 1)], { soft: true });
assert.deepEqual(await snapshotOf(store), ["t/a:1"]);
const before = await readFile(path);
await store.pushAll([entryOf("b", "t", 2)], { soft: true });
assert.deepEqual([...(await readFile(path))], [...before], "soft leaves an existing store byte for byte");
await store.pushAll([entryOf("b", "t", 2)]);
assert.deepEqual(await snapshotOf(store), ["t/b:2"]);
} finally {
await rm(directory, { recursive: true, force: true });
}
});
test("a read of a current store never waits for the store lock, and a missing store reads empty", async () => {
const directory = await scratch();
try {
const path = join(directory, "store.cc");
const store = new SingleFileMessagePackBackingStore(path);
await store.push(entryOf("a"));
await withStoreWriteLock(path, async () => {
assert.deepEqual(await within(store.pullAll().then((rows) => rows.map((row) => row.key)), 2000, "a read"), ["a"]);
const missing = new SingleFileMessagePackBackingStore(join(directory, "absent.cc"));
assert.deepEqual(await within(missing.pullAll(), 2000, "a read of a missing store"), []);
});
} finally {
await rm(directory, { recursive: true, force: true });
}
});
// A share-none open succeeds only when no other handle on the file is open, as C# readers and
// writers need it to.
function canOpenShareNone(path: string): boolean {
// eslint-disable-next-line @typescript-eslint/no-var-requires
const koffi = require("koffi");
const lib = koffi.load("kernel32.dll");
const createFileW = lib.func(
"intptr_t CreateFileW(str16 name, uint32_t access, uint32_t share, void *security, uint32_t disposition, uint32_t flags, void *template)",
);
const closeHandle = lib.func("int CloseHandle(intptr_t handle)");
const handle = Number(createFileW(path, 0x80000000, 0, null, 3, 0x80, null));
if (handle === -1) {
return false;
}
closeHandle(handle);
return true;
}
test(
"windows: the real kernel replaces through both primitives and reports the Win32 error of a missing source",
{ skip: process.platform !== "win32" },
async () => {
const directory = await scratch();
try {
const kernel = windowsKernel();
for (const [name, replace] of [
["posix", (staged: string, store: string) => kernel.renamePosix(staged, store)],
["moveex", (staged: string, store: string) => kernel.moveFileEx(staged, store, MOVEFILE_REPLACE_WRITE_THROUGH)],
] as const) {
const store = join(directory, `${name}.cc`);
const staged = join(directory, `${name}.tmp`);
await writeFile(store, Buffer.from([1]));
await writeFile(staged, Buffer.from([2]));
assert.equal(replace(staged, store), 0, name);
assert.ok(canOpenShareNone(store), `${name}: the replace left a handle open on the store`);
assert.deepEqual([...(await readFile(store))], [2], name);
assert.deepEqual(await readdir(directory).then((names) => names.filter((n) => n === `${name}.tmp`)), [], name);
assert.equal(replace(join(directory, "missing.tmp"), store), 2, `${name}: ERROR_FILE_NOT_FOUND`);
}
} finally {
await rm(directory, { recursive: true, force: true });
}
},
);
test(
"windows: after a write and after a waiter gave up, the lock file is held by no handle, so a share-none open (C#'s lock) succeeds",
{ skip: process.platform !== "win32" },
async () => {
const directory = await scratch();
try {
const store = join(directory, "store.cc");
await withStoreWriteLock(store, async () => {
// A waiter that polls and gives up must not leave a handle on the lock file either.
await assert.rejects(withStoreWriteLock(store, async () => undefined, { timeoutMs: 100 }), /timed out/);
});
// eslint-disable-next-line @typescript-eslint/no-var-requires
const koffi = require("koffi");
const lib = koffi.load("kernel32.dll");
const createFileW = lib.func(
"intptr_t CreateFileW(str16 name, uint32_t access, uint32_t share, void *security, uint32_t disposition, uint32_t flags, void *template)",
);
const closeHandle = lib.func("int CloseHandle(intptr_t handle)");
const handle = Number(createFileW(`${store}.lock`, 0xc0000000, 0, null, 4, 0x80, null));
assert.notEqual(handle, -1, "the lock file is still held open after the write");
closeHandle(handle);
} finally {
await rm(directory, { recursive: true, force: true });
}
},
);
test("a store file of an unrecognized shape is refused, and a store naming a missing schema id is refused naming the record", async () => {
const directory = await scratch();
try {
const odd = join(directory, "odd.cc");
await writeFile(odd, encode({ not: "a store" }));
await assert.rejects(new SingleFileMessagePackBackingStore(odd).pullAll(), /is not a recognized CultCache MessagePack store/);
const dangling = join(directory, "dangling.cc");
await writeFile(dangling, encode(["cultcache.store.v1", [], [["rec:1", "sha256:missing", "2026-10-20T00:00:00Z", Uint8Array.of(0x90)]]]));
await assert.rejects(
new SingleFileMessagePackBackingStore(dangling).pullAll(),
/persisted record "rec:1" references missing schema id "sha256:missing"/,
);
} finally {
await rm(directory, { recursive: true, force: true });
}
});
test("opening a store that needs no healing writes nothing: a missing store leaves no lock file, a legacy store with byte payloads keeps its bytes", async () => {
const directory = await scratch();
try {
const absent = join(directory, "nested", "absent.cc");
assert.deepEqual(await new SingleFileMessagePackBackingStore(absent).pullAll(), []);
assert.deepEqual(await readdir(directory), [], "a read of a missing store creates neither its directory nor its lock");
const legacyBytes = encode([{ key: "k", type: "t", payload: Uint8Array.of(0x90), storedAt: "2026-10-20T00:00:00Z" }]);
const path = join(directory, "legacy-bytes.cc");
await writeFile(path, legacyBytes);
assert.deepEqual((await new SingleFileMessagePackBackingStore(path).pullAll()).map((entry) => entry.key), ["k"]);
assert.deepEqual([...(await readFile(path))], [...legacyBytes], "a legacy store whose payloads are already bytes is not rewritten");
assert.deepEqual((await readdir(directory)).sort(), ["legacy-bytes.cc"]);
} finally {
await rm(directory, { recursive: true, force: true });
}
});

30
packages/cultmath/README.md
View file

@ -28,8 +28,19 @@ shader semantics.
`min`, `max`, `clamp`, `saturate`, `lerp`, `step`, `smoothstep`, `dot`,
`cross`, `length`, `distance`, `normalize`, `reflect`, `csum`, `erf`,
`erfinv` (scalar only), exponential
`decay`/`damp`, intercept/segment helpers, Catmull-Rom, Bezier curves, and
`decay`/`damp`, intercept/segment helpers (`distance_to_segment` in 2D and 3D, with and without `out closest`, and `segment_closest`), Catmull-Rom, Bezier curves, and
deterministic value-noise primitives
- Site cells: `power_cells` and `multiplicative_cells` return the two nearest sites of a point as `CultSiteCells`,
`soft_gabriel_gate` gates a pair of sites by the empty-disc margin. Each is a loop over the shader steps
`cultmath_power_score` / `cultmath_multiplicative_score`, `cultmath_site_cells_empty`, `cultmath_site_cells_insert`,
`cultmath_site_cells_border`, `cultmath_power_border_step` (the exact power cell border, `power_border`) and `cultmath_gabriel_margin`; a shader loops over its own buffer.
- Weighted lattice cells: `weighted_lattice_cells` returns `float4(F1, F2, smooth, id)` over a jittered,
drifting lattice whose cell weights come from the exact power border of the body span at each cell centre, with a stretched
metric and an exact ring search over the stated range (`LatticeMaxBoost`, `LatticeMaxStretch`, `LatticeMaxSmoothing`). It composes the
public steps `lattice_site`, `lattice_weight`, `lattice_distance` and `lattice_smooth_step`. The shaders carry the four steps as
`cultmath_lattice_site`, `cultmath_lattice_weight`, `cultmath_lattice_distance` and `cultmath_lattice_smooth_step`, and the whole function as `cultmath_weighted_lattice_cells(p, time, bodies,
count, ...)` over a `float4 bodies[CULTMATH_MAX_SITES]` array (128 by default; define the macro before including to change it);
`cultmath_power_border(p, sites, count)` is the matching border. The argument checks become the shader caller's preconditions.
- `shaders/CultMath.hlsl`, a canonical HLSL mirror include for shader-side
parity. HLSL already owns `float2`, `float3`, and `float4`; the include
exposes `cultmath_*` functions for shared semantics HLSL does not provide.
@ -82,7 +93,11 @@ shader semantics.
the analytic Hessian of `snoise(float3)` on the same lattice, hash and tie order as
`snoise_grad` (a symmetric `float3x3`; GLSL `mat3`), and `integral_clamp_linear(a,
b, lo, hi, t0, t1)`, the closed-form integral over `[t0, t1]` of `clamp(a + b t,
lo, hi)`. Three measured bounds describe `snoise`: `SNOISE_LIPSCHITZ` (gradient
lo, hi)`; and the octave stages `span_fbm(path, stage,
octaves, lacunarity, gain)` and `span_ridged` (shader twins `cultmath_span_fbm` and `cultmath_span_ridged`,
whose callers build the `SpanStage` themselves), over a `SpanPath` and a `SpanStage` (`SpanStage.Identity`
is the plain fbm in C#), which return a `SpanNoise` (`field` as `float4(gradient, value)`,
`curvature`, `fold`) for a stretched, offset, shifted octave sum over a span. Three measured bounds describe `snoise`: `SNOISE_LIPSCHITZ` (gradient
length, `NoiseBoundTests.MeasureLipschitz`), `SNOISE_HESSIAN` (Hessian spectral
norm, `MeasureHessian`) and `SNOISE_THIRD` (third-derivative operator norm,
`MeasureThird`); each is 1.10 times a seeded maximum refined by ascent, and a
@ -164,6 +179,7 @@ CultMath keeps portable DXC outside git under `packages/cultmath/.tools/`:
```powershell
.\packages\cultmath\tools\get-dxc.ps1
.\packages\cultmath\tools\compile-hlsl.ps1
.\packages\cultmath\tools\get-spirv-cross.ps1
.\packages\cultmath\tools\compile-hlsl-spirv.ps1 `
-ShaderPath E:\Projects\Odin\crates\muninn-move-tracker\shaders\MoveSphereCandidate.comp.hlsl `
@ -174,6 +190,10 @@ CultMath keeps portable DXC outside git under `packages/cultmath/.tools/`:
-OutputPath E:\Projects\Odin\crates\muninn-move-tracker\artifacts\shader\MoveSphereCandidate.comp.metal
```
`tools/glsl/build-glsl.sh <outdir>` (Linux, repository root) builds the library GLSL from the HLSL with dxc, SPIRV-Cross and SPIRV-Tools and writes `core.glsl`, `cycles.glsl` and `phacelle.glsl` bodies into `<outdir>`. It does not yet own the committed GLSL; `GlslLowering` still does. `GLSL_DIR=<outdir> tools/device/run-lavapipe.sh` runs that output on lavapipe and compares it with the HLSL results bit for bit instead of judging the committed files. That comparison is in strict form (dxc `-Gis` and a `NoContraction` decoration on every float result of both modules), and a consumer overrides `CULTMATH_MAX_SITES` in the built GLSL the same way as in HLSL: define it before including `core.glsl`.
`compile-hlsl.ps1` compiles both `CultMath.hlsl` copies as a `lib_6_3` library with the x64 DXC; a warning fails it too (`-WX`).
DXC emits Vulkan SPIR-V with `-spirv`. `-TargetEnv vulkan1.2` is exposed by the
script, but the current Windows DXC release compiled this shader successfully
with DXC's default Vulkan target while rejecting the dotted target-env value.
@ -183,6 +203,12 @@ class. SPIRV-Cross emits Metal Shading Language from SPIR-V on Windows; building
an iPad-ready `.metallib` still belongs to Apple's Metal compiler on an Apple
toolchain.
The lavapipe conforming run executes every fixture family through the HLSL (dxc) and the committed GLSL
(glslang) on Mesa lavapipe and judges the results in `GlslMirrorTests.DeviceResultsHoldFixture`. It needs a Linux
container with apt: from the repo root, `tools/stopgap/ygg-verify.sh <repo> <rev> mcr.microsoft.com/dotnet/sdk:10.0
'bash packages/cultmath/tools/device/run-lavapipe.sh'`. `WRITE_BOUNDS=1` in front of the script re-records the
device's bounds file and prints it between `BOUNDS-BEGIN` and `BOUNDS-END`.
## Doctrine
The goal is mechanical sympathy between CPU-side authoring, native kernels, and

219
packages/cultmath/docs/design.md
View file

@ -95,7 +95,8 @@ and HLSL's one-based `_11`..`_33` element names.
Roslyn against CultMath. These transformations are the complete list; if a body
needs anything else, the test fails:
1. Include guards (`#ifndef`/`#define`/`#endif`) are dropped.
1. Include guards (`#ifndef`/`#define`/`#endif`) are dropped, except the capacity macro (`#ifndef CULTMATH_MAX_SITES`,
`#define CULTMATH_MAX_SITES 128`, `#endif`), which becomes `const int CULTMATH_MAX_SITES = 128;`.
2. Functions taking `Texture2D`/`SamplerState` are dropped; GPU resources have
no CultMath analog.
3. File-scope `static const` becomes `const`.
@ -103,7 +104,8 @@ needs anything else, the test fails:
5. Struct fields gain `public`. HLSL struct members have no access-modifier
concept; a C# struct's fields default to `private`, which would hide them
from the mirror test's reflection-based field comparison.
6. The file body is wrapped in a class, since C# has no free functions; HLSL
6. An array parameter `T name[CULTMATH_MAX_SITES]` becomes `T[] name`.
7. The file body is wrapped in a class, since C# has no free functions; HLSL
functions become private instance methods.
HLSL constructs that stay outside the target and must not appear in the mirror:
@ -112,8 +114,10 @@ local) and writes through a swizzle component (`v.xy.x = s`, which C# rejects as
modifying a temporary).
The HLSL side is checked too: `tools/get-dxc.ps1` fetches Microsoft's
DirectXShaderCompiler, and
`dxc -T lib_6_3 -HV 2021 shaders/CultMath.hlsl` must compile cleanly.
DirectXShaderCompiler (x64), and `tools/compile-hlsl.ps1` compiles both copies of
`CultMath.hlsl` with it (`dxc -T lib_6_3 -HV 2021 -WX`, so a warning fails like an error and the
text must compile cleanly); the script throws when a shader file is missing, when dxc cannot
launch or when it exits non-zero, so a stale exit code never reads as a pass.
The mirror test compares bit patterns (any NaN equals any NaN; -0 differs from
0). It proves the text of `CultMath.hlsl` computes the same float32 results as
@ -123,13 +127,21 @@ compiled under `using static CultMath.math;`, so its intrinsics are C# `math`
itself: the test proves the composition in the text, not the intrinsic rules,
which only `HlslSemanticsTests` pins.
Every mirror function is compared, internals included: its counterpart is the `math`
function of the same name, public or private (`snoise_mod289` and `snoise_permute` are
private), and a function with no counterpart fails by name. A struct parameter maps to the
CultMath type of the same name, and only if both have the same instance fields by name, type
and order (`SpanPath`, `SpanStage`); the argument is built from the case's scalars field by
field and copied to its C# twin by field name.
The comparison also allows exactly one struct return shape: a mirror function
whose HLSL return type is a plain struct of `float`/`floatN`/`int`/`intN`
fields is compared to its C# counterpart field by field, recursing into each
field's own components, down to bit-for-bit scalars, matching each leaf's
declaration path (field names, not just position) and declared type as well
as its value. `CultCellular` (`math.cs`, `cellular`) and `CultPhasor`
(`math.Phacelle.cs`, `phacelle`) are its consumers. A struct return with a mismatched, reordered-by-name, or
as its value. `CultCellular` (`math.cs`, `cellular`), `CultPhasor`
(`math.Phacelle.cs`, `phacelle`), `SpanNoise` (`math.Span.cs`, `span_fbm`) and `CultSiteCells`
(`math.Sites.cs`, `site_cells_insert`) are its consumers. `CultSiteCells` is also a struct parameter (`site_cells_insert`, `site_cells_border`). A struct return with a mismatched, reordered-by-name, or
retyped field does not get a second comparison path or a shape of its own;
it fails the same walk that already handles a bare vector return.
@ -146,6 +158,10 @@ hand. Regenerate them, and the golden fixture
$env:CULTMATH_WRITE_GLSL = "1"; dotnet test packages/cultmath/tests/CultMath.Tests --filter GlslMirrorTests
```
Run it on Yggdrasil's `mcr.microsoft.com/dotnet/sdk:10.0` image (Ubuntu 24.04) only: that image owns the fixture's platform
families and `device-bounds-lavapipe.txt`, and `CULTMATH_WRITE_GLSL=1` fails on any other OS. Re-record the bounds there in
the same revision (`WRITE_BOUNDS=1`, `tools/device/run-lavapipe.sh`).
`GlslMirrorTests.CommittedGlslEqualsLowering` fails on any difference between
either committed file and the lowering of the committed HLSL. These
transformations, in order, are the complete list:
@ -204,6 +220,12 @@ test, bit for bit; GLSL against C# by the golden fixture, which records 256
seeded cases per function family as float32 bit patterns, evaluated by the
consumer on WebGL2 and read back. The families `pcg3d` and `pcg4d` are marked
exact; the float families are ulp-bounded, the bound measured on the device.
A conforming leg runs every fixture family through both routes on a device: `tools/device/run-lavapipe.sh`
compiles the HLSL with dxc and the committed GLSL with glslang, dispatches each family on Mesa lavapipe, and
`GlslMirrorTests.DeviceResultsHoldFixture` judges the results. Exact and integer families must match bit for bit,
families with a check pass their `Verify`, the sampled-form families are checked on the device's own noise values,
and every other family is held to the worst error recorded for that device in
`tests/CultMath.Tests/fixtures/device-bounds-lavapipe.txt`.
`iv_frustum_ball` is ulp-bounded, not exact: it calls `sqrt`, which GLSL ES
3.00 does not require to be correctly rounded, and WebGL2 compilers may
@ -234,6 +256,13 @@ the main families' reach is at least about 1e-3, where the second-order remainde
allowance still passes. `GoldenFixtureMatchesCSharp` requires each of the two to hold cases that fail on C#'s own results once
the allowance is removed.
Four more appended families, the coordinate-band `snoise_grad` and `snoise_hessian` reads (256 cases each, the magnitude of the
point log-uniform from 1e-4 to 1e2 and from 1e2 to 4e6), carry an allowance `check`: with `S = |x|_1` and the allowances
`math.span_slack(math.span_rounding(1, x, zero stage, 0, 0, 0))` (the owners of the span stage's allowances in C#, HLSL and GLSL),
the result must lie within the allowance of the kernel's exact value, gradient or Hessian in double, no tolerance. The two small
families must also fail it in at least 16 cases once the absolute term is removed. `GoldenFixtureMatchesCSharp` applies the check
to C#'s own results, and a device capture is judged by the same `Verify`.
`GoldenFixtureMatchesCSharp` evaluates C# on the committed fixture's own
arguments. Arguments are drawn only when the fixture is regenerated
(`CULTMATH_WRITE_GLSL=1`), because the draw itself calls `MathF.Pow`, whose
@ -243,17 +272,19 @@ name the platform that generated them and the test compares them case by case
against the running platform, printing the largest distance. `iv_exp`
("ulp-bounded, platform exp") must agree within one ulp. `phacelle`
("ulp-bounded, platform exp, sin, cos; across platforms |diff| <= 2^-20
max(|v|, 1)") must agree within `2^-20 max(|v|, 1)` per component, `v` being
max(|v|, 1); on a device |diff| <= 2^-18 max(|v|, 1)") must agree across
platforms within `2^-20 max(|v|, 1)` per component, `v` being
the fixture's value: its sums of `exp`, `sin` and `cos` cancel near zero, where
ulps mean nothing, and Windows differs from the Linux fixture by up to
`3.6e-7 max(|v|, 1)`. Every other family is compared as text. `phacelle` is in
`3.6e-7 max(|v|, 1)`. The device clause is `2^-18` because the devices measured
are farther: lavapipe is `2.2e-6 max(|v|, 1)` and the GTX 1070 `3.1e-6`; the conforming
run judges it. Every other family is compared as text. `phacelle` is in
`CultMath.Phacelle.glsl`, so the evaluator concatenates
`CultMath.Phacelle.glsl` after `CultMath.glsl` for that family, with
normalization 0.5. The site does not vendor the MPL file and skips the family.
Its evaluator is the cultmath-tapes follow-up `glsl-browser-parity-run`, owned
by the next CultMath parity cut: a repeatable CultLib WebGL2 run of every
fixture family, `phacelle` included. Until it lands, the one GLSL evaluation of
`phacelle` is a single probe (GTX 1070, ANGLE D3D11: within `3.1e-6 max(|v|, 1)`
Its evaluator is the lavapipe GLSL route of the conforming run above, which covers
every fixture family, `phacelle` included. The real-device leg (ANGLE D3D11 on the GTX 1070)
is the later cut `angle-capture`; the one GLSL evaluation of `phacelle` on that device is a single probe (GTX 1070, ANGLE D3D11: within `3.1e-6 max(|v|, 1)`
of C#). Asura's `gpu-snoise-parity` is a different check, an FXC and Unity
readback of `snoise`, and evaluates no GLSL.
@ -398,7 +429,7 @@ together, laid out as `float4(gradient.xyz, value.w)`, with no value-only twin
- `phacelle(float3 p, float3 side, float offset, float normalization)` is Rune
Skovbo Johansen's Phacelle noise ("phase + cell": a stripe pattern built by
blending one cosine/sine wave per jittered cell), generalised to 3D cells. It
returns `CultPhasor { cos, sin }`, each `(∇, value)`. `side` is the stripe
returns `CultPhasor { cosine, sine }`, each `(∇, value)`. `side` is the stripe
wave vector, whose length is 2π times the stripe frequency; in 3D the
perpendicular to the flow is not unique, so the caller chooses it (on a unit
sphere, `cross(p̂, flow)`). `offset` is the phase in cycles. It visits 4×4×4
@ -472,6 +503,83 @@ Known C# friction: under `using static CultMath.math;` the constructor
functions (`float3(...)`, `float3x3(...)`) hide the type names in member access,
so static members need a qualified type (`CultMath.float3x3.Euler(...)`).
## Sites
A site is `float4(position.xyz, weight)`; a 2D consumer uses z = 0. `math.Sites.cs` scores a query point
against a site two ways and keeps the two best:
- `power_score(p, site)`: value `|p - s|^2 - w^2`, gradient `2 (p - s)`. A power cell is the intersection of the
half-spaces bounded by the radical planes against every other site, so `power_border` is exact.
- `multiplicative_score(p, site)`: value `|p - s| / w`, gradient `(p - s) / (|p - s| w)`, zero at `p = s`
(cellular's degenerate convention). Its borders are Apollonius circles, and the border distance is first order.
A span function is a C# loop over a per-site step that has a shader twin, because the HLSL mirror cannot take a
span or a buffer. `site_cells_empty()` starts the fold, `site_cells_insert(cells, score, index)` folds one score
in (strict `<` against F1, the old F1 moving to second, else strict `<` against F2, so an exact tie keeps the
earlier index first), and `site_cells_border(cells)` reads `float4(unit normal, distance)`, with `n = grad F2 -
grad F1` pointing toward the nearest site and `(0, 0, 0, 1.0e30)` when there is no second site. That border is
the distance to the bisector of the two lowest scores, first order and for either metric; it is not the distance
to the nearest site's cell boundary (the F2 - F1 edge error: the second-lowest score is not the nearest plane in
general). The exact power cell border is a second pass: `power_border_step(p, owner, other, border)` takes the
running `float4(unit normal toward the owner, distance)` and the radical plane of `owner` against `other`, with
the distance built from the offset `owner - p` and the weight difference `(w1 - w2)(w1 + w2)` so it does not
cancel for close sites seen from far; the cell boundary distance is the least over every site except the owner
(the fold's nearest), and a site coincident with the owner has no plane. The step trusts the caller's owner: a
query outside the owner's cell reports distance 0 with that plane's normal, not an error. `power_border(p, sites)` is that loop;
the shader's `cultmath_power_border(p, sites, count)` is the same loop over a by-value array (see "Sites in a shader" below).
Cell spans refuse a non-finite query, site position or weight, and more than 2^24 sites (indices are floats).
`power_cells` and
`multiplicative_cells` are that loop over a `ReadOnlySpan<float4>` and nothing else; a shader caller writes the
same loop over its own array. Every site of an explicit span is visited, so no window and no weight-ratio range
apply. `CultSiteCells` holds the two score fields and the two site indices (floats, -1 for none).
`gabriel_margin(a, b, other)` is `(|other - (a + b) / 2| - r) / r` with `r = |a - b| / 2`: zero on the diameter
circle, negative inside it, -1 when `a` and `b` coincide. `soft_gabriel_gate(sites, first, second, softness)`
is `smoothstep(-softness, softness, least margin over every other site)` on positions alone, 1 when no other site
exists; it is C# only. `distance_to_segment(point, start, end)` has out-free 2D and 3D forms beside the `out`
form, all over one public `segment_closest` per dimension, each with its 2D and 3D shader overload.
## Weighted lattice cells
`weighted_lattice_cells(p, time, bodies, cellSize, drift, boost, falloff, smoothing, stretchNormal, stretch)` in
`math.Lattice.cs` returns `float4(F1, F2, smooth, id)` over a jittered, drifting lattice, in cell units times weight.
Four public steps compose it, and a shader or a tape composes the same four:
- `lattice_site(cell, time, drift)`: the site `cell + 0.5 + 0.5 sin(drift time + 2 pi u)` per axis (so it stays within
one cell of the corner) and the base weight `0.5 + 0.5 u`, in [0.5, 1), from its own hash lane.
- `lattice_weight(base, border, boost, falloff)`: `base (1 + boost exp(-(border / falloff)^2))`. `border` is
`power_border` at the cell centre over the body span (exact, so the weight is continuous as bodies move; the first
order `site_cells_border` jumps where the second-nearest body changes and is not used).
- `lattice_distance(q, site, weight, normal, stretch)`: `weight |(stretch a, c)|`, `a` and `c` the offset along and
across the unit normal; isotropic at stretch 1.
- `lattice_smooth_step(m, d, f1, smoothing)`: the smooth minimum fold, started at `F1 + smoothing` and clamped to it,
so a cell with `d >= F1 + 2 smoothing` changes nothing and the result does not depend on the box visited.
The search visits Chebyshev rings around the query's cell and stops before ring `k` once `0.5 (k - 1) >= max(F2,
F1 + 2 smoothing)`: every cell on ring `k` has `d >= 0.5 (k - 1)` (site within one cell of its corner, weight at
least 0.5, stretch at least 1). A second pass folds the visited box in absolute row-major order, so ties and the
smooth minimum do not depend on how far the search went. The proof is the comment above `weighted_lattice_cells`.
The accepted range is boost 0 to `LatticeMaxBoost` (4, a weight ratio of 10:1), stretch 1 to `LatticeMaxStretch` (2)
and smoothing up to `LatticeMaxSmoothing` (1); outside it the function throws `ArgumentException`. `LatticeMaxRing`
(37) is the proven greatest ring at the range edge. Measured over 24,000 points on orbiting bodies (six bodies,
sigma ratio 6:1, the ruled look and the range edge), the greatest ring visited has median 3, 99th percentile 7 and
maximum 8 (49, 225 and 289 cells); `RingsStayWellInsideTheCap` asserts these.
### Sites in a shader
A shader function cannot take a span, so `cultmath_power_border(p, sites, count)`, `cultmath_lattice_cell_distance(...)`
and `cultmath_weighted_lattice_cells(p, time, bodies, count, ...)` take `float4 sites[CULTMATH_MAX_SITES]` by value and the
count in use, with `count` in `[0, CULTMATH_MAX_SITES]`. `CULTMATH_MAX_SITES` is 128 unless the consumer defines it before
including the library (`#ifndef`): 128 covers every Aetheria zone within GLSL ES 3.00's 224 guaranteed fragment uniform
vectors, and loops run to `count`, so the capacity costs uniform space, not iterations. A caller whose sites live in a
buffer copies them into a local array first; the library takes no resource, accessor macro or uniform of its own. The preconditions the C#
functions throw for are the caller's in a shader (finite arguments, `cellSize > 0`, the stated ranges, `p` within 2^23
cells). The ring pass is bounded `k <= 37` and exits as C# does; the box pass recomputes each distance instead of caching
75 x 75 of them, which gives the same values. The C# mirror compiles the array as `T[]`, and
`HlslSourceCompatibilityTests.SiteArrayMirrorsMatchCSharp` compares the three functions through the span twins in
`SiteArrays` (reflection cannot pass a `ReadOnlySpan`); `CULTMATH_MAX_SITES` is a shader-only capacity with no C# twin.
## Intervals
`math.Interval.cs` and `shaders/CultMath.Interval.hlsl` (included from
@ -777,11 +885,18 @@ differentiating each corner's gradient `-8 m^3 s x + m^4 p` once more. Each
off-diagonal entry is computed once and placed twice, so the matrix equals its
transpose bit for bit, and a clamped corner (`m = 0`) contributes nothing. Built
from nine scalars, the matrix reads as columns in GLSL (`mat3`) and rows in HLSL,
which is the same matrix because it is symmetric. `SNOISE_THIRD = 536.9152`
which is the same matrix because it is symmetric. `SNOISE_THIRD = 535.8009`
(`math.Span.cs`) is the next order: 1.10 times the maximum, over 1e6 seeded points,
of the operator norm of the third-derivative tensor of `snoise` (central
differences of `snoise_hessian` at h = 1e-3, then an ascent over unit vectors),
refined from the 1e4 largest; measured 488.1047 on the tied kernel.
of the operator norm of the third-derivative tensor of `snoise`, refined from the
1e4 largest; measured 487.0917 on the tied kernel. The tensor is closed form, in
double: `snoise_hessian`'s per-corner entry differentiated once more, on the
kernel's lattice, tie order and hash, and the test
`ThirdTensorMatchesDifferencesOfTheHessian` holds it against Richardson-extrapolated
differences of the double Hessian. The operator norm is a local ascent over unit
vectors (a dense sweep of 20,000 directions at the witness finds 99.9987% of it) and
the outer maximum a sampled search refined by ascent, so the constant is empirical
with a 10% margin, not a proof; an earlier pin from central differences of the
float Hessian read 488.1047, its roundoff.
`MeasureThird` takes the measurement and `ThirdConstantPinsSampled` pins it.
`integral_clamp_linear(a, b, lo, hi, t0, t1)` is the closed-form integral over
`[t0, t1]` of `clamp(a + b t, lo, hi)`. All three have HLSL and GLSL lowerings and
@ -799,6 +914,76 @@ constant of 40 misses there). The margin over the function itself: a scan of the
independently over every hash configuration gives 52.22, so 55.64 sits 7% above the
configuration bound. A change to the kernels must re-run the measurements.
### Span stages
`span_fbm` and `span_ridged` (`math.Span.cs`; `cultmath_span_fbm` and `cultmath_span_ridged` in HLSL over
the same struct types, `cultmath_span_stage` behind both, and GLSL derived) are the octave stages
of a warp chain, `span_fbm(path, stage, octaves, lacunarity, gain)`, each returning
one `SpanNoise { float4 field; float curvature; float fold; }`. The inputs are two
structs with named public fields and no positional constructor, so the three floats
of the span and the six fields of the transform cannot be swapped silently.
`SpanPath { midpoint, velocity, speed, acceleration, halfLength }` is the path
`q(t)`, `|t| <= halfLength`, through `midpoint` with `velocity` at `t = 0`,
`|q'| <= speed` and `|q''| <= acceleration` along it. `SpanStage { frequency,
stretch, axis, shift, offset, octaveStep }` makes octave `o` sample `snoise_grad`
at `x_o(q) = f_o (M q + shift) + offset + o octaveStep`, with `f_o =
lacunarity^o`, amplitude `gain^o`, and `M = frequency (I + (1/stretch - 1) a a^T)`,
`a = normalize(axis)`; a zero axis or `stretch` 1 means no stretch, and `stretch`
must be positive and finite (unguarded, like `lacunarity`). `default(SpanStage)`
has frequency 0; `SpanStage.Identity` is the stage whose field equals `fbm_grad`
and `ridged_grad` bit for bit. `octaves`, `lacunarity` and `gain` stay positional:
they are `fbm_grad`'s own trailing triple, and `octaves` is the one int among the
scalars. `field` is the octave sum and its gradient in `q` (`f_o M grad n_o`).
`span_ridged` sums `gain^o (1 - |n_o|)`, `ridged_grad`'s convention. The Span
Lab's stage is `1 - 2|n|` per octave: with `G` the sum of `gain^o` over the
octaves (signed, so the alternating sum for a negative gain), its value is `2 value
- G`, and its gradient, curvature and fold are twice `span_ridged`'s.
The shaders have no `SpanStage.Identity`: a shader caller builds every stage field itself (in GLSL,
`SpanStage(1.0, 1.0, vec3(0.0), vec3(0.0), vec3(0.0), vec3(0.0))` is the identity). A constant would
need a second transformation rule (C# has no const structs, GLSL wants a constructor) and a mirror
exemption, for one value; the mirror test proves instead that the shader text at a C#-built identity
returns `cultmath_fbm_grad` and `cultmath_ridged_grad`'s field bit for bit.
Bounds come from the three pins only. With `|M| = |frequency| max(1, 1/stretch)` and
`nu = f_o |M| speed` the noise-space speed, the Hessian of the noise on the span has
operator norm at most `Hl = min(SNOISE_HESSIAN, ||snoise_hessian(x_mid)||_F + e_h +
SNOISE_THIRD nu halfLength)`, so `curvature = sum gain^o f_o^2 |M|^2 Hl` bounds the
operator norm of the field's Hessian in `q` at every point of the span (the pin takes
over once the third-derivative term exceeds it, and at `halfLength` 0 the local tier
alone remains). For `span_ridged`, an octave leaves the branch it is on at the
midpoint (`1 - s n`, `s` the midpoint's sign of `n`) only where `n` changes sign, and
there `|n| <= V - |n_mid|`, with `V = (|grad n_mid . w| + e_g |w|) halfLength + (Hl nu |w|
+ Gl f_o |M| acceleration) halfLength^2 / 2 + 2 e_n`, `w = f_o M velocity` and `Gl =
min(SNOISE_LIPSCHITZ, |grad n_mid| + e_g + Hl nu halfLength)`; `fold = sum 2 gain^o
max(V - |n_mid|, 0)`, exactly 0 for `span_fbm` and for an octave that cannot reach zero.
The float32 `snoise`, `snoise_grad` and `snoise_hessian` are not the exact functions, so
the quantities read at the midpoint carry allowances, by the derivation of `af_snoise`'s
comment. `S_o = |f_o| (|M q|_1 + |shift|_1) + |offset|_1 + |o octaveStep|_1 + sqrt(3) nu
halfLength` is the sum of the magnitudes the octave coordinate's roundings are relative
to, and `e_n = 2^-14 + L 2^-20 S_o` (value), `e_g = 2^-12 + H 2^-20 S_o` (gradient
length) and `e_h = 2^-10 + T 2^-20 S_o` (Frobenius norm of the Hessian), with `L`, `H`,
`T` the three pins; they add no pin. Measured against a double kernel with the float
hash (6000 points per band, `|x|` from 2^-2 to 2^23), two samples observed the error's
maxima as fractions of `e_n`, `e_g` and `e_h`: 0.057, 0.046 and 0.075 in the cut's own
sample (seed not recorded; the 0.075 is against `T 2^-20 |x|_1` alone, at the smallest
band, where the absolute Hessian error is 2.3e-5 and the absolute term carries it), and
0.064, 0.0485 and 0.057 in an audit's (seed 0x501). They are samples, not bounds: a new
seed moves them. They say the allowances enclose the error with a margin of about 15.
Errors grow linearly in `|x|`: the value's is 2e-4 at 2^8, 1.5e-2 at 2^14 and 0.94 at 2^20.
Rounding in the Frobenius-to-operator-norm step needs no allowance: the Frobenius norm
exceeds the operator norm.
Domain. No coordinate range is checked, clamped or refused. The allowances grow with
`S_o`, so `curvature` saturates at `SNOISE_HESSIAN` (true everywhere) by `S_o` of about
1.1e5, and `fold` turns positive for every octave whose `V + 2 e_n` reaches `|n_mid|`:
the bounds stay sound at every magnitude and stop being informative where float32 noise
is (see "Precision domain" above). `SpanStageTests` encloses the exact (double) Hessian
norm and both the float and the exact branch departure at 64 points of seeded spans in
bands of `S_o` from under 1e2 to over 1e6, and in a fixed span where the local tier
with its third-derivative term halved fails.
The slice. `af_frustum_ball(centreSlope, z0, z1, footprintPerDepth,
warpVariation)` returns the centre `(m_c z_m, z_m)` and a radius `z1
footprintPerDepth + warpVariation`, with `af_frustum_axis` the axis `(m_c, 1) h`:

10
packages/cultmath/shaders/CultMath.Affine.hlsl
View file

@ -12,9 +12,15 @@
// math.Affine.cs (SNOISE_HESSIAN).
static const float CULTMATH_SNOISE_HESSIAN = 55.642143;
// The absolute rounding of one float32 read of cultmath_snoise_grad, value and gradient; the owner is
// math.Affine.cs (SNOISE_READ_VALUE, SNOISE_READ_GRADIENT), read here by cultmath_af_snoise and in
// CultMath.hlsl by cultmath_span_slack.
static const float CULTMATH_SNOISE_READ_VALUE = 6.103515625e-5;
static const float CULTMATH_SNOISE_READ_GRADIENT = 2.44140625e-4;
// An upper bound on the operator norm of the third-derivative tensor of cultmath_snoise(float3), the
// next order after CULTMATH_SNOISE_HESSIAN; provenance in math.Span.cs (SNOISE_THIRD).
static const float CULTMATH_SNOISE_THIRD = 536.9152;
static const float CULTMATH_SNOISE_THIRD = 535.8009;
float3 cultmath_af_point(float value) { return float3(value, 0.0, 0.0); }
float3 cultmath_af_symbol(float x0, float a) { return float3(x0, a, 0.0); }
@ -87,7 +93,7 @@ float3 cultmath_af_snoise(float3 centre, float3 axis, float radius)
float lipschitz = CULTMATH_SNOISE_LIPSCHITZ * reach;
float a = dot(gradient, axis);
float remainder = length(gradient) * radius + 0.5 * CULTMATH_SNOISE_HESSIAN * reach * reach;
float allowance = 6.103515625e-5 + 2.44140625e-4 * reach
float allowance = CULTMATH_SNOISE_READ_VALUE + CULTMATH_SNOISE_READ_GRADIENT * reach
+ CULTMATH_SNOISE_LIPSCHITZ * 9.5367431640625e-7 * (abs(centre.x) + abs(centre.y) + abs(centre.z) + reach);
bool centred = abs(a) + remainder < lipschitz;
float slope = centred ? a : 0.0;

12
packages/cultmath/shaders/CultMath.Phacelle.glsl
View file

@ -14,8 +14,8 @@
// against CultMath.CultPhasor, same field order.
struct CultPhasor
{
vec4 cos;
vec4 sin;
vec4 cosine;
vec4 sine;
};
// Johansen's Phacelle noise on 3D cells. See math.phacelle's comment (math.Phacelle.cs) for the stripe wave
@ -84,13 +84,13 @@ CultPhasor cultmath_phacelle(vec3 p, vec3 side, float offset, float normalizatio
float outCos = rawCos / rawLength;
float outSin = rawSin / rawLength;
vec3 gradLength = (rawCos * gradRawCos + rawSin * gradRawSin) / rawLength;
result.cos = vec4((gradRawCos - outCos * gradLength) / rawLength, outCos);
result.sin = vec4((gradRawSin - outSin * gradLength) / rawLength, outSin);
result.cosine = vec4((gradRawCos - outCos * gradLength) / rawLength, outCos);
result.sine = vec4((gradRawSin - outSin * gradLength) / rawLength, outSin);
}
else
{
result.cos = vec4(gradRawCos / floorLength, rawCos / floorLength);
result.sin = vec4(gradRawSin / floorLength, rawSin / floorLength);
result.cosine = vec4(gradRawCos / floorLength, rawCos / floorLength);
result.sine = vec4(gradRawSin / floorLength, rawSin / floorLength);
}
return result;

12
packages/cultmath/shaders/CultMath.Phacelle.hlsl
View file

@ -14,8 +14,8 @@
// against CultMath.CultPhasor, same field order.
struct CultPhasor
{
float4 cos;
float4 sin;
float4 cosine;
float4 sine;
};
// Johansen's Phacelle noise on 3D cells. See math.phacelle's comment (math.Phacelle.cs) for the stripe wave
@ -84,13 +84,13 @@ CultPhasor cultmath_phacelle(float3 p, float3 side, float offset, float normaliz
float outCos = rawCos / rawLength;
float outSin = rawSin / rawLength;
float3 gradLength = (rawCos * gradRawCos + rawSin * gradRawSin) / rawLength;
result.cos = float4((gradRawCos - outCos * gradLength) / rawLength, outCos);
result.sin = float4((gradRawSin - outSin * gradLength) / rawLength, outSin);
result.cosine = float4((gradRawCos - outCos * gradLength) / rawLength, outCos);
result.sine = float4((gradRawSin - outSin * gradLength) / rawLength, outSin);
}
else
{
result.cos = float4(gradRawCos / floorLength, rawCos / floorLength);
result.sin = float4(gradRawSin / floorLength, rawSin / floorLength);
result.cosine = float4(gradRawCos / floorLength, rawCos / floorLength);
result.sine = float4(gradRawSin / floorLength, rawSin / floorLength);
}
return result;

382
packages/cultmath/shaders/CultMath.glsl
View file

@ -302,10 +302,10 @@ mat3 cultmath_snoise_hessian(vec3 value)
// The integral over [t0, t1] of clamp(a + b t, lo, hi); see the C# math.integral_clamp_linear comment.
float cultmath_integral_clamp_linear(float a, float b, float lo, float hi, float t0, float t1)
{
if (b == 0.0)
return clamp(a, lo, hi) * (t1 - t0);
lo = min(lo, hi);
if (b == 0.0)
return t1 != t0 ? clamp(a, lo, hi) * (t1 - t0) : 0.0;
float direction = t1 < t0 ? -1.0 : 1.0;
float start = min(t0, t1);
float stop = max(t0, t1);
@ -315,7 +315,9 @@ float cultmath_integral_clamp_linear(float a, float b, float lo, float hi, float
float leave = clamp(max(atLo, atHi), start, stop);
float before = b > 0.0 ? lo : hi;
float after = b > 0.0 ? hi : lo;
float total = before * (enter - start) + (a + b * (0.5 * (enter + leave))) * (leave - enter) + after * (stop - leave);
float held = enter > start ? before * (enter - start) : 0.0;
float kept = stop > leave ? after * (stop - leave) : 0.0;
float total = held + (a + b * (0.5 * (enter + leave))) * (leave - enter) + kept;
return direction * total;
}
@ -598,6 +600,241 @@ CultCellular cultmath_cellular(vec3 p)
return result;
}
// Sites: a site is vec4(position.xyz, weight). A shader function cannot take a span (design.md, "Sites"), so a span is a
// fixed-capacity array and a count (cultmath_power_border, below), or a loop over a per-site step:
// CultSiteCells cells = cultmath_site_cells_empty();
// for (int i = 0; i < count; i++)
// cells = cultmath_site_cells_insert(cells, cultmath_power_score(p, sites[i]), i);
// The exact power cell border is cultmath_power_border, the second loop over the sites that skips the owner cells.nearestSite.
// The step trusts the owner: a p outside the owner's cell reports distance 0 with that plane's normal, not an error.
// Statement for statement with math.Sites.cs (power_score, multiplicative_score, site_cells_*, gabriel_margin).
struct CultSiteCells
{
vec4 nearest;
vec4 second;
float nearestSite;
float secondSite;
};
vec4 cultmath_power_score(vec3 p, vec4 site)
{
vec3 offset = p - site.xyz;
return vec4(offset * 2.0, dot(offset, offset) - site.w * site.w);
}
vec4 cultmath_multiplicative_score(vec3 p, vec4 site)
{
vec3 offset = p - site.xyz;
float r = length(offset);
vec3 gradient = r > 0.0 ? offset / (r * site.w) : vec3(0.0, 0.0, 0.0);
return vec4(gradient, r / site.w);
}
CultSiteCells cultmath_site_cells_empty()
{
CultSiteCells result;
result.nearest = vec4(0.0, 0.0, 0.0, 1.0e30);
result.second = vec4(0.0, 0.0, 0.0, 1.0e30);
result.nearestSite = -1.0;
result.secondSite = -1.0;
return result;
}
CultSiteCells cultmath_site_cells_insert(CultSiteCells cells, vec4 score, int index)
{
CultSiteCells result = cells;
if (score.w < cells.nearest.w)
{
result.second = cells.nearest;
result.secondSite = cells.nearestSite;
result.nearest = score;
result.nearestSite = float(index);
}
else if (score.w < cells.second.w)
{
result.second = score;
result.secondSite = float(index);
}
return result;
}
vec4 cultmath_site_cells_border(CultSiteCells cells)
{
vec3 n = cells.second.xyz - cells.nearest.xyz;
float len = length(n);
if (cells.secondSite < 0.0 || len <= 0.0)
return vec4(0.0, 0.0, 0.0, 1.0e30);
return vec4(n / len, (cells.second.w - cells.nearest.w) / len);
}
vec4 cultmath_power_border_step(vec3 p, vec4 owner, vec4 other, vec4 border)
{
vec3 d = other.xyz - owner.xyz;
float len = length(d);
if (len <= 0.0)
return border;
float plane = max((dot(owner.xyz - p, d) + 0.5 * (dot(d, d) + (owner.w - other.w) * (owner.w + other.w))) / len, 0.0);
return plane < border.w ? vec4(-d / len, plane) : border;
}
float cultmath_gabriel_margin(vec3 a, vec3 b, vec3 other)
{
float r = cultmath_distance(a, b) * 0.5;
if (r <= 0.0)
return -1.0;
return (cultmath_distance(other, (a + b) * 0.5) - r) / r;
}
// Weighted lattice cells: the four per-cell steps of math.Lattice.cs, statement for statement; cultmath_weighted_lattice_cells,
// after them, is the whole function.
vec3 cultmath_lattice_site(vec2 cell, float time, float drift)
{
ivec3 h = cultmath_pcg3d(ivec3(int(cell.x), int(cell.y), 0));
vec2 phase = vec2(float(uint(h.x) >> 8) * (1.0 / 16777216.0), float(uint(h.y) >> 8) * (1.0 / 16777216.0));
vec2 site = cell + 0.5 + 0.5 * vec2(sin(drift * time + CULTMATH_TAU * phase.x), sin(drift * time + CULTMATH_TAU * phase.y));
return vec3(site, 0.5 + 0.5 * (float(uint(h.z) >> 8) * (1.0 / 16777216.0)));
}
float cultmath_lattice_weight(float base_weight, float border, float boost, float falloff)
{
float t = border / falloff;
return base_weight * (1.0 + boost * exp(-(t * t)));
}
float cultmath_lattice_distance(vec2 q, vec2 site, float weight, vec2 normal, float stretch)
{
vec2 v = site - q;
float a = dot(v, normal);
float c = v.x * normal.y - v.y * normal.x;
return weight * length(vec2(stretch * a, c));
}
float cultmath_lattice_smooth_step(float m, float d, float f1, float smoothing)
{
if (d >= f1 + 2.0 * smoothing)
return m;
float h = cultmath_smoothstep(0.0, 1.0, 0.5 + 0.5 * (m - d) / smoothing);
float next = m + (d - m) * h - h * (1.0 - h) * smoothing / (1.0 + 3.0 * smoothing);
return min(next, f1 + smoothing);
}
// Sites by value: a shader function cannot take a span, so these take vec4 sites[CULTMATH_MAX_SITES] and the count in use, with
// count in [0, CULTMATH_MAX_SITES]. A consumer defines CULTMATH_MAX_SITES before including this file to change the capacity; loops
// run to count, so the capacity costs uniform space, not iterations. A caller whose sites live in a buffer copies them into a local array first.
// The preconditions the C# functions throw for are the caller's in a shader: finite arguments, cellSize > 0, boost in [0, 4],
// falloff > 0, smoothing in (0, 1], stretch in [1, 2], p within 2^23 cells. A zero stretchNormal means (1, 0) and needs stretch == 1.
// cultmath_weighted_lattice_cells is math.weighted_lattice_cells: the ring pass is bounded by k <= 37 and exits as C# does, and the
// box pass recomputes each distance instead of caching it (a 75 x 75 private array is not a shader option), with the same values.
#ifndef CULTMATH_MAX_SITES
#define CULTMATH_MAX_SITES 128
#endif
vec4 cultmath_power_border(vec3 p, vec4 sites[CULTMATH_MAX_SITES], int count)
{
CultSiteCells cells = cultmath_site_cells_empty();
for (int i = 0; i < count; i++)
cells = cultmath_site_cells_insert(cells, cultmath_power_score(p, sites[i]), i);
vec4 border = vec4(0.0, 0.0, 0.0, 1.0e30);
int owner = int(cells.nearestSite);
if (owner < 0)
return border;
for (int k = 0; k < count; k++)
if (k != owner)
border = cultmath_power_border_step(p, sites[owner], sites[k], border);
return border;
}
float cultmath_lattice_cell_distance(vec2 q, vec2 cell, float time, float drift, vec4 bodies[CULTMATH_MAX_SITES], int count,
float cellSize, float boost, float falloff, vec2 normal, float stretch)
{
vec3 site = cultmath_lattice_site(cell, time, drift);
float border = cultmath_power_border(vec3((cell + 0.5) * cellSize, 0.0), bodies, count).w;
return cultmath_lattice_distance(q, site.xy, cultmath_lattice_weight(site.z, border, boost, falloff), normal, stretch);
}
vec4 cultmath_weighted_lattice_cells(vec2 p, float time, vec4 bodies[CULTMATH_MAX_SITES], int count, float cellSize, float drift,
float boost, float falloff, float smoothing, vec2 stretchNormal, float stretch)
{
float scale = max(abs(stretchNormal.x), abs(stretchNormal.y));
vec2 normal = vec2(1.0, 0.0);
if (scale > 0.0)
{
normal = stretchNormal / scale;
normal = normal / length(normal);
}
vec2 q = p / cellSize;
vec2 origin = floor(q);
int cx0 = int(origin.x);
int cz0 = int(origin.y);
float f1 = 1.0e30;
float f2 = 1.0e30;
int reach = 0;
for (int k = 0; k <= 37; k++)
{
if (k > 1 && 0.5 * float(k - 1) >= max(f2, f1 + 2.0 * smoothing))
break;
for (int dz = -k; dz <= k; dz++)
{
int step = dz == -k || dz == k || k == 0 ? 1 : 2 * k;
for (int dx = -k; dx <= k; dx += step)
{
float d = cultmath_lattice_cell_distance(q, vec2(cx0 + dx, cz0 + dz), time, drift, bodies, count, cellSize, boost, falloff, normal, stretch);
if (d < f1) { f2 = f1; f1 = d; }
else if (d < f2) f2 = d;
}
}
reach = k;
}
float m = f1 + smoothing;
float best = 1.0e30;
float second = 1.0e30;
int winDx = 0;
int winDz = 0;
for (int dz = -reach; dz <= reach; dz++)
for (int dx = -reach; dx <= reach; dx++)
{
float d = cultmath_lattice_cell_distance(q, vec2(cx0 + dx, cz0 + dz), time, drift, bodies, count, cellSize, boost, falloff, normal, stretch);
if (d < best) { second = best; best = d; winDx = dx; winDz = dz; }
else if (d < second) second = d;
m = cultmath_lattice_smooth_step(m, d, f1, smoothing);
}
ivec4 id = cultmath_pcg4d(ivec4(cx0 + winDx, cz0 + winDz, 0, 0));
return vec4(best, second, m, float(uint(id.w) >> 8) * (1.0 / 16777216.0));
}
vec2 cultmath_segment_closest(vec2 point_value, vec2 start, vec2 end)
{
vec2 segment = end - start;
float segmentLengthSq = cultmath_lengthsq(segment);
if (segmentLengthSq <= 0.0)
return start;
float t = cultmath_saturate(dot(point_value - start, segment) / segmentLengthSq);
return start + segment * t;
}
vec3 cultmath_segment_closest(vec3 point_value, vec3 start, vec3 end)
{
vec3 segment = end - start;
float segmentLengthSq = cultmath_lengthsq(segment);
if (segmentLengthSq <= 0.0)
return start;
float t = cultmath_saturate(dot(point_value - start, segment) / segmentLengthSq);
return start + segment * t;
}
float cultmath_distance_to_segment(vec2 point_value, vec2 start, vec2 end)
{
return cultmath_distance(point_value, cultmath_segment_closest(point_value, start, end));
}
float cultmath_distance_to_segment(vec3 point_value, vec3 start, vec3 end)
{
return cultmath_distance(point_value, cultmath_segment_closest(point_value, start, end));
}
@ -746,9 +983,15 @@ vec4 cultmath_iv_frustum_ball(vec2 centreSlope, float z0, float z1, float footpr
// math.Affine.cs (SNOISE_HESSIAN).
const float CULTMATH_SNOISE_HESSIAN = 55.642143;
// The absolute rounding of one float32 read of cultmath_snoise_grad, value and gradient; the owner is
// math.Affine.cs (SNOISE_READ_VALUE, SNOISE_READ_GRADIENT), read here by cultmath_af_snoise and in
// CultMath.hlsl by cultmath_span_slack.
const float CULTMATH_SNOISE_READ_VALUE = 6.103515625e-5;
const float CULTMATH_SNOISE_READ_GRADIENT = 2.44140625e-4;
// An upper bound on the operator norm of the third-derivative tensor of cultmath_snoise(vec3), the
// next order after CULTMATH_SNOISE_HESSIAN; provenance in math.Span.cs (SNOISE_THIRD).
const float CULTMATH_SNOISE_THIRD = 536.9152;
const float CULTMATH_SNOISE_THIRD = 535.8009;
vec3 cultmath_af_point(float value) { return vec3(value, 0.0, 0.0); }
vec3 cultmath_af_symbol(float x0, float a) { return vec3(x0, a, 0.0); }
@ -821,7 +1064,7 @@ vec3 cultmath_af_snoise(vec3 centre, vec3 axis, float radius)
float lipschitz = CULTMATH_SNOISE_LIPSCHITZ * reach;
float a = dot(gradient, axis);
float remainder = length(gradient) * radius + 0.5 * CULTMATH_SNOISE_HESSIAN * reach * reach;
float allowance = 6.103515625e-5 + 2.44140625e-4 * reach
float allowance = CULTMATH_SNOISE_READ_VALUE + CULTMATH_SNOISE_READ_GRADIENT * reach
+ CULTMATH_SNOISE_LIPSCHITZ * 9.5367431640625e-7 * (abs(centre.x) + abs(centre.y) + abs(centre.z) + reach);
bool centred = abs(a) + remainder < lipschitz;
float slope = centred ? a : 0.0;
@ -870,6 +1113,133 @@ vec4 cultmath_af_frustum_ball(vec2 centreSlope, float z0, float z1, float footpr
// Span stages (invariant 8): the SpanPath, SpanStage and SpanNoise of math.Span.cs, field for field in
// C#'s order, and cultmath_span_stage, its octave stage, statement for statement so the text is a bit-for-bit
// mirror. They follow the includes because the bounds read CULTMATH_SNOISE_LIPSCHITZ, _HESSIAN and _THIRD.
// A shader caller builds a SpanStage itself (SpanStage(1.0, 1.0, vec3(0.0), vec3(0.0), vec3(0.0), vec3(0.0))
// is the identity of math.SpanStage.Identity, which has no shader form); see math.span_stage's comments for
// the span, the transform, the bounds and the allowances. cultmath_span_rounding and cultmath_span_slack
// are math.span_rounding and math.span_slack, the owners of S_o and of the three allowances.
struct SpanPath
{
vec3 midpoint;
vec3 velocity;
float speed;
float acceleration;
float halfLength;
};
struct SpanStage
{
float frequency;
float stretch;
vec3 axis;
vec3 shift;
vec3 offset;
vec3 octaveStep;
};
struct SpanNoise
{
vec4 field;
float curvature;
float fold;
};
float cultmath_span_rounding(float scale, vec3 mq, SpanStage stage, vec3 octaveShift, float nu, float halfLength)
{
return 9.5367431640625e-7 * (abs(scale) * (cultmath_csum(abs(mq)) + cultmath_csum(abs(stage.shift))) + cultmath_csum(abs(stage.offset)) + cultmath_csum(abs(octaveShift)) + 1.7320508 * nu * halfLength);
}
vec3 cultmath_span_slack(float rounding)
{
return vec3(CULTMATH_SNOISE_READ_VALUE + CULTMATH_SNOISE_LIPSCHITZ * rounding,
CULTMATH_SNOISE_READ_GRADIENT + CULTMATH_SNOISE_HESSIAN * rounding,
9.765625e-4 + CULTMATH_SNOISE_THIRD * rounding);
}
// The octave stage behind cultmath_span_fbm and cultmath_span_ridged, as math.span_stage is behind span_fbm
// and span_ridged. The Frobenius norm of the Hessian is written out in C#'s order (GLSL has no _mRC member
// access); the C# local named step is octaveShift here (it would shadow GLSL's step()).
SpanNoise cultmath_span_stage(bool ridged, SpanPath path, SpanStage stage, int octaves, float lacunarity, float gain)
{
vec3 q = path.midpoint;
float halfLength = path.halfLength;
octaves = clamp(octaves, 0, 16);
bool stretched = dot(stage.axis, stage.axis) > 0.0 && stage.stretch != 1.0;
vec3 direction = stretched ? normalize(stage.axis) : vec3(0.0, 0.0, 0.0);
float squeeze = stretched ? 1.0 / stage.stretch - 1.0 : 0.0;
float norm = abs(stage.frequency) * (stretched ? max(1.0, 1.0 / stage.stretch) : 1.0);
vec3 mq = stretched ? stage.frequency * (q + squeeze * dot(direction, q) * direction) : stage.frequency * q;
vec3 mv = stretched ? stage.frequency * (path.velocity + squeeze * dot(direction, path.velocity) * direction) : stage.frequency * path.velocity;
float amplitude = 1.0;
float scale = 1.0;
float value = 0.0;
vec3 gradient = vec3(0.0, 0.0, 0.0);
float curvature = 0.0;
float fold = 0.0;
for (int i = 0; i < octaves; i++)
{
vec3 octaveShift = float(i) * stage.octaveStep;
vec3 x = scale * (mq + stage.shift) + (stage.offset + octaveShift);
vec4 n = cultmath_snoise_grad(x);
vec3 g = vec3(n.x, n.y, n.z);
vec3 mg = stretched ? stage.frequency * (g + squeeze * dot(direction, g) * direction) : stage.frequency * g;
if (ridged)
{
float s = sign(n.w);
value += amplitude * (1.0 - abs(n.w));
gradient += -amplitude * scale * s * mg;
}
else
{
value += amplitude * n.w;
gradient += amplitude * scale * mg;
}
float weight = abs(amplitude);
float factor = abs(scale) * norm;
float nu = factor * path.speed;
float rounding = cultmath_span_rounding(scale, mq, stage, octaveShift, nu, halfLength);
vec3 slack = cultmath_span_slack(rounding);
float valueSlack = slack.x;
float gradientSlack = slack.y;
float hessianSlack = slack.z;
mat3 hm = cultmath_snoise_hessian(x);
float frobenius = sqrt(hm[0][0] * hm[0][0] + hm[0][1] * hm[0][1] + hm[0][2] * hm[0][2] + hm[1][0] * hm[1][0] + hm[1][1] * hm[1][1] + hm[1][2] * hm[1][2]
+ hm[2][0] * hm[2][0] + hm[2][1] * hm[2][1] + hm[2][2] * hm[2][2]);
float hessianBound = min(CULTMATH_SNOISE_HESSIAN, frobenius + hessianSlack + CULTMATH_SNOISE_THIRD * nu * halfLength);
curvature += weight * factor * factor * hessianBound;
if (ridged)
{
vec3 w = scale * mv;
float reach = hessianBound * nu * halfLength;
float gl = min(CULTMATH_SNOISE_LIPSCHITZ, length(g) + gradientSlack + reach);
float travel = (abs(dot(g, w)) + gradientSlack * length(w)) * halfLength + (hessianBound * nu * length(w) + gl * factor * path.acceleration) * (0.5 * halfLength * halfLength) + 2.0 * valueSlack;
fold += 2.0 * weight * max(travel - abs(n.w), 0.0);
}
scale *= lacunarity;
amplitude *= gain;
}
SpanNoise result;
result.field = vec4(gradient, value);
result.curvature = curvature;
result.fold = fold;
return result;
}
SpanNoise cultmath_span_fbm(SpanPath path, SpanStage stage, int octaves, float lacunarity, float gain)
{
return cultmath_span_stage(false, path, stage, octaves, lacunarity, gain);
}
SpanNoise cultmath_span_ridged(SpanPath path, SpanStage stage, int octaves, float lacunarity, float gain)
{
return cultmath_span_stage(true, path, stage, octaves, lacunarity, gain);
}
float cultmath_value_noise(vec2 position)
{
vec2 cell = floor(position);

372
packages/cultmath/shaders/CultMath.hlsl
View file

@ -302,10 +302,10 @@ float3x3 cultmath_snoise_hessian(float3 value)
// The integral over [t0, t1] of clamp(a + b t, lo, hi); see the C# math.integral_clamp_linear comment.
float cultmath_integral_clamp_linear(float a, float b, float lo, float hi, float t0, float t1)
{
if (b == 0.0)
return clamp(a, lo, hi) * (t1 - t0);
lo = min(lo, hi);
if (b == 0.0)
return t1 != t0 ? clamp(a, lo, hi) * (t1 - t0) : 0.0;
float direction = t1 < t0 ? -1.0 : 1.0;
float start = min(t0, t1);
float stop = max(t0, t1);
@ -315,7 +315,9 @@ float cultmath_integral_clamp_linear(float a, float b, float lo, float hi, float
float leave = clamp(max(atLo, atHi), start, stop);
float before = b > 0.0 ? lo : hi;
float after = b > 0.0 ? hi : lo;
float total = before * (enter - start) + (a + b * (0.5 * (enter + leave))) * (leave - enter) + after * (stop - leave);
float held = enter > start ? before * (enter - start) : 0.0;
float kept = stop > leave ? after * (stop - leave) : 0.0;
float total = held + (a + b * (0.5 * (enter + leave))) * (leave - enter) + kept;
return direction * total;
}
@ -598,11 +600,373 @@ CultCellular cultmath_cellular(float3 p)
return result;
}
// Sites: a site is float4(position.xyz, weight). A shader function cannot take a span (design.md, "Sites"), so a span is a
// fixed-capacity array and a count (cultmath_power_border, below), or a loop over a per-site step:
// CultSiteCells cells = cultmath_site_cells_empty();
// for (int i = 0; i < count; i++)
// cells = cultmath_site_cells_insert(cells, cultmath_power_score(p, sites[i]), i);
// The exact power cell border is cultmath_power_border, the second loop over the sites that skips the owner cells.nearestSite.
// The step trusts the owner: a p outside the owner's cell reports distance 0 with that plane's normal, not an error.
// Statement for statement with math.Sites.cs (power_score, multiplicative_score, site_cells_*, gabriel_margin).
struct CultSiteCells
{
float4 nearest;
float4 second;
float nearestSite;
float secondSite;
};
float4 cultmath_power_score(float3 p, float4 site)
{
float3 offset = p - site.xyz;
return float4(offset * 2.0, dot(offset, offset) - site.w * site.w);
}
float4 cultmath_multiplicative_score(float3 p, float4 site)
{
float3 offset = p - site.xyz;
float r = length(offset);
float3 gradient = r > 0.0 ? offset / (r * site.w) : float3(0.0, 0.0, 0.0);
return float4(gradient, r / site.w);
}
CultSiteCells cultmath_site_cells_empty()
{
CultSiteCells result;
result.nearest = float4(0.0, 0.0, 0.0, 1.0e30);
result.second = float4(0.0, 0.0, 0.0, 1.0e30);
result.nearestSite = -1.0;
result.secondSite = -1.0;
return result;
}
CultSiteCells cultmath_site_cells_insert(CultSiteCells cells, float4 score, int index)
{
CultSiteCells result = cells;
if (score.w < cells.nearest.w)
{
result.second = cells.nearest;
result.secondSite = cells.nearestSite;
result.nearest = score;
result.nearestSite = (float)index;
}
else if (score.w < cells.second.w)
{
result.second = score;
result.secondSite = (float)index;
}
return result;
}
float4 cultmath_site_cells_border(CultSiteCells cells)
{
float3 n = cells.second.xyz - cells.nearest.xyz;
float len = length(n);
if (cells.secondSite < 0.0 || len <= 0.0)
return float4(0.0, 0.0, 0.0, 1.0e30);
return float4(n / len, (cells.second.w - cells.nearest.w) / len);
}
float4 cultmath_power_border_step(float3 p, float4 owner, float4 other, float4 border)
{
float3 d = other.xyz - owner.xyz;
float len = length(d);
if (len <= 0.0)
return border;
float plane = max((dot(owner.xyz - p, d) + 0.5 * (dot(d, d) + (owner.w - other.w) * (owner.w + other.w))) / len, 0.0);
return plane < border.w ? float4(-d / len, plane) : border;
}
float cultmath_gabriel_margin(float3 a, float3 b, float3 other)
{
float r = cultmath_distance(a, b) * 0.5;
if (r <= 0.0)
return -1.0;
return (cultmath_distance(other, (a + b) * 0.5) - r) / r;
}
// Weighted lattice cells: the four per-cell steps of math.Lattice.cs, statement for statement; cultmath_weighted_lattice_cells,
// after them, is the whole function.
float3 cultmath_lattice_site(float2 cell, float time, float drift)
{
int3 h = cultmath_pcg3d(int3((int)cell.x, (int)cell.y, 0));
float2 phase = float2((float)((uint)h.x >> 8) * (1.0 / 16777216.0), (float)((uint)h.y >> 8) * (1.0 / 16777216.0));
float2 site = cell + 0.5 + 0.5 * float2(sin(drift * time + CULTMATH_TAU * phase.x), sin(drift * time + CULTMATH_TAU * phase.y));
return float3(site, 0.5 + 0.5 * ((float)((uint)h.z >> 8) * (1.0 / 16777216.0)));
}
float cultmath_lattice_weight(float base_weight, float border, float boost, float falloff)
{
float t = border / falloff;
return base_weight * (1.0 + boost * exp(-(t * t)));
}
float cultmath_lattice_distance(float2 q, float2 site, float weight, float2 normal, float stretch)
{
float2 v = site - q;
float a = dot(v, normal);
float c = v.x * normal.y - v.y * normal.x;
return weight * length(float2(stretch * a, c));
}
float cultmath_lattice_smooth_step(float m, float d, float f1, float smoothing)
{
if (d >= f1 + 2.0 * smoothing)
return m;
float h = cultmath_smoothstep(0.0, 1.0, 0.5 + 0.5 * (m - d) / smoothing);
float next = m + (d - m) * h - h * (1.0 - h) * smoothing / (1.0 + 3.0 * smoothing);
return min(next, f1 + smoothing);
}
// Sites by value: a shader function cannot take a span, so these take float4 sites[CULTMATH_MAX_SITES] and the count in use, with
// count in [0, CULTMATH_MAX_SITES]. A consumer defines CULTMATH_MAX_SITES before including this file to change the capacity; loops
// run to count, so the capacity costs uniform space, not iterations. A caller whose sites live in a buffer copies them into a local array first.
// The preconditions the C# functions throw for are the caller's in a shader: finite arguments, cellSize > 0, boost in [0, 4],
// falloff > 0, smoothing in (0, 1], stretch in [1, 2], p within 2^23 cells. A zero stretchNormal means (1, 0) and needs stretch == 1.
// cultmath_weighted_lattice_cells is math.weighted_lattice_cells: the ring pass is bounded by k <= 37 and exits as C# does, and the
// box pass recomputes each distance instead of caching it (a 75 x 75 private array is not a shader option), with the same values.
#ifndef CULTMATH_MAX_SITES
#define CULTMATH_MAX_SITES 128
#endif
float4 cultmath_power_border(float3 p, float4 sites[CULTMATH_MAX_SITES], int count)
{
CultSiteCells cells = cultmath_site_cells_empty();
for (int i = 0; i < count; i++)
cells = cultmath_site_cells_insert(cells, cultmath_power_score(p, sites[i]), i);
float4 border = float4(0.0, 0.0, 0.0, 1.0e30);
int owner = (int)cells.nearestSite;
if (owner < 0)
return border;
for (int k = 0; k < count; k++)
if (k != owner)
border = cultmath_power_border_step(p, sites[owner], sites[k], border);
return border;
}
float cultmath_lattice_cell_distance(float2 q, float2 cell, float time, float drift, float4 bodies[CULTMATH_MAX_SITES], int count,
float cellSize, float boost, float falloff, float2 normal, float stretch)
{
float3 site = cultmath_lattice_site(cell, time, drift);
float border = cultmath_power_border(float3((cell + 0.5) * cellSize, 0.0), bodies, count).w;
return cultmath_lattice_distance(q, site.xy, cultmath_lattice_weight(site.z, border, boost, falloff), normal, stretch);
}
float4 cultmath_weighted_lattice_cells(float2 p, float time, float4 bodies[CULTMATH_MAX_SITES], int count, float cellSize, float drift,
float boost, float falloff, float smoothing, float2 stretchNormal, float stretch)
{
float scale = max(abs(stretchNormal.x), abs(stretchNormal.y));
float2 normal = float2(1.0, 0.0);
if (scale > 0.0)
{
normal = stretchNormal / scale;
normal = normal / length(normal);
}
float2 q = p / cellSize;
float2 origin = floor(q);
int cx0 = (int)origin.x;
int cz0 = (int)origin.y;
float f1 = 1.0e30;
float f2 = 1.0e30;
int reach = 0;
for (int k = 0; k <= 37; k++)
{
if (k > 1 && 0.5 * (float)(k - 1) >= max(f2, f1 + 2.0 * smoothing))
break;
for (int dz = -k; dz <= k; dz++)
{
int step = dz == -k || dz == k || k == 0 ? 1 : 2 * k;
for (int dx = -k; dx <= k; dx += step)
{
float d = cultmath_lattice_cell_distance(q, float2(cx0 + dx, cz0 + dz), time, drift, bodies, count, cellSize, boost, falloff, normal, stretch);
if (d < f1) { f2 = f1; f1 = d; }
else if (d < f2) f2 = d;
}
}
reach = k;
}
float m = f1 + smoothing;
float best = 1.0e30;
float second = 1.0e30;
int winDx = 0;
int winDz = 0;
for (int dz = -reach; dz <= reach; dz++)
for (int dx = -reach; dx <= reach; dx++)
{
float d = cultmath_lattice_cell_distance(q, float2(cx0 + dx, cz0 + dz), time, drift, bodies, count, cellSize, boost, falloff, normal, stretch);
if (d < best) { second = best; best = d; winDx = dx; winDz = dz; }
else if (d < second) second = d;
m = cultmath_lattice_smooth_step(m, d, f1, smoothing);
}
int4 id = cultmath_pcg4d(int4(cx0 + winDx, cz0 + winDz, 0, 0));
return float4(best, second, m, (float)((uint)id.w >> 8) * (1.0 / 16777216.0));
}
float2 cultmath_segment_closest(float2 point_value, float2 start, float2 end)
{
float2 segment = end - start;
float segmentLengthSq = cultmath_lengthsq(segment);
if (segmentLengthSq <= 0.0)
return start;
float t = cultmath_saturate(dot(point_value - start, segment) / segmentLengthSq);
return start + segment * t;
}
float3 cultmath_segment_closest(float3 point_value, float3 start, float3 end)
{
float3 segment = end - start;
float segmentLengthSq = cultmath_lengthsq(segment);
if (segmentLengthSq <= 0.0)
return start;
float t = cultmath_saturate(dot(point_value - start, segment) / segmentLengthSq);
return start + segment * t;
}
float cultmath_distance_to_segment(float2 point_value, float2 start, float2 end)
{
return cultmath_distance(point_value, cultmath_segment_closest(point_value, start, end));
}
float cultmath_distance_to_segment(float3 point_value, float3 start, float3 end)
{
return cultmath_distance(point_value, cultmath_segment_closest(point_value, start, end));
}
#include "CultMath.Phacelle.hlsl"
#include "CultMath.Interval.hlsl"
#include "CultMath.Cycles.hlsl"
#include "CultMath.Affine.hlsl"
// Span stages (invariant 8): the SpanPath, SpanStage and SpanNoise of math.Span.cs, field for field in
// C#'s order, and cultmath_span_stage, its octave stage, statement for statement so the text is a bit-for-bit
// mirror. They follow the includes because the bounds read CULTMATH_SNOISE_LIPSCHITZ, _HESSIAN and _THIRD.
// A shader caller builds a SpanStage itself (SpanStage(1.0, 1.0, vec3(0.0), vec3(0.0), vec3(0.0), vec3(0.0))
// is the identity of math.SpanStage.Identity, which has no shader form); see math.span_stage's comments for
// the span, the transform, the bounds and the allowances. cultmath_span_rounding and cultmath_span_slack
// are math.span_rounding and math.span_slack, the owners of S_o and of the three allowances.
struct SpanPath
{
float3 midpoint;
float3 velocity;
float speed;
float acceleration;
float halfLength;
};
struct SpanStage
{
float frequency;
float stretch;
float3 axis;
float3 shift;
float3 offset;
float3 octaveStep;
};
struct SpanNoise
{
float4 field;
float curvature;
float fold;
};
float cultmath_span_rounding(float scale, float3 mq, SpanStage stage, float3 octaveShift, float nu, float halfLength)
{
return 9.5367431640625e-7 * (abs(scale) * (cultmath_csum(abs(mq)) + cultmath_csum(abs(stage.shift))) + cultmath_csum(abs(stage.offset)) + cultmath_csum(abs(octaveShift)) + 1.7320508 * nu * halfLength);
}
float3 cultmath_span_slack(float rounding)
{
return float3(CULTMATH_SNOISE_READ_VALUE + CULTMATH_SNOISE_LIPSCHITZ * rounding,
CULTMATH_SNOISE_READ_GRADIENT + CULTMATH_SNOISE_HESSIAN * rounding,
9.765625e-4 + CULTMATH_SNOISE_THIRD * rounding);
}
// The octave stage behind cultmath_span_fbm and cultmath_span_ridged, as math.span_stage is behind span_fbm
// and span_ridged. The Frobenius norm of the Hessian is written out in C#'s order (GLSL has no _mRC member
// access); the C# local named step is octaveShift here (it would shadow GLSL's step()).
SpanNoise cultmath_span_stage(bool ridged, SpanPath path, SpanStage stage, int octaves, float lacunarity, float gain)
{
float3 q = path.midpoint;
float halfLength = path.halfLength;
octaves = clamp(octaves, 0, 16);
bool stretched = dot(stage.axis, stage.axis) > 0.0 && stage.stretch != 1.0;
float3 direction = stretched ? normalize(stage.axis) : float3(0.0, 0.0, 0.0);
float squeeze = stretched ? 1.0 / stage.stretch - 1.0 : 0.0;
float norm = abs(stage.frequency) * (stretched ? max(1.0, 1.0 / stage.stretch) : 1.0);
float3 mq = stretched ? stage.frequency * (q + squeeze * dot(direction, q) * direction) : stage.frequency * q;
float3 mv = stretched ? stage.frequency * (path.velocity + squeeze * dot(direction, path.velocity) * direction) : stage.frequency * path.velocity;
float amplitude = 1.0;
float scale = 1.0;
float value = 0.0;
float3 gradient = float3(0.0, 0.0, 0.0);
float curvature = 0.0;
float fold = 0.0;
for (int i = 0; i < octaves; i++)
{
float3 octaveShift = (float)i * stage.octaveStep;
float3 x = scale * (mq + stage.shift) + (stage.offset + octaveShift);
float4 n = cultmath_snoise_grad(x);
float3 g = float3(n.x, n.y, n.z);
float3 mg = stretched ? stage.frequency * (g + squeeze * dot(direction, g) * direction) : stage.frequency * g;
if (ridged)
{
float s = sign(n.w);
value += amplitude * (1.0 - abs(n.w));
gradient += -amplitude * scale * s * mg;
}
else
{
value += amplitude * n.w;
gradient += amplitude * scale * mg;
}
float weight = abs(amplitude);
float factor = abs(scale) * norm;
float nu = factor * path.speed;
float rounding = cultmath_span_rounding(scale, mq, stage, octaveShift, nu, halfLength);
float3 slack = cultmath_span_slack(rounding);
float valueSlack = slack.x;
float gradientSlack = slack.y;
float hessianSlack = slack.z;
float3x3 hm = cultmath_snoise_hessian(x);
float frobenius = sqrt(hm[0][0] * hm[0][0] + hm[0][1] * hm[0][1] + hm[0][2] * hm[0][2] + hm[1][0] * hm[1][0] + hm[1][1] * hm[1][1] + hm[1][2] * hm[1][2]
+ hm[2][0] * hm[2][0] + hm[2][1] * hm[2][1] + hm[2][2] * hm[2][2]);
float hessianBound = min(CULTMATH_SNOISE_HESSIAN, frobenius + hessianSlack + CULTMATH_SNOISE_THIRD * nu * halfLength);
curvature += weight * factor * factor * hessianBound;
if (ridged)
{
float3 w = scale * mv;
float reach = hessianBound * nu * halfLength;
float gl = min(CULTMATH_SNOISE_LIPSCHITZ, length(g) + gradientSlack + reach);
float travel = (abs(dot(g, w)) + gradientSlack * length(w)) * halfLength + (hessianBound * nu * length(w) + gl * factor * path.acceleration) * (0.5 * halfLength * halfLength) + 2.0 * valueSlack;
fold += 2.0 * weight * max(travel - abs(n.w), 0.0);
}
scale *= lacunarity;
amplitude *= gain;
}
SpanNoise result;
result.field = float4(gradient, value);
result.curvature = curvature;
result.fold = fold;
return result;
}
SpanNoise cultmath_span_fbm(SpanPath path, SpanStage stage, int octaves, float lacunarity, float gain)
{
return cultmath_span_stage(false, path, stage, octaves, lacunarity, gain);
}
SpanNoise cultmath_span_ridged(SpanPath path, SpanStage stage, int octaves, float lacunarity, float gain)
{
return cultmath_span_stage(true, path, stage, octaves, lacunarity, gain);
}
float cultmath_value_noise(float2 position)
{
float2 cell = floor(position);

2
packages/cultmath/src/CultMath/CultMath.csproj
View file

@ -10,7 +10,7 @@
<PackageId>GameCult.Math</PackageId>
<!-- MIT overall; math.Phacelle.cs and CultPhasor.cs (and the two Phacelle HLSL files) are MPL-2.0; math.Cycles.cs and CultMath.Cycles.hlsl are Apache-2.0. -->
<PackageLicenseExpression>MIT AND MPL-2.0 AND Apache-2.0</PackageLicenseExpression>
<Version>0.4.1</Version>
<Version>0.5.0</Version>
<Description>Clean-room C# math shaped for HLSL-style renderer code.</Description>
<Authors>GameCult</Authors>
<PackageReadmeFile>README.md</PackageReadmeFile>

10
packages/cultmath/src/CultMath/CultPhasor.cs
View file

@ -14,14 +14,14 @@ namespace CultMath;
public struct CultPhasor
{
/// <summary>(&#8711;cos, cos): the normalized cosine wave and its gradient.</summary>
public float4 cos;
public float4 cosine;
/// <summary>(&#8711;sin, sin): the normalized sine wave and its gradient.</summary>
public float4 sin;
public float4 sine;
public CultPhasor(float4 cos, float4 sin)
public CultPhasor(float4 cosine, float4 sine)
{
this.cos = cos;
this.sin = sin;
this.cosine = cosine;
this.sine = sine;
}
}

30
packages/cultmath/src/CultMath/CultSiteCells.cs Normal file
View file

@ -0,0 +1,30 @@
namespace CultMath;
/// <summary>
/// The two nearest sites of a query point under one site score (<see cref="math.power_score"/> or
/// <see cref="math.multiplicative_score"/>), built by folding sites through
/// <see cref="math.site_cells_insert"/>. Each score field follows the
/// <c>float4(gradient.xyz, value.w)</c> layout of a bare value-and-gradient return.
/// </summary>
public struct CultSiteCells
{
/// <summary>(&#8711;F1, F1): the smallest score and its gradient. 1.0e30 in w when no site was inserted.</summary>
public float4 nearest;
/// <summary>(&#8711;F2, F2): the second smallest score and its gradient. 1.0e30 in w when fewer than two sites were inserted.</summary>
public float4 second;
/// <summary>Index of the nearest site as inserted, -1 when none. A float so the struct holds one scalar type, as <see cref="CultCellular.id"/> does; exact below 2^24.</summary>
public float nearestSite;
/// <summary>Index of the second nearest site as inserted, -1 when none.</summary>
public float secondSite;
public CultSiteCells(float4 nearest, float4 second, float nearestSite, float secondSite)
{
this.nearest = nearest;
this.second = second;
this.nearestSite = nearestSite;
this.secondSite = secondSite;
}
}

2
packages/cultmath/src/CultMath/CultSiteCells.cs.meta Normal file
View file

@ -0,0 +1,2 @@
fileFormatVersion: 2
guid: f6479c37f7d349708f54ec7e4d671200

29
packages/cultmath/src/CultMath/SpanNoise.cs Normal file
View file

@ -0,0 +1,29 @@
namespace CultMath;
/// <summary>
/// The result of <see cref="math.span_fbm"/> and <see cref="math.span_ridged"/> (invariant 8's one struct
/// return shape): an octave stage's value and gradient at the midpoint of a span, with the bounds that
/// hold over the whole span.
/// </summary>
public struct SpanNoise
{
/// <summary>(&#8711;, value): the stage's gradient in the span's space and its value, in the <c>float4(gradient.xyz, value.w)</c> layout of a bare value-and-gradient return.</summary>
public float4 field;
/// <summary>An upper bound, at every point the span can reach, on the operator norm of the field's Hessian in the span's space.</summary>
public float curvature;
/// <summary>
/// An upper bound on how far a ridged sum can leave its midpoint branch over the span (each octave's
/// 1 - |n| against the same octave continued with the midpoint's sign of n); 0 for
/// <see cref="math.span_fbm"/> and for any octave that cannot cross zero over the span.
/// </summary>
public float fold;
public SpanNoise(float4 field, float curvature, float fold)
{
this.field = field;
this.curvature = curvature;
this.fold = fold;
}
}

2
packages/cultmath/src/CultMath/SpanNoise.cs.meta Normal file
View file

@ -0,0 +1,2 @@
fileFormatVersion: 2
guid: fe0da4c343e24c3781b076943f08ddbc

26
packages/cultmath/src/CultMath/SpanPath.cs Normal file
View file

@ -0,0 +1,26 @@
namespace CultMath;
/// <summary>
/// The span <see cref="math.span_fbm"/> and <see cref="math.span_ridged"/> bound over: the path q(t),
/// |t| &lt;= <see cref="halfLength"/>, through <see cref="midpoint"/> at t = 0 with <see cref="velocity"/> there,
/// |q'| &lt;= <see cref="speed"/> and |q''| &lt;= <see cref="acceleration"/> along it. speed is at least
/// |velocity|; speed, acceleration and halfLength are non-negative. Name every field: the three floats
/// have no order a caller could mistake for a convention.
/// </summary>
public struct SpanPath
{
/// <summary>q(0), the point the field and its gradient are evaluated at.</summary>
public float3 midpoint;
/// <summary>q'(0).</summary>
public float3 velocity;
/// <summary>An upper bound on |q'(t)| over the span; at least |velocity|.</summary>
public float speed;
/// <summary>An upper bound on |q''(t)| over the span.</summary>
public float acceleration;
/// <summary>The span covers |t| &lt;= halfLength; 0 is the midpoint alone.</summary>
public float halfLength;
}

2
packages/cultmath/src/CultMath/SpanPath.cs.meta Normal file
View file

@ -0,0 +1,2 @@
fileFormatVersion: 2
guid: 3a8c1d52e7b94f06a1d5c2b9e04f7a31

33
packages/cultmath/src/CultMath/SpanStage.cs Normal file
View file

@ -0,0 +1,33 @@
namespace CultMath;
/// <summary>
/// The transform of one octave stage of a warp chain, as <see cref="math.span_fbm"/> and
/// <see cref="math.span_ridged"/> read it. Octave o samples snoise at x_o = f_o (M q + shift) + offset +
/// o octaveStep, with f_o = lacunarity^o and M = <see cref="frequency"/> (I + (1/<see cref="stretch"/> - 1) a a^T),
/// a = normalize(<see cref="axis"/>); a zero axis, or a stretch of 1, means no stretch. stretch must be
/// positive and finite, and nothing guards it. <c>default(SpanStage)</c> has frequency 0, which samples
/// one point: start from <see cref="Identity"/>.
/// </summary>
public struct SpanStage
{
/// <summary>The stage whose span_fbm/span_ridged field equals fbm_grad/ridged_grad: frequency 1, stretch 1, every vector zero.</summary>
public static readonly SpanStage Identity = new SpanStage { frequency = 1.0f, stretch = 1.0f };
/// <summary>The scale of M; its sign is kept.</summary>
public float frequency;
/// <summary>The factor the axis direction is stretched by (M scales it by 1/stretch); 1 is none.</summary>
public float stretch;
/// <summary>The direction stretch applies along; any length, zero for none.</summary>
public float3 axis;
/// <summary>Added to M q before the per-octave scaling.</summary>
public float3 shift;
/// <summary>Added after the per-octave scaling.</summary>
public float3 offset;
/// <summary>Added to the octave coordinate once per octave index: o octaveStep.</summary>
public float3 octaveStep;
}

2
packages/cultmath/src/CultMath/SpanStage.cs.meta Normal file
View file

@ -0,0 +1,2 @@
fileFormatVersion: 2
guid: 9d46b0f27c1e4a85b3e8f1a60c2d5b97

9
packages/cultmath/src/CultMath/math.Affine.cs
View file

@ -35,6 +35,13 @@ public static partial class math
// kernel must re-run MeasureHessian and re-pin it. SNOISE_THIRD (math.Span.cs) is the next-order pin.
public const float SNOISE_HESSIAN = 55.642143f;
// The absolute rounding of one float32 read of the snoise kernel, apart from the coordinate's own
// rounding (af_snoise's comment derives both): 2^-14 on the value and 2^-12 on the gradient. The single
// owner of the two, read by af_snoise and by span_slack (math.Span.cs); the shader twins are
// CULTMATH_SNOISE_READ_VALUE and CULTMATH_SNOISE_READ_GRADIENT.
internal const float SNOISE_READ_VALUE = 6.103515625e-5f;
internal const float SNOISE_READ_GRADIENT = 2.44140625e-4f;
/// <summary>The point form (x, 0, 0): the value x at every eps. Exact.</summary>
public static float3 af_point(float value) => new(value, 0.0f, 0.0f);
@ -165,7 +172,7 @@ public static partial class math
var lipschitz = SNOISE_LIPSCHITZ * reach;
var a = dot(gradient, axis);
var remainder = length(gradient) * radius + 0.5f * SNOISE_HESSIAN * reach * reach;
var allowance = 6.103515625e-5f + 2.44140625e-4f * reach
var allowance = SNOISE_READ_VALUE + SNOISE_READ_GRADIENT * reach
+ SNOISE_LIPSCHITZ * 9.5367431640625e-7f * (abs(centre.x) + abs(centre.y) + abs(centre.z) + reach);
var centred = abs(a) + remainder < lipschitz;
var slope = centred ? a : 0.0f;

212
packages/cultmath/src/CultMath/math.Lattice.cs Normal file
View file

@ -0,0 +1,212 @@
namespace CultMath;
public static partial class math
{
// Weighted lattice cells: a jittered, drifting cellular field whose cells are weighted by how close the cell's
// centre lies to a border of the body diagram (the power diagram of the zone's body span). The public steps are
// the four per-cell rules a tape or a shader composes; weighted_lattice_cells is the loop over them.
/// <summary>The greatest boost <see cref="weighted_lattice_cells"/> accepts: weights span 0.5 to 5, a ratio of 10:1.</summary>
public const float LatticeMaxBoost = 4.0f;
/// <summary>The greatest stretch <see cref="weighted_lattice_cells"/> accepts.</summary>
public const float LatticeMaxStretch = 2.0f;
/// <summary>The greatest smoothing <see cref="weighted_lattice_cells"/> accepts, in cell units.</summary>
public const float LatticeMaxSmoothing = 1.0f;
/// <summary>
/// The outermost ring the search can reach anywhere in the accepted range (see the proof above
/// <see cref="weighted_lattice_cells"/>).
/// </summary>
public const int LatticeMaxRing = 37;
// A query beyond this many cells from the origin has no integer cell id in float precision.
private const float LatticeMaxCoordinate = 8388608.0f;
/// <summary>
/// The site of a lattice cell: <c>float3(site.xy, base weight)</c>. The site is <c>cell + 0.5 + 0.5 sin(drift
/// time + 2 pi u)</c> per axis, so it stays within one cell of the cell's corner and drifts at rate
/// <paramref name="drift"/>. The base weight is <c>0.5 + 0.5 u</c>, in [0.5, 1), from its own hash lane.
/// </summary>
public static float3 lattice_site(float2 cell, float time, float drift)
{
var h = pcg3d(new int3((int)cell.x, (int)cell.y, 0));
var phase = new float2(cellular_unit(h.x), cellular_unit(h.y));
var site = cell + 0.5f + 0.5f * new float2(sin(drift * time + TAU * phase.x), sin(drift * time + TAU * phase.y));
return new float3(site, 0.5f + 0.5f * cellular_unit(h.z));
}
/// <summary>
/// The weight of a cell: <c>base (1 + boost exp(-(border / falloff)^2))</c>, where <paramref name="border"/> is
/// the distance from the cell's centre to the nearest body-diagram border.
/// </summary>
public static float lattice_weight(float @base, float border, float boost, float falloff)
{
var t = border / falloff;
return @base * (1.0f + boost * exp(-(t * t)));
}
/// <summary>
/// The weighted distance from <paramref name="q"/> to <paramref name="site"/>, in cell units:
/// <c>weight |(stretch a, c)|</c> with <c>a</c> and <c>c</c> the offset's components along and across the unit
/// <paramref name="normal"/>. Isotropic at stretch 1.
/// </summary>
public static float lattice_distance(float2 q, float2 site, float weight, float2 normal, float stretch)
{
var v = site - q;
var a = dot(v, normal);
var c = v.x * normal.y - v.y * normal.x;
return weight * length(new float2(stretch * a, c));
}
/// <summary>
/// One step of the smooth minimum fold: <paramref name="m"/> folded with distance <paramref name="d"/>, for a
/// nearest distance <paramref name="f1"/>. A distance of at least <c>f1 + 2 smoothing</c> leaves
/// <paramref name="m"/> unchanged, and the result never exceeds <c>f1 + smoothing</c>. Start the fold at
/// <c>f1 + smoothing</c> and visit every cell once, in an order that does not depend on the search box.
/// </summary>
public static float lattice_smooth_step(float m, float d, float f1, float smoothing)
{
if (d >= f1 + 2.0f * smoothing)
return m;
var h = smoothstep(0.0f, 1.0f, 0.5f + 0.5f * (m - d) / smoothing);
var next = m + (d - m) * h - h * (1.0f - h) * smoothing / (1.0f + 3.0f * smoothing);
return min(next, f1 + smoothing);
}
// Proof that the ring search finds every cell that can change an output (ruling lattice-ring-exit-sound).
//
// Let c0 = floor(q) and let a cell lie on Chebyshev ring k around c0 (an offset of k on some axis). Its site lies
// within [cell, cell + 1] per axis and q within [c0, c0 + 1), so on that axis |site - q| >= k - 1. The length of
// the offset is at least that, the stretch is at least 1 (it only lengthens), and the weight is at least
// 0.5 (base >= 0.5, boost >= 0). So every cell on ring k or beyond has d >= 0.5 (k - 1), and the same holds for
// every farther ring.
//
// A cell with d >= F2 changes neither F1 nor F2, and a cell with d >= F1 + 2 smoothing leaves the smooth minimum
// unchanged (lattice_smooth_step returns m before folding it). So the search stops before ring k once
// 0.5 (k - 1) >= max(F2, F1 + 2 smoothing), with F1 and F2 taken over the rings visited. Those values only fall
// as rings are added, so the running cutoff is at least the final one: stopping on it never stops early. Every
// unvisited cell then changes no output, for every weight, stretch and drift.
//
// The loop terminates inside the accepted range. Cell c0 gives a distance below sqrt(2) in cell units. The
// neighbour on the nearer side of the axis whose query fraction sits closest to a cell boundary (fraction m <= 1/2)
// lies within (1 + m, 1) per axis, below sqrt(3.25). Weights are below 1 + LatticeMaxBoost and stretch is at most
// LatticeMaxStretch, so F2 < 10 sqrt(3.25) = 18.03 and F1 + 2 smoothing < 10 sqrt(2) + 2 = 16.2. The exit holds at
// 0.5 (k - 1) >= 18.03, that is at k = 38, so rings 0 to 37 (LatticeMaxRing) are the most visited.
//
// The result does not depend on the box visited. The second pass folds F1, F2 and the winning id in absolute
// row-major order (strict <, so an exact tie keeps the earlier cell), and the smooth minimum starts at
// F1 + smoothing and is clamped to it, so a cell with d >= F1 + 2 smoothing, visited or not, changes nothing.
/// <summary>
/// Weighted cellular noise over a jittered lattice: <c>float4(F1, F2, smooth, id)</c>. F1 and F2 are the two
/// smallest weighted distances from <paramref name="p"/> in cell units times weight, smooth is the smooth minimum
/// (<paramref name="smoothing"/>, in the same units), and id is a hash of the winning cell in [0, 1).
/// </summary>
/// <remarks>
/// A cell's weight is <see cref="lattice_weight"/> of the exact power border distance (<see cref="power_border"/>)
/// of the cell's centre in the body span <paramref name="bodies"/> (<c>float4(position.xyz, sigma)</c>, z = 0), so it is
/// continuous as the bodies move. Distances are stretched by <paramref name="stretch"/> along
/// <paramref name="stretchNormal"/> (isotropic at 1). The search is exact in the accepted range: boost 0 to
/// <see cref="LatticeMaxBoost"/>, stretch 1 to <see cref="LatticeMaxStretch"/>, smoothing up to <see cref="LatticeMaxSmoothing"/>.
/// </remarks>
/// <exception cref="ArgumentException">An argument is not finite, a size is not above zero, a value is outside its range, the normal is zero while stretch exceeds 1, or the body span is refused as <see cref="power_cells"/> refuses it.</exception>
public static float4 weighted_lattice_cells(
float2 p, float time, ReadOnlySpan<float4> bodies, float cellSize, float drift, float boost, float falloff,
float smoothing, float2 stretchNormal, float stretch)
{
// Cell centres stay finite: the farthest one the search can reach is a ring past the query's cell.
var reachSpan = cellSize * (LatticeMaxRing + 2);
if (!float.IsFinite(cellSize) || !(cellSize > 0.0f) || !float.IsFinite(reachSpan))
throw new ArgumentException("cellSize must be finite and greater than zero, and small enough for its cell centres to be finite.", nameof(cellSize));
if (!float.IsFinite(p.x) || !float.IsFinite(p.y) || !(abs(p.x / cellSize) < LatticeMaxCoordinate) || !(abs(p.y / cellSize) < LatticeMaxCoordinate)
|| !float.IsFinite(abs(p.x) + reachSpan) || !float.IsFinite(abs(p.y) + reachSpan))
throw new ArgumentException("p must be finite and within 2^23 cells of the origin.", nameof(p));
if (!float.IsFinite(time))
throw new ArgumentException("time must be finite.", nameof(time));
if (!float.IsFinite(drift))
throw new ArgumentException("drift must be finite.", nameof(drift));
if (!float.IsFinite(drift * time))
throw new ArgumentException("drift times time must be finite.", nameof(drift));
if (!float.IsFinite(boost) || boost < 0.0f || boost > LatticeMaxBoost)
throw new ArgumentException("boost must be within [0, LatticeMaxBoost].", nameof(boost));
if (!float.IsFinite(falloff) || !(falloff > 0.0f))
throw new ArgumentException("falloff must be finite and greater than zero.", nameof(falloff));
if (!float.IsFinite(smoothing) || !(smoothing > 0.0f) || smoothing > LatticeMaxSmoothing)
throw new ArgumentException("smoothing must be within (0, LatticeMaxSmoothing].", nameof(smoothing));
if (!float.IsFinite(stretch) || stretch < 1.0f || stretch > LatticeMaxStretch)
throw new ArgumentException("stretch must be within [1, LatticeMaxStretch].", nameof(stretch));
if (!float.IsFinite(stretchNormal.x) || !float.IsFinite(stretchNormal.y))
throw new ArgumentException("stretchNormal must be finite.", nameof(stretchNormal));
// Scaled by its largest component first, so a tiny or huge normal normalises to unit length.
var scale = max(abs(stretchNormal.x), abs(stretchNormal.y));
if (scale > 0.0f)
{
stretchNormal /= scale;
stretchNormal /= length(stretchNormal);
}
else if (stretch > 1.0f)
throw new ArgumentException("stretchNormal must not be zero while stretch exceeds 1.", nameof(stretchNormal));
else
stretchNormal = new float2(1.0f, 0.0f);
require_cell_span(new float3(p, 0.0f), bodies, positiveWeights: false, nameof(bodies));
const int Width = 2 * LatticeMaxRing + 1;
Span<float> cache = stackalloc float[Width * Width];
var q = p / cellSize;
var origin = floor(q);
var cx0 = (int)origin.x;
var cz0 = (int)origin.y;
var f1 = 1.0e30f;
var f2 = 1.0e30f;
var reach = 0;
for (var k = 0; ; k++)
{
if (k > 1 && 0.5f * (k - 1) >= max(f2, f1 + 2.0f * smoothing))
break;
if (k > LatticeMaxRing)
throw new InvalidOperationException("The lattice ring search left its proven range.");
for (var dz = -k; dz <= k; dz++)
{
var step = dz == -k || dz == k || k == 0 ? 1 : 2 * k;
for (var dx = -k; dx <= k; dx += step)
{
var d = lattice_cell_distance(q, new float2(cx0 + dx, cz0 + dz), time, drift, bodies, cellSize, boost, falloff, stretchNormal, stretch);
cache[(dz + LatticeMaxRing) * Width + dx + LatticeMaxRing] = d;
if (d < f1) { f2 = f1; f1 = d; }
else if (d < f2) f2 = d;
}
}
reach = k;
}
var m = f1 + smoothing;
var best = 1.0e30f;
var second = 1.0e30f;
var winDx = 0;
var winDz = 0;
for (var dz = -reach; dz <= reach; dz++)
for (var dx = -reach; dx <= reach; dx++)
{
var d = cache[(dz + LatticeMaxRing) * Width + dx + LatticeMaxRing];
if (d < best) { second = best; best = d; winDx = dx; winDz = dz; }
else if (d < second) second = d;
m = lattice_smooth_step(m, d, f1, smoothing);
}
var id = pcg4d(new int4(cx0 + winDx, cz0 + winDz, 0, 0));
return new float4(best, second, m, cellular_unit(id.w));
}
internal static float lattice_cell_distance(
float2 q, float2 cell, float time, float drift, ReadOnlySpan<float4> bodies, float cellSize, float boost, float falloff,
float2 normal, float stretch)
{
var site = lattice_site(cell, time, drift);
var centre = (cell + 0.5f) * cellSize;
var border = power_border_unchecked(new float3(centre, 0.0f), bodies).w;
return lattice_distance(q, site.xy, lattice_weight(site.z, border, boost, falloff), normal, stretch);
}
}

2
packages/cultmath/src/CultMath/math.Lattice.cs.meta Normal file
View file

@ -0,0 +1,2 @@
fileFormatVersion: 2
guid: 0a182e056c8c4258881c347ead6d571d

209
packages/cultmath/src/CultMath/math.Sites.cs Normal file
View file

@ -0,0 +1,209 @@
namespace CultMath;
public static partial class math
{
// Sites: the model page's span element float4(position.xyz, weight). A span function is a C# loop over
// a per-site step that has a shader twin (site_cells_empty, then site_cells_insert per site); a shader
// cannot take a span, so shader callers write the same loop over their own buffer. One fold, one rule.
// Power score of a site: value |p - s|^2 - w^2, gradient 2 (p - s). The cell of a site is the intersection
// of the half-spaces bounded by its radical planes against every other site (power_border_step).
public static float4 power_score(float3 p, float4 site)
{
var offset = p - site.xyz;
return new float4(offset * 2.0f, dot(offset, offset) - site.w * site.w);
}
// Multiplicative score of a site: value |p - s| / w, gradient (p - s) / (|p - s| w). At p = s the
// gradient is zero, cellular's degenerate-gradient convention.
public static float4 multiplicative_score(float3 p, float4 site)
{
var offset = p - site.xyz;
var r = length(offset);
var gradient = r > 0.0f ? offset / (r * site.w) : new float3(0.0f, 0.0f, 0.0f);
return new float4(gradient, r / site.w);
}
// The fold's identity: no site yet. 1.0e30 is cellular's sentinel (GLSL ES has no inf literal).
public static CultSiteCells site_cells_empty() =>
new(new float4(0.0f, 0.0f, 0.0f, 1.0e30f), new float4(0.0f, 0.0f, 0.0f, 1.0e30f), -1.0f, -1.0f);
// One fold step. Strict < against F1 (the old F1 moves to second), else strict < against F2, so an exact
// tie keeps the earlier index first.
public static CultSiteCells site_cells_insert(CultSiteCells cells, float4 score, int index)
{
var result = cells;
if (score.w < cells.nearest.w)
{
result.second = cells.nearest;
result.secondSite = cells.nearestSite;
result.nearest = score;
result.nearestSite = (float)index;
}
else if (score.w < cells.second.w)
{
result.second = score;
result.secondSite = (float)index;
}
return result;
}
// float4(unit normal, distance) of the bisector of the two lowest scores: n = grad F2 - grad F1, distance
// (F2 - F1) / |n|, normal n / |n| (into the nearest site's cell). First order, for either metric. It is NOT
// the distance to the nearest site's cell boundary, which the second-lowest score does not give in general;
// for power scores that is power_border. (0, 0, 0, 1.0e30) when there is no second site or |n| = 0.
public static float4 site_cells_border(CultSiteCells cells)
{
var n = cells.second.xyz - cells.nearest.xyz;
var len = length(n);
if (cells.secondSite < 0.0f || len <= 0.0f)
return new float4(0.0f, 0.0f, 0.0f, 1.0e30f);
return new float4(n / len, (cells.second.w - cells.nearest.w) / len);
}
// One step of the exact power cell border. owner is the site whose cell p lies in (the fold's nearest);
// other is another site. The radical plane of the pair is {x : |x - s1|^2 - w1^2 = |x - s2|^2 - w2^2};
// with d = s2 - s1 the signed distance from p to it, positive on the owner's side, is
// ((s1 - p).d + (|d|^2 + w1^2 - w2^2) / 2) / |d|. It is built from the offset s1 - p and the weight
// difference (w1 - w2)(w1 + w2), never from a difference of two large squared distances, so it does not
// cancel for a far query or close sites. border is the running (unit normal toward the owner, distance),
// starting at (0, 0, 0, 1.0e30): the cell's boundary distance is the least plane distance over every other
// site. A site coincident with the owner has no plane and leaves border unchanged. The step trusts the caller's
// owner: a p outside the owner's cell has a negative signed distance to some plane, which is clamped to 0, so the
// step reports distance 0 with that plane's normal, a plausible on-the-border answer, not an error.
public static float4 power_border_step(float3 p, float4 owner, float4 other, float4 border)
{
var d = other.xyz - owner.xyz;
var len = length(d);
if (len <= 0.0f)
return border;
var plane = max((dot(owner.xyz - p, d) + 0.5f * (dot(d, d) + (owner.w - other.w) * (owner.w + other.w))) / len, 0.0f);
return plane < border.w ? new float4(-d / len, plane) : border;
}
// Gabriel margin of the pair (a, b) against a third site: (|other - mid| - r) / r with mid = (a + b) / 2
// and r = |a - b| / 2. 0 on the diameter circle, negative inside it, -1 when a and b coincide.
public static float gabriel_margin(float3 a, float3 b, float3 other)
{
var r = distance(a, b) * 0.5f;
if (r <= 0.0f)
return -1.0f;
return (distance(other, (a + b) * 0.5f) - r) / r;
}
// Index floats hold integers exactly up to 2^24, so a longer span is refused.
private const int MaxSiteSpan = 1 << 24;
private static bool all_finite(float3 v) => float.IsFinite(v.x) && float.IsFinite(v.y) && float.IsFinite(v.z);
// The one validation of a cell span: finite query, finite positions and weights, weights above zero when
// positiveWeights, at most 2^24 sites. Messages name the parameter, never a value.
private static void require_cell_span(float3 p, ReadOnlySpan<float4> sites, bool positiveWeights, string spanName = "sites")
{
if (!all_finite(p))
throw new ArgumentException("The query point must be finite.", nameof(p));
if (sites.Length > MaxSiteSpan)
throw new ArgumentException("A site span holds at most 2^24 sites.", spanName);
for (var i = 0; i < sites.Length; i++)
{
if (!all_finite(sites[i].xyz))
throw new ArgumentException("Site positions must be finite.", spanName);
if (!float.IsFinite(sites[i].w) || (positiveWeights && !(sites[i].w > 0.0f)))
throw new ArgumentException(positiveWeights ? "Site weights must be finite and greater than zero." : "Site weights must be finite.", spanName);
}
}
/// <summary>
/// The two nearest sites of <paramref name="p"/> by <see cref="power_score"/> (the power diagram), every
/// site visited. A site is <c>float4(position.xyz, weight)</c>; use z = 0 for 2D. Ties keep the lower index first.
/// </summary>
/// <exception cref="ArgumentException">The query or a site position or weight is not finite, or the span holds more than 2^24 sites.</exception>
public static CultSiteCells power_cells(float3 p, ReadOnlySpan<float4> sites)
{
require_cell_span(p, sites, positiveWeights: false);
return power_cells_unchecked(p, sites);
}
private static CultSiteCells power_cells_unchecked(float3 p, ReadOnlySpan<float4> sites)
{
var cells = site_cells_empty();
for (var i = 0; i < sites.Length; i++)
cells = site_cells_insert(cells, power_score(p, sites[i]), i);
return cells;
}
/// <summary>
/// The exact distance from <paramref name="p"/> to the boundary of its power cell, and the unit normal of the
/// nearest boundary plane pointing into the cell: <c>float4(normal, distance)</c>. The cell is the
/// intersection of the half-spaces bounded by the radical planes against every other site, so the distance is
/// the least <see cref="power_border_step"/> over them. A site coincident with the owner has no plane; a cell
/// with no bounding plane (one site, or only coincident ones) gives <c>(0, 0, 0, 1.0e30)</c>.
/// </summary>
/// <exception cref="ArgumentException">As <see cref="power_cells"/>.</exception>
public static float4 power_border(float3 p, ReadOnlySpan<float4> sites)
{
require_cell_span(p, sites, positiveWeights: false);
return power_border_unchecked(p, sites);
}
// The border over an already validated span, which the lattice calls once per cell.
private static float4 power_border_unchecked(float3 p, ReadOnlySpan<float4> sites)
{
var border = new float4(0.0f, 0.0f, 0.0f, 1.0e30f);
var owner = (int)power_cells_unchecked(p, sites).nearestSite;
if (owner < 0)
return border;
for (var k = 0; k < sites.Length; k++)
if (k != owner)
border = power_border_step(p, sites[owner], sites[k], border);
return border;
}
/// <summary>
/// The two nearest sites of <paramref name="p"/> by <see cref="multiplicative_score"/> (distance divided by
/// weight), every site visited. A site is <c>float4(position.xyz, weight)</c>; use z = 0 for 2D.
/// </summary>
/// <exception cref="ArgumentException">The query or a site position is not finite, a weight is not finite and greater than zero, or the span holds more than 2^24 sites.</exception>
public static CultSiteCells multiplicative_cells(float3 p, ReadOnlySpan<float4> sites)
{
require_cell_span(p, sites, positiveWeights: true);
var cells = site_cells_empty();
for (var i = 0; i < sites.Length; i++)
cells = site_cells_insert(cells, multiplicative_score(p, sites[i]), i);
return cells;
}
/// <summary>
/// Whether sites <paramref name="first"/> and <paramref name="second"/> are neighbours in the Gabriel graph
/// of <paramref name="sites"/> (positions only; weights are ignored): smoothstep(-softness, softness, the
/// least <see cref="gabriel_margin"/> over every other site). 1 when no other site exists.
/// </summary>
/// <exception cref="ArgumentOutOfRangeException">An index is outside the span.</exception>
/// <exception cref="ArgumentException">The indices are equal, softness is not finite and greater than zero, or a site position is not finite.</exception>
public static float soft_gabriel_gate(ReadOnlySpan<float4> sites, int first, int second, float softness)
{
if (first < 0 || first >= sites.Length)
throw new ArgumentOutOfRangeException(nameof(first), "Index is outside the site span.");
if (second < 0 || second >= sites.Length)
throw new ArgumentOutOfRangeException(nameof(second), "Index is outside the site span.");
if (first == second)
throw new ArgumentException("The two indices must differ.", nameof(second));
if (!(float.IsFinite(softness) && softness > 0.0f))
throw new ArgumentException("Softness must be finite and greater than zero.", nameof(softness));
for (var k = 0; k < sites.Length; k++)
if (!all_finite(sites[k].xyz))
throw new ArgumentException("Site positions must be finite.", nameof(sites));
var margin = float.PositiveInfinity;
for (var k = 0; k < sites.Length; k++)
{
if (k == first || k == second)
continue;
margin = min(margin, gabriel_margin(sites[first].xyz, sites[second].xyz, sites[k].xyz));
}
return smoothstep(-softness, softness, margin);
}
}

2
packages/cultmath/src/CultMath/math.Sites.cs.meta Normal file
View file

@ -0,0 +1,2 @@
fileFormatVersion: 2
guid: 856a5e425b6b45f4a8b3e9a8cd32fe38

183
packages/cultmath/src/CultMath/math.Span.cs
View file

@ -5,15 +5,18 @@ public static partial class math
// An upper bound on the operator norm of the third-derivative tensor of snoise(float3):
// max over unit u of |D^3 snoise(p)[u, u, u]|, the next order after SNOISE_LIPSCHITZ (gradient)
// and SNOISE_HESSIAN (curvature). Provenance: NoiseBoundTests.MeasureThird (slow, explicit) takes
// the largest such norm over 1e6 seeded points of [-256, 256]^3 (central differences of
// snoise_hessian at h = 1e-3, then an ascent over unit vectors for the tensor's operator norm),
// refines the 1e4 largest by ascent over the point, and multiplies the maximum by 1.10; it prints
// the witness start. NoiseBoundTests.ThirdConstantPinsSampled pins the value against a fast
// re-measure and the ascent from that witness. Measured on the tied kernel (snoise_grad's order),
// not on any pre-tie splice. Measured 488.1047 (a dense sweep of 20,000 directions at the witness
// found 99.9998% of the tensor norm there). Empirical with a margin, not a proof. A change to the snoise kernel
// must re-run MeasureThird and re-pin it.
public const float SNOISE_THIRD = 536.9152f;
// the largest such norm over 1e6 seeded points of [-256, 256]^3 of the closed-form tensor (snoise_hessian's
// per-corner derivation differentiated once more, in double, with the kernel's lattice, tie order and
// hash; NoiseBoundTests.ThirdTensorMatchesDifferencesOfTheHessian pins it against differences of the
// double Hessian), then an ascent over unit vectors for the tensor's operator norm, refines the 1e4
// largest by ascent over the point, and multiplies the maximum by 1.10; it prints the witness start.
// NoiseBoundTests.ThirdConstantPinsSampled pins the value against a fast re-measure and the ascent
// from that witness. Measured on the tied kernel (snoise_grad's order), not on any pre-tie splice:
// refined maximum 487.0917, a dense sweep of 20,000 directions at the witness's ascent end found
// 99.9987% of the tensor norm there. The inner maximisation is a local ascent and the outer a sampled
// search refined by ascent: empirical with a margin, not a proof. A change to the snoise kernel must
// re-run MeasureThird and re-pin it.
public const float SNOISE_THIRD = 535.8009f;
// Analytic Hessian of snoise(float3): the symmetric matrix of second partial derivatives, same
// lattice, hash and tie order as snoise_grad, whose kernel it differentiates once more. Every local
@ -100,14 +103,17 @@ public static partial class math
// at the bound the line starts beyond, the stretch on the ramp (a trapezoid: its length times the
// line's value at the stretch's midpoint), and the stretch held at the far bound. b = 0 is the
// constant clamp(a, lo, hi); t1 < t0 is the negative of the integral over [t1, t0]; lo above hi
// clamps to hi everywhere, as clamp does. A crossing that overflows (b tiny) lies past the span and
// cuts nothing.
// clamps to hi everywhere, as clamp does, and is normalised (lo = min(lo, hi)) before anything reads it,
// so no clamp ever sees lo above hi (GLSL leaves that undefined). A crossing that overflows (b tiny) lies past the span and
// cuts nothing. A held stretch contributes only when its length is positive, so an infinite bound
// (relu has hi = +inf) times a zero length never forms: the integral of a ramp clamped on one side
// only is finite.
public static float integral_clamp_linear(float a, float b, float lo, float hi, float t0, float t1)
{
if (b == 0.0f)
return clamp(a, lo, hi) * (t1 - t0);
lo = min(lo, hi);
if (b == 0.0f)
return t1 != t0 ? clamp(a, lo, hi) * (t1 - t0) : 0.0f;
var direction = t1 < t0 ? -1.0f : 1.0f;
var start = min(t0, t1);
var stop = max(t0, t1);
@ -117,7 +123,154 @@ public static partial class math
var leave = clamp(max(atLo, atHi), start, stop);
var before = b > 0.0f ? lo : hi;
var after = b > 0.0f ? hi : lo;
var total = before * (enter - start) + (a + b * (0.5f * (enter + leave))) * (leave - enter) + after * (stop - leave);
var held = enter > start ? before * (enter - start) : 0.0f;
var kept = stop > leave ? after * (stop - leave) : 0.0f;
var total = held + (a + b * (0.5f * (enter + leave))) * (leave - enter) + kept;
return direction * total;
}
private static float span_frobenius(float3x3 m) =>
sqrt(m._m00 * m._m00 + m._m01 * m._m01 + m._m02 * m._m02 + m._m10 * m._m10 + m._m11 * m._m11 + m._m12 * m._m12
+ m._m20 * m._m20 + m._m21 * m._m21 + m._m22 * m._m22);
// Plain fbm of snoise over a span: the octave stage of a warp chain, its value and gradient at the
// span's midpoint and the bounds that hold over the whole span. See span_stage for the span, the
// transform and the bound rules. At SpanStage.Identity the field equals fbm_grad(path.midpoint,
// octaves, lacunarity, gain) bit for bit.
public static SpanNoise span_fbm(SpanPath path, SpanStage stage, int octaves, float lacunarity, float gain) =>
span_stage(false, path, stage, octaves, lacunarity, gain);
// Ridged fbm over a span: octave o contributes gain^o (1 - |n_o|), ridged_grad's convention. fold
// bounds how far the sum can leave the branch it is on at the midpoint. At SpanStage.Identity the field
// equals ridged_grad bit for bit.
//
// The Span Lab's stage is 1 - 2|n| per octave. With G the sum of gain^o over the octaves (signed: a
// negative gain gives its alternating sum), the lab's value is 2 value - G, and its gradient,
// curvature and fold are twice this one's. A consumer in the lab form applies that mapping and does
// not derive its own bounds.
public static SpanNoise span_ridged(SpanPath path, SpanStage stage, int octaves, float lacunarity, float gain) =>
span_stage(true, path, stage, octaves, lacunarity, gain);
// 2^-20 S_o: the rounding one octave's read of the noise carries, S_o being the sum of the magnitudes the
// octave's coordinate is formed from and the span's reach (the allowance paragraph above). The single
// owner of S_o; span_slack turns it into the three allowances.
internal static float span_rounding(float scale, float3 mq, SpanStage stage, float3 step, float nu, float halfLength) =>
9.5367431640625e-7f * (abs(scale) * (csum(abs(mq)) + csum(abs(stage.shift))) + csum(abs(stage.offset)) + csum(abs(step)) + 1.7320508f * nu * halfLength);
// The allowances (e_n, e_g, e_h) of the value, the gradient's length and the Hessian's Frobenius norm at
// a rounding of 2^-20 S_o: the absolute term of each plus its derivative's pin times the rounding. The
// single owner of the three allowances of a span; af_snoise (math.Affine.cs) forms its own allowance
// from the same SNOISE_READ_VALUE and SNOISE_READ_GRADIENT, the owners of the first two absolute terms,
// so widening either moves every reader of the kernel at once.
internal static float3 span_slack(float rounding) => new(
SNOISE_READ_VALUE + SNOISE_LIPSCHITZ * rounding,
SNOISE_READ_GRADIENT + SNOISE_HESSIAN * rounding,
9.765625e-4f + SNOISE_THIRD * rounding);
// The span is SpanPath: q(t), |t| <= halfLength, through midpoint with velocity at t = 0, |q'| <= speed
// and |q''| <= acceleration along it (speed at least |velocity|; all three and halfLength non-negative).
// The transform is SpanStage: octave o samples snoise_grad at x_o(q) = f_o (M q + shift) + offset +
// o octaveStep with f_o = lacunarity^o, amplitude gain^o and M = frequency (I + (1/stretch - 1) a a^T),
// a = normalize(axis); a zero axis, or stretch 1, means no stretch. stretch must be positive and
// finite, and nothing guards it, as nothing guards lacunarity. octaves clamps to [0, 16] as fbm_grad's
// does. The field is the octave sum in q: gradient f_o M grad n_o (M is symmetric), the same octave
// loop as fbm_grad and ridged_grad.
//
// Bounds, per octave, from SNOISE_LIPSCHITZ (L), SNOISE_HESSIAN (H) and SNOISE_THIRD (T) only. With
// |M| = |frequency| max(1, 1/stretch) (1 without stretch) and nu = f_o |M| speed, the noise-space speed:
// * |x(t) - x_mid| <= nu halfLength, so the Hessian of n at x(t) has operator norm at most
// Hl = min(H, ||hessian(x_mid)||_F + e_h + T nu halfLength) (the Hessian is T-Lipschitz in operator
// norm, and the Frobenius norm bounds the operator norm), and |grad n(x(t))| at most
// Gl = min(L, |grad n_mid| + e_g + Hl nu halfLength).
// * curvature = sum gain^o f_o^2 |M|^2 Hl: the Hessian in q of n(x_o(q)) is f_o^2 M H M.
// * fold, ridged only. Over the span the octave's value departs from its midpoint branch
// gain^o (1 - s n) (s the sign of n at the midpoint) only where n has changed sign, and there
// |n(t)| <= V - |n_mid|, V bounding |n(t) - n_mid|. Writing n(t) - n_mid as the integral of
// grad n . x', x' = w + rho with w = f_o M velocity and |rho| <= f_o |M| acceleration |t|, and
// grad n = grad n_mid + delta with |delta| <= Hl nu |t|:
// V = (|grad n_mid . w| + e_g |w|) halfLength + (Hl nu |w| + Gl f_o |M| acceleration) halfLength^2 / 2
// + 2 e_n,
// and the octave's fold is 2 gain^o max(V - |n_mid|, 0), zero when it cannot reach zero.
//
// The float32 snoise, snoise_grad and snoise_hessian are not the exact functions, and the field a
// consumer reads is the float one, so what is read at the midpoint carries an allowance, by the
// derivation af_snoise's comment gives (math.Affine.cs): the skew and the corner offsets round the
// coordinate by about 2u of the magnitudes it is formed from, which moves a result by its derivative
// times that, and the caller's scaling of the point and a GPU's evaluation take 2u to 16u = 2^-20.
// S_o is the sum of those magnitudes, |f_o| (|M q|_1 + |shift|_1) + |offset|_1 + |o octaveStep|_1, plus
// the span's reach sqrt(3) nu halfLength (the coordinate of a path point); span_rounding is its owner. The
// allowances are the value's e_n = 2^-14 + L 2^-20 S_o, the gradient length's e_g = 2^-12 + H 2^-20 S_o
// (H is the gradient's own derivative's pin) and the Hessian's Frobenius norm's e_h = 2^-10 + T 2^-20 S_o
// (T is the Hessian's derivative's pin): SNOISE_READ_VALUE and SNOISE_READ_GRADIENT (math.Affine.cs) are the
// absolute terms of the first two, 2^-10 the Hessian's; span_slack is the owner of the three. They are observed where they are decided: the coordinate-band
// snoise_grad and snoise_hessian families of glsl-parity.json and SpanAllowanceCoversReadsAlongTheSpan
// check every read against them. Those observations are samples, not bounds: a new seed moves them. What
// they say is that the allowances enclose the error with a margin of about 15.
// No new pin enters.
//
// Domain: no coordinate range is checked, clamped or refused. The allowances grow with S_o, so
// curvature saturates at H (true everywhere) by S_o of about 1.1e5, and fold turns positive for every
// octave whose V + 2 e_n reaches |n_mid|: the bounds stay sound at every magnitude and stop being
// informative where float32 noise is (design.md, Precision domain).
private static SpanNoise span_stage(bool ridged, SpanPath path, SpanStage stage, int octaves, float lacunarity, float gain)
{
var q = path.midpoint;
var halfLength = path.halfLength;
octaves = clamp(octaves, 0, 16);
var stretched = dot(stage.axis, stage.axis) > 0.0f && stage.stretch != 1.0f;
var direction = stretched ? normalize(stage.axis) : new float3(0.0f, 0.0f, 0.0f);
var squeeze = stretched ? 1.0f / stage.stretch - 1.0f : 0.0f;
var norm = abs(stage.frequency) * (stretched ? max(1.0f, 1.0f / stage.stretch) : 1.0f);
var mq = stretched ? stage.frequency * (q + squeeze * dot(direction, q) * direction) : stage.frequency * q;
var mv = stretched ? stage.frequency * (path.velocity + squeeze * dot(direction, path.velocity) * direction) : stage.frequency * path.velocity;
var amplitude = 1.0f;
var scale = 1.0f;
var value = 0.0f;
var gradient = new float3(0.0f, 0.0f, 0.0f);
var curvature = 0.0f;
var fold = 0.0f;
for (var i = 0; i < octaves; i++)
{
var step = (float)i * stage.octaveStep;
var x = scale * (mq + stage.shift) + (stage.offset + step);
var n = snoise_grad(x);
var g = new float3(n.x, n.y, n.z);
var mg = stretched ? stage.frequency * (g + squeeze * dot(direction, g) * direction) : stage.frequency * g;
if (ridged)
{
var s = (float)sign(n.w);
value += amplitude * (1.0f - abs(n.w));
gradient += -amplitude * scale * s * mg;
}
else
{
value += amplitude * n.w;
gradient += amplitude * scale * mg;
}
var weight = abs(amplitude);
var factor = abs(scale) * norm;
var nu = factor * path.speed;
var rounding = span_rounding(scale, mq, stage, step, nu, halfLength);
var slack = span_slack(rounding);
var valueSlack = slack.x;
var gradientSlack = slack.y;
var hessianSlack = slack.z;
var local = min(SNOISE_HESSIAN, span_frobenius(snoise_hessian(x)) + hessianSlack + SNOISE_THIRD * nu * halfLength);
curvature += weight * factor * factor * local;
if (ridged)
{
var w = scale * mv;
var reach = local * nu * halfLength;
var gl = min(SNOISE_LIPSCHITZ, length(g) + gradientSlack + reach);
var travel = (abs(dot(g, w)) + gradientSlack * length(w)) * halfLength + (local * nu * length(w) + gl * factor * path.acceleration) * (0.5f * halfLength * halfLength) + 2.0f * valueSlack;
fold += 2.0f * weight * max(travel - abs(n.w), 0.0f);
}
scale *= lacunarity;
amplitude *= gain;
}
return new SpanNoise(new float4(gradient, value), curvature, fold);
}
}

37
packages/cultmath/src/CultMath/math.cs
View file

@ -317,18 +317,41 @@ public static partial class math
}
public static float distance_to_segment(float2 point, float2 start, float2 end, out float2 closest)
{
closest = segment_closest(point, start, end);
return distance(point, closest);
}
/// <summary>The distance from <paramref name="point"/> to the segment <paramref name="start"/>-<paramref name="end"/>; a degenerate segment is its start.</summary>
public static float distance_to_segment(float2 point, float2 start, float2 end) =>
distance(point, segment_closest(point, start, end));
/// <summary>The distance from <paramref name="point"/> to the segment <paramref name="start"/>-<paramref name="end"/>; a degenerate segment is its start.</summary>
public static float distance_to_segment(float3 point, float3 start, float3 end) =>
distance(point, segment_closest(point, start, end));
/// <summary>The point of the segment <paramref name="start"/>-<paramref name="end"/> nearest <paramref name="point_value"/>; a degenerate segment is its start.</summary>
public static float2 segment_closest(float2 point_value, float2 start, float2 end)
{
var segment = end - start;
var segmentLengthSq = lengthsq(segment);
if (segmentLengthSq <= 0.0f)
{
closest = start;
return distance(point, start);
}
return start;
var t = saturate(dot(point - start, segment) / segmentLengthSq);
closest = start + segment * t;
return distance(point, closest);
var t = saturate(dot(point_value - start, segment) / segmentLengthSq);
return start + segment * t;
}
/// <summary>The point of the segment <paramref name="start"/>-<paramref name="end"/> nearest <paramref name="point_value"/>; a degenerate segment is its start.</summary>
public static float3 segment_closest(float3 point_value, float3 start, float3 end)
{
var segment = end - start;
var segmentLengthSq = lengthsq(segment);
if (segmentLengthSq <= 0.0f)
return start;
var t = saturate(dot(point_value - start, segment) / segmentLengthSq);
return start + segment * t;
}
public static float catmullrom(float p0, float p1, float p2, float p3, float t)

28
packages/cultmath/tests/CultMath.Tests/AffineTests.cs
View file

@ -295,9 +295,35 @@ public sealed class AffineTests
// af_snoise's float32 allowance, as its doc comment states it, for the width comparison below.
private static double Allowance(float3 centre, double reach) =>
Math.ScaleB(1.0, -14) + Math.ScaleB(reach, -12)
SNOISE_READ_VALUE + SNOISE_READ_GRADIENT * reach
+ SNOISE_LIPSCHITZ * Math.ScaleB(Math.Abs(centre.x) + Math.Abs(centre.y) + Math.Abs(centre.z) + reach, -20);
/// <summary>
/// af_snoise reads the same absolute rounding terms of the kernel as span_slack: at the origin with no
/// axis, the form's error is span_slack's value allowance at zero rounding plus the Lipschitz form's
/// L R, and each unit of reach adds span_slack's gradient term at zero rounding plus L 2^-20 (the
/// coordinate's own rounding). Widening the kernel's absolute read in one reader and not the other,
/// the device judge's possible outcome, fails here. The expected bits are formed from span_slack, not
/// from a literal.
/// </summary>
[Fact]
public void SnoiseFormAndSpanSlackShareTheKernelsAbsoluteTerms()
{
var zero = new float3(0.0f, 0.0f, 0.0f);
var slack = span_slack(0.0f);
var value = snoise_grad(zero).w;
foreach (var reach in new[] { 0.0f, 1.0f, 3.0f })
{
var lipschitz = SNOISE_LIPSCHITZ * reach;
var allowance = slack.x + slack.y * reach + SNOISE_LIPSCHITZ * 9.5367431640625e-7f * reach;
var e = lipschitz + allowance;
var expected = e + (abs(value) + 0.0f + e) * 9.5367431640625e-7f;
var form = af_snoise(zero, zero, reach);
Assert.True(BitConverter.SingleToInt32Bits(form.z) == BitConverter.SingleToInt32Bits(expected),
$"af_snoise(0, 0, {reach:R}).z is {form.z:R}; the allowances span_slack reads give {expected:R}");
}
}
private static float3 UnitVector(System.Random random)
{
float3 v;

5
packages/cultmath/tests/CultMath.Tests/GlslLowering.cs
View file

@ -70,8 +70,9 @@ internal static class GlslLowering
var source = hlsl.ReplaceLineEndings("\n");
// 2. Include guards are dropped, and so are the functions taking Texture2D/SamplerState, exactly
// as the C# mirror drops them; the host shader samples its own textures.
source = Regex.Replace(source, @"(?m)^[ \t]*#(ifndef|define|endif)\b.*$", string.Empty);
// as the C# mirror drops them; the host shader samples its own textures. The capacity macro
// (#ifndef CULTMATH_MAX_SITES ... #endif) stays: GLSL has the preprocessor, and a consumer overrides it.
source = HlslSourceCompatibilityTests.DropGuards(source, (name, value) => $"#ifndef {name}\n#define {name} {value}\n#endif");
source = Regex.Replace(source, @"(?ms)^\w+ \w+\([^)]*\b(Texture2D|SamplerState)\b[^)]*\)\s*\{.*?^\}[ \t]*$", string.Empty);
// 3. Type names: floatN, intN, uintN and boolN become vecN, ivecN, uvecN and bvecN, and float3x3

701
packages/cultmath/tests/CultMath.Tests/GlslMirrorTests.Device.cs Normal file
View file

@ -0,0 +1,701 @@
using System.Globalization;
using System.Text.Json.Nodes;
using CultMath;
using Xunit;
using Xunit.Sdk;
namespace CultMath.Tests;
/// <summary>
/// The conforming run's C# half. <see cref="WriteWork"/> says what a device must compute: every fixture
/// family on the fixture's own arguments, plus one reference family for each sampled-form family, whose
/// points are the ones the form check evaluates. <see cref="JudgeDevice"/> says whether a device's results
/// hold: exact and integer families bit for bit, check families by their Verify, sampled-form families by
/// the form check on the device's own reference values, every other family within its recorded bound.
/// The generator, the Vulkan host and the job script only spell, dispatch and run; they compare nothing.
/// </summary>
public sealed partial class GlslMirrorTests
{
private static readonly string[] DeviceRoutes = { "lavapipe-hlsl", "lavapipe-glsl" };
private static string BoundsPath => Path.Combine(Root, "tests", "CultMath.Tests", "fixtures", "device-bounds-lavapipe.txt");
private static string? Env(string name) => Environment.GetEnvironmentVariable(name) is { Length: > 0 } value ? value : null;
private static JsonArray FixtureFamilies() => JsonNode.Parse(File.ReadAllText(FixturePath))!["functions"]!.AsArray();
private static string[] Sides(JsonNode? entry) => ((string)entry!).Split(" -> ");
private static uint[] DevWords(string hex) => hex.Split(' ').Select(h => Convert.ToUInt32(h, 16)).ToArray();
private static string DevHex(IEnumerable<uint> words) => string.Join(' ', words.Select(w => w.ToString("x8")));
private static uint[] ReadWords(string path)
{
var bytes = File.ReadAllBytes(path);
var words = new uint[bytes.Length / 4];
for (var i = 0; i < words.Length; i++)
words[i] = BitConverter.ToUInt32(bytes, i * 4);
return words;
}
private static void WriteWords(string path, IEnumerable<uint> words)
{
using var writer = new BinaryWriter(File.Create(path));
foreach (var word in words)
writer.Write(word);
}
private static bool IsSampledForm(Family family) =>
family.Check is not null && (family.Glsl.Contains("cultmath_af_snoise(") || family.Glsl.Contains("cultmath_af_fbm("));
private static string Tag(int family) => $"f{family:D3}";
private static string RefTag(int family) => $"r{family:D3}";
private sealed record WorkItem(string Tag, string Index, string Glsl, int N, int WordsIn, int WordsOut);
// ---- What a device must compute ----
private static List<WorkItem> WriteWork(string dir)
{
Directory.CreateDirectory(dir);
var fixture = FixtureFamilies();
Assert.Equal(Families.Length, fixture.Count);
var items = new List<WorkItem>();
for (var f = 0; f < Families.Length; f++)
{
Assert.Equal(Families[f].Glsl, (string?)fixture[f]!["glsl"]);
var cases = fixture[f]!["cases"]!.AsArray();
WriteWords(Path.Combine(dir, Tag(f) + ".in"), cases.SelectMany(c => DevWords(Sides(c)[0])));
items.Add(new WorkItem(Tag(f), f.ToString(CultureInfo.InvariantCulture), Families[f].Glsl, cases.Count, DevWords(Sides(cases[0])[0]).Length, DevWords(Sides(cases[0])[1]).Length));
if (IsSampledForm(Families[f]))
items.Add(WriteReference(dir, f, cases));
}
File.WriteAllLines(Path.Combine(dir, "manifest.txt"), items.Select(i => string.Join('\t', i.Tag, i.Index, i.Glsl, i.N, i.WordsIn, i.WordsOut)));
return items;
}
// The points the form check evaluates the reference at, in the order it evaluates them: the reference
// func records p, and for fBm the case's octaves, lacunarity and gain, then answers as C# does.
private static WorkItem WriteReference(string dir, int family, JsonArray cases)
{
var fbm = Families[family].Glsl.Contains("cultmath_af_fbm(");
var recorded = new List<uint>();
Func<string[], float3, float> recorder = (t, p) =>
{
recorded.AddRange(new[] { BitConverter.SingleToUInt32Bits(p.x), BitConverter.SingleToUInt32Bits(p.y), BitConverter.SingleToUInt32Bits(p.z) });
if (fbm)
recorded.AddRange(t[7..10].Select(h => Convert.ToUInt32(h, 16)));
return fbm ? FbmReference(t, p) : SnoiseReference(t, p);
};
if (fbm)
AssertFbmForm(cases, false, recorder);
else
AssertSnoiseForm(cases, false, recorder);
var wordsIn = fbm ? 6 : 3;
WriteWords(Path.Combine(dir, RefTag(family) + ".in"), recorded);
return new WorkItem(RefTag(family), "ref:" + family.ToString(CultureInfo.InvariantCulture),
fbm ? "vec4 cultmath_fbm_grad(vec3, int, float, float)" : "float cultmath_snoise(vec3)", recorded.Count / wordsIn, wordsIn, fbm ? 4 : 1);
}
[Fact]
public void WriteDeviceWork()
{
var dir = Env("CULTMATH_DEVICE_WORK");
Assert.SkipUnless(dir is not null, "CULTMATH_DEVICE_WORK=<dir> writes the work list a device runs");
var items = WriteWork(dir!);
output.WriteLine($"{items.Count} work items ({items.Count(i => i.Index.StartsWith("ref:"))} reference families) in {dir}");
}
// ---- Whether a device's results hold ----
private sealed record BoundRow(string Route, int Index, string Glsl, string Tolerance, int Exact, double Worst, int NanClass, string Bits);
private sealed record Measured(BoundRow Row, string Class);
private sealed record Judgement(List<string> Failures, List<Measured> Rows);
// Which words of a family's result are floats; the rest are integers and must match bit for bit.
private static IEnumerable<bool> FloatWords(Type type) =>
type == typeof(float) ? new[] { true }
: type == typeof(int) || type == typeof(uint) ? new[] { false }
: type == typeof(float3x3) ? Enumerable.Repeat(true, 9)
: type.GetFields().Where(f => !f.IsStatic).SelectMany(f => FloatWords(f.FieldType));
// u1 = |device - fixture| / max(|fixture|, 1) / 2^-23 per float component, worst over the family; NaN and
// Inf are compared as classes (NaN against not-NaN, an Inf against anything else), never by value.
private static (int Exact, double Worst, int NanClass) Measure(JsonArray cases, bool[] floatWord, uint[] got)
{
int exact = 0, nanClass = 0;
var worst = 0.0;
var wordsOut = floatWord.Length;
for (var c = 0; c < cases.Count; c++)
{
var want = DevWords(Sides(cases[c])[1]);
var same = true;
for (var w = 0; w < wordsOut; w++)
{
var have = got[c * wordsOut + w];
if (have == want[w])
continue;
same = false;
if (!floatWord[w])
{
nanClass++;
continue;
}
var e = BitConverter.UInt32BitsToSingle(want[w]);
var h = BitConverter.UInt32BitsToSingle(have);
if (float.IsNaN(e) || float.IsNaN(h))
nanClass += float.IsNaN(e) == float.IsNaN(h) ? 0 : 1;
else if (float.IsInfinity(e) || float.IsInfinity(h))
nanClass += e == h ? 0 : 1;
else
worst = Math.Max(worst, Math.Abs((double)e - h) / Math.Max(Math.Abs((double)e), 1.0) / Math.ScaleB(1.0, -23));
}
if (same)
exact++;
}
return (exact, worst, nanClass);
}
// The device's own reference values, in the order WriteReference recorded their points. The check asks
// for the reference at a point; the point it asks for must be the one the device was given at that place.
private static Func<string[], float3, float> DeviceReference(string work, string routeResults, WorkItem reference)
{
var points = ReadWords(Path.Combine(work, reference.Tag + ".in"));
var values = ReadWords(Path.Combine(routeResults, reference.Tag + ".out"));
Assert.Equal(reference.N * reference.WordsOut, values.Length);
var next = 0;
return (_, p) =>
{
Assert.True(next < reference.N, "reference point asked for past the recorded work");
var at = next * reference.WordsIn;
Assert.True(points[at] == BitConverter.SingleToUInt32Bits(p.x) && points[at + 1] == BitConverter.SingleToUInt32Bits(p.y) && points[at + 2] == BitConverter.SingleToUInt32Bits(p.z),
$"reference point {next} is not the point the device was given");
return BitConverter.UInt32BitsToSingle(values[next++ * reference.WordsOut + reference.WordsOut - 1]);
};
}
// sha256 of the family's result bytes, and of its reference result when it is judged on one.
private static string ResultBits(string path, string? reference)
{
var bytes = File.ReadAllBytes(path);
if (reference is not null)
bytes = bytes.Concat(File.ReadAllBytes(reference)).ToArray();
return Convert.ToHexString(System.Security.Cryptography.SHA256.HashData(bytes)).ToLowerInvariant();
}
// A tolerance text may state what a device is allowed: "on a device |diff| <= 2^-N max(|v|, 1)", which in
// u1 units (2^-23) is 2^(23-N). The judge reads it here and nowhere else.
private static double? StatedDeviceLimit(string tolerance)
{
var m = System.Text.RegularExpressions.Regex.Match(tolerance, @"on a device \|diff\| <= 2\^-(\d+) max\(\|v\|, 1\)");
return m.Success ? Math.ScaleB(1.0, 23 - int.Parse(m.Groups[1].Value)) : null;
}
private static string FirstLine(Exception e)
{
var line = e.Message.Split('\n')[0];
return line.Length > 300 ? line[..300] : line;
}
private static Judgement JudgeDevice(string work, string results, string renderer, string identity, string boundsPath, bool write)
{
var failures = new List<string>();
var measured = new List<Measured>();
Dictionary<(string, int), BoundRow>? recorded = null;
if (!write)
{
if (!File.Exists(boundsPath))
return new Judgement(new List<string> { "no recorded device bounds; record them with CULTMATH_WRITE_DEVICE_BOUNDS=1" }, measured);
var (header, rows) = ReadBounds(boundsPath);
if (header.GetValueOrDefault("renderer") != renderer || header.GetValueOrDefault("identity") != identity)
return new Judgement(new List<string> { "the device or its toolchain is not the one the bounds were recorded on; re-record them with CULTMATH_WRITE_DEVICE_BOUNDS=1" }, measured);
recorded = rows;
}
var fixture = FixtureFamilies();
var items = File.ReadAllLines(Path.Combine(work, "manifest.txt")).Select(l => l.Split('\t'))
.ToDictionary(c => c[1], c => new WorkItem(c[0], c[1], c[2], int.Parse(c[3]), int.Parse(c[4]), int.Parse(c[5])));
foreach (var route in DeviceRoutes)
for (var f = 0; f < Families.Length; f++)
{
var family = Families[f];
var cases = fixture[f]!["cases"]!.AsArray();
var name = $"{route} #{f} {family.Glsl}";
var path = Path.Combine(results, route, Tag(f) + ".out");
if (!File.Exists(path))
{
failures.Add($"{name}: missing {Tag(f)}.out");
continue;
}
var floatWord = FloatWords(Resolve(family.Glsl).ReturnType).ToArray();
var got = ReadWords(path);
if (got.Length != cases.Count * floatWord.Length)
{
failures.Add($"{name}: {Tag(f)}.out holds {got.Length} words, not {cases.Count * floatWord.Length}");
continue;
}
var (exact, worst, nanClass) = Measure(cases, floatWord, got);
var bits = ResultBits(path, IsSampledForm(family) ? Path.Combine(results, route, RefTag(f) + ".out") : null);
var exactClass = family.Tolerance == "exact" || !floatWord.Any(w => w);
var kind = exactClass ? "exact" : IsSampledForm(family) ? "sampled" : family.Verify is not null ? "check" : "bound";
measured.Add(new Measured(new BoundRow(route, f, family.Glsl, family.Tolerance, exact, worst, nanClass, bits), kind));
try
{
if (exactClass)
Assert.True(exact == cases.Count, $"{exact} of {cases.Count} cases are bit-identical to the fixture");
else if (kind == "sampled")
{
var reference = DeviceReference(work, Path.Combine(results, route), items["ref:" + f]);
if (family.Glsl.Contains("cultmath_af_fbm("))
AssertFbmForm(cases, false, reference);
else
AssertSnoiseForm(cases, false, reference);
}
else if (kind == "check")
family.Verify!(new JsonArray(cases.Select((c, i) => (JsonNode)JsonValue.Create(Sides(c)[0] + " -> " + DevHex(got.Skip(i * floatWord.Length).Take(floatWord.Length)))!).ToArray()));
}
catch (XunitException e)
{
failures.Add($"{name}: {FirstLine(e)}");
}
}
if (write)
{
WriteBounds(boundsPath, renderer, identity, measured.Select(m => m.Row));
recorded = measured.ToDictionary(m => (m.Row.Route, m.Row.Index), m => m.Row);
}
foreach (var m in measured)
{
var name = $"{m.Row.Route} #{m.Row.Index} {m.Row.Glsl}";
if (!recorded!.TryGetValue((m.Row.Route, m.Row.Index), out var bound))
failures.Add($"{name}: no recorded row; re-record with CULTMATH_WRITE_DEVICE_BOUNDS=1");
else if (bound.Glsl != m.Row.Glsl || bound.Tolerance != m.Row.Tolerance)
failures.Add($"{name}: the recorded row is for another family; re-record with CULTMATH_WRITE_DEVICE_BOUNDS=1");
else if (m.Class == "bound" && (m.Row.Worst > bound.Worst || m.Row.NanClass > bound.NanClass))
failures.Add($"{name}: worst u1 {m.Row.Worst:R} and {m.Row.NanClass} NaN-class mismatches, past the recorded {bound.Worst:R} and {bound.NanClass}");
else if (m.Class == "bound" && StatedDeviceLimit(m.Row.Tolerance) is { } limit && m.Row.Worst > limit)
failures.Add($"{name}: worst u1 {m.Row.Worst:R}, past the {limit:R} its stated tolerance allows a device");
else if (m.Row.Bits != bound.Bits && !failures.Any(x => x.StartsWith(name + ":")))
failures.Add($"{name}: the result bits differ from the recorded run's; if the change is meant, re-record with CULTMATH_WRITE_DEVICE_BOUNDS=1");
}
return new Judgement(failures, measured);
}
// ---- The recorded bounds: plain text, tab-separated, one row per route and family ----
private static void WriteBounds(string path, string renderer, string identity, IEnumerable<BoundRow> rows)
{
var lines = new List<string>
{
"# renderer\t" + renderer,
"# identity\t" + identity,
"# route\tindex\tglsl\ttolerance\texact cases\tworst u1\tNaN-class mismatches result sha256; written only by CULTMATH_WRITE_DEVICE_BOUNDS=1",
};
lines.AddRange(rows.OrderBy(r => r.Route, StringComparer.Ordinal).ThenBy(r => r.Index)
.Select(r => string.Join('\t', r.Route, r.Index, r.Glsl, r.Tolerance, r.Exact, r.Worst.ToString("R", CultureInfo.InvariantCulture), r.NanClass, r.Bits)));
File.WriteAllText(path, string.Join('\n', lines) + "\n");
}
private static (Dictionary<string, string> Header, Dictionary<(string, int), BoundRow> Rows) ReadBounds(string path)
{
var header = new Dictionary<string, string>();
var rows = new Dictionary<(string, int), BoundRow>();
foreach (var line in File.ReadAllLines(path))
{
var c = line.Split('\t');
if (line.StartsWith("# renderer\t") || line.StartsWith("# identity\t"))
header[c[0][2..]] = c[1];
else if (!line.StartsWith('#') && line.Length > 0)
rows[(c[0], int.Parse(c[1]))] = new BoundRow(c[0], int.Parse(c[1]), c[2], c[3], int.Parse(c[4]), double.Parse(c[5], CultureInfo.InvariantCulture), int.Parse(c[6]), c[7]);
}
return (header, rows);
}
[Fact]
public void DeviceResultsHoldFixture()
{
var results = Env("CULTMATH_DEVICE_RESULTS");
Assert.SkipUnless(results is not null, "CULTMATH_DEVICE_RESULTS=<dir> (holding <route>/<tag>.out) judges a device's results");
var work = Env("CULTMATH_DEVICE_WORK");
var renderer = Env("CULTMATH_DEVICE_NAME");
var identity = Env("CULTMATH_DEVICE_IDENTITY");
Assert.True(work is not null && renderer is not null && identity is not null, "CULTMATH_DEVICE_WORK, CULTMATH_DEVICE_NAME and CULTMATH_DEVICE_IDENTITY are required beside CULTMATH_DEVICE_RESULTS");
var judgement = JudgeDevice(work!, results!, renderer!, identity!, BoundsPath, Env("CULTMATH_WRITE_DEVICE_BOUNDS") == "1");
foreach (var route in DeviceRoutes)
{
var rows = judgement.Rows.Where(m => m.Row.Route == route).ToList();
output.WriteLine($"{route}: {rows.Count} families judged: " + string.Join(", ", rows.GroupBy(m => m.Class).OrderBy(g => g.Key).Select(g => $"{g.Count()} {g.Key}")));
foreach (var m in rows.OrderByDescending(m => m.Row.Worst).Take(5))
output.WriteLine($" worst u1 {m.Row.Worst:R}, {m.Row.NanClass} NaN-class: #{m.Row.Index} {m.Row.Glsl} ({m.Class})");
}
foreach (var failure in judgement.Failures.Take(60))
output.WriteLine("FAIL " + failure);
Assert.True(judgement.Failures.Count == 0, $"{judgement.Failures.Count} device results do not hold the fixture; first: {judgement.Failures.FirstOrDefault()}");
}
// ---- The judge against simulated devices ----
// A faithful device answers the fixture's own result bits and, for a reference family, C#'s snoise or
// fBm at the recorded points. A test alters what one family or one route answers.
private sealed record Simulated(string Root, string Work, string Results, string Bounds) : IDisposable
{
public void Dispose() => Directory.Delete(Root, true);
}
private static Simulated Simulate(Func<string, WorkItem, uint[], uint[]>? alter = null)
{
var root = Path.Combine(Path.GetTempPath(), "cultmath-device-" + Guid.NewGuid().ToString("N"));
var work = Path.Combine(root, "work");
var items = WriteWork(work);
var fixture = FixtureFamilies();
foreach (var route in DeviceRoutes)
{
Directory.CreateDirectory(Path.Combine(root, "results", route));
foreach (var item in items)
{
uint[] words;
if (item.Index.StartsWith("ref:"))
{
var points = ReadWords(Path.Combine(work, item.Tag + ".in"));
words = new uint[item.N * item.WordsOut];
for (var k = 0; k < item.N; k++)
{
var at = k * item.WordsIn;
var p = new float3(BitConverter.UInt32BitsToSingle(points[at]), BitConverter.UInt32BitsToSingle(points[at + 1]), BitConverter.UInt32BitsToSingle(points[at + 2]));
var tokens = new string[10];
if (item.WordsIn == 6)
for (var t = 0; t < 3; t++)
tokens[7 + t] = points[at + 3 + t].ToString("x8");
words[k * item.WordsOut + item.WordsOut - 1] = BitConverter.SingleToUInt32Bits(item.WordsIn == 6 ? FbmReference(tokens, p) : SnoiseReference(tokens, p));
}
}
else
words = fixture[int.Parse(item.Index)]!["cases"]!.AsArray().SelectMany(c => DevWords(Sides(c)[1])).ToArray();
WriteWords(Path.Combine(root, "results", route, item.Tag + ".out"), alter?.Invoke(route, item, words) ?? words);
}
}
return new Simulated(root, work, Path.Combine(root, "results"), Path.Combine(root, "bounds.txt"));
}
private static Judgement Record(Simulated device) => JudgeDevice(device.Work, device.Results, "test renderer", "test identity", device.Bounds, true);
private static Judgement Judge(Simulated device, Simulated recordedOn, string renderer = "test renderer", string identity = "test identity") =>
JudgeDevice(device.Work, device.Results, renderer, identity, recordedOn.Bounds, false);
// Every case's first result word, altered.
private static uint[] Alter(WorkItem item, uint[] words, Func<uint, uint> change, int cases = int.MaxValue)
{
var altered = (uint[])words.Clone();
for (var c = 0; c < Math.Min(item.N, cases); c++)
altered[c * item.WordsOut] = change(altered[c * item.WordsOut]);
return altered;
}
// A family judged by its recorded bound alone: not exact, not checked, not a sampled form, a float first word.
private static int BoundFamily() =>
Array.FindIndex(Families, f => f.Tolerance == "ulp-bounded" && f.Verify is null && FloatWords(Resolve(f.Glsl).ReturnType).First());
[Fact]
public void AFaithfulDeviceHoldsEveryFamilyOnBothRoutes()
{
using var device = Simulate();
var recorded = Record(device);
Assert.Empty(recorded.Failures);
Assert.Equal(DeviceRoutes.Length * Families.Length, recorded.Rows.Count);
Assert.All(recorded.Rows, m => Assert.Equal(0.0, m.Row.Worst));
Assert.Equal(DeviceRoutes.Length * Families.Count(IsSampledForm), recorded.Rows.Count(m => m.Class == "sampled"));
Assert.Equal(DeviceRoutes.Length * Families.Count(f => f.Tolerance == "exact"), recorded.Rows.Count(m => m.Class == "exact"));
Assert.Empty(Judge(device, device).Failures);
}
[Fact]
public void AnExactFamilyAllowsNoDifferenceAtAll()
{
// Bounds recorded on the very device that is off by one ulp: a bound would pass it, the exact rule must not.
var exact = Array.FindIndex(Families, f => f.Tolerance == "exact");
using var off = Simulate((route, item, words) => route == "lavapipe-glsl" && item.Index == exact.ToString() ? Alter(item, words, w => w ^ 1u) : words);
Record(off);
var failures = Judge(off, off).Failures;
Assert.Single(failures);
Assert.Contains($"lavapipe-glsl #{exact} ", failures[0]);
// One case in 256 is enough.
using var oneCase = Simulate((_, item, words) => item.Index == exact.ToString() ? Alter(item, words, w => w ^ 1u, 1) : words);
Record(oneCase);
Assert.Equal(DeviceRoutes.Length, Judge(oneCase, oneCase).Failures.Count);
}
[Fact]
public void AFamilyIsHeldToItsRecordedWorstAndNotBeyond()
{
var f = BoundFamily();
using var near = Simulate((_, item, words) => item.Index == f.ToString() ? Alter(item, words, w => w ^ 4u) : words);
Record(near);
Assert.Empty(Judge(near, near).Failures);
using var farther = Simulate((_, item, words) => item.Index == f.ToString() ? Alter(item, words, w => w ^ 7u) : words);
var failures = Judge(farther, near).Failures;
Assert.Equal(DeviceRoutes.Length, failures.Count);
Assert.All(failures, line => Assert.Contains($"#{f} ", line));
var lines = File.ReadAllLines(near.Bounds).ToList();
var at = lines.FindIndex(l => l.StartsWith($"lavapipe-hlsl\t{f}\t"));
var row = lines[at].Split('\t');
Assert.NotEqual(0.0, double.Parse(row[5], CultureInfo.InvariantCulture));
row[5] = "0";
lines[at] = string.Join('\t', row);
File.WriteAllLines(near.Bounds, lines);
Assert.Single(Judge(near, near).Failures);
lines.RemoveAt(at);
File.WriteAllLines(near.Bounds, lines);
var missing = Judge(near, near).Failures;
Assert.Single(missing);
Assert.Contains("no recorded row", missing[0]);
}
[Fact]
public void ABoundMeasuresTheErrorAgainstTheFixturesMagnitude()
{
// The metric divides by max(|fixture|, 1): a result of twice a fixture value of at least 2 is 2^23 away,
// and of twice a fixture value under 1 is |fixture| 2^23 away.
var fixture = FixtureFamilies();
float Fixture(int family, int c) => BitConverter.UInt32BitsToSingle(DevWords(Sides(fixture[family]!["cases"]![c])[1])[0]);
int Find(int family, Func<float, bool> wanted) => Enumerable.Range(0, Points).FirstOrDefault(c => wanted(Math.Abs(Fixture(family, c))), -1);
bool Large(float v) => v is >= 2.0f and < 1e30f;
bool Small(float v) => v is > 1e-3f and < 1.0f;
var f = Enumerable.Range(0, Families.Length).First(g => Families[g].Tolerance == "ulp-bounded" && Families[g].Verify is null
&& FloatWords(Resolve(Families[g].Glsl).ReturnType).First() && Find(g, Large) >= 0 && Find(g, Small) >= 0);
var large = Find(f, Large);
var small = Find(f, Small);
foreach (var (c, expected) in new[] { (large, Math.ScaleB(1.0, 23)), (small, Math.Abs((double)Fixture(f, small)) * Math.ScaleB(1.0, 23)) })
{
using var device = Simulate((_, item, words) =>
{
if (item.Index != f.ToString())
return words;
var doubled = (uint[])words.Clone();
doubled[c * item.WordsOut] = BitConverter.SingleToUInt32Bits(2.0f * BitConverter.UInt32BitsToSingle(words[c * item.WordsOut]));
return doubled;
});
Assert.Equal(expected, Record(device).Rows.First(m => m.Row.Route == "lavapipe-hlsl" && m.Row.Index == f).Row.Worst);
}
}
[Fact]
public void ANaNWhereTheFixtureHasANumberFailsAFamilyRecordedWithout()
{
var f = BoundFamily();
using var faithful = Simulate();
Record(faithful);
using var nan = Simulate((_, item, words) => item.Index == f.ToString() ? Alter(item, words, _ => 0x7fc00000u) : words);
var failures = Judge(nan, faithful).Failures;
Assert.Equal(DeviceRoutes.Length, failures.Count);
Assert.All(failures, line => Assert.Contains("NaN-class", line));
}
[Fact]
public void AMissingOutputFailsOnlyItsFamily()
{
using var device = Simulate();
Record(device);
File.Delete(Path.Combine(device.Results, "lavapipe-hlsl", Tag(3) + ".out"));
var failures = Judge(device, device).Failures;
Assert.Single(failures);
Assert.Contains("missing", failures[0]);
Assert.Contains("lavapipe-hlsl #3 ", failures[0]);
}
[Fact]
public void ADeviceOrToolchainOtherThanTheRecordedOneIsRefusedBeforeJudging()
{
using var device = Simulate();
Record(device);
foreach (var (renderer, identity) in new[] { ("another renderer", "test identity"), ("test renderer", "another toolchain") })
{
var judgement = Judge(device, device, renderer, identity);
Assert.Single(judgement.Failures);
Assert.Empty(judgement.Rows);
}
}
[Theory]
[InlineData("cultmath_af_snoise(")]
[InlineData("cultmath_af_fbm(")]
public void ASampledFormIsJudgedOnTheDevicesOwnReferenceAtTheRecordedPoints(string function)
{
var sampled = Array.FindIndex(Families, f => IsSampledForm(f) && f.Glsl.Contains(function));
using var faithful = Simulate();
Record(faithful);
// A reference that answers wrongly puts the form's enclosure out of reach.
using var wrong = Simulate((_, item, words) => item.Index == "ref:" + sampled
? words.Select(w => BitConverter.SingleToUInt32Bits(BitConverter.UInt32BitsToSingle(w) + 100.0f)).ToArray() : words);
var failures = Judge(wrong, faithful).Failures;
Assert.Equal(DeviceRoutes.Length, failures.Count);
Assert.All(failures, line => Assert.Contains($"#{sampled} ", line));
// Two points in the other order: the device's i-th answer is not for the point the check asks about.
using var swapped = Simulate();
var path = Path.Combine(swapped.Work, RefTag(sampled) + ".in");
var stride = function.Contains("fbm") ? 6 : 3;
var points = ReadWords(path);
(points[0], points[stride]) = (points[stride], points[0]);
WriteWords(path, points);
var order = Judge(swapped, faithful).Failures;
Assert.Equal(DeviceRoutes.Length, order.Count);
Assert.All(order, line => Assert.Contains("not the point the device was given", line));
}
[Fact]
public void AnInfinityWhereTheFixtureHasAnotherValueIsAMismatchOfClass()
{
var fixture = FixtureFamilies();
int f = -1, c = 0, w = 0;
for (var g = 0; g < Families.Length && f < 0; g++)
{
if (Families[g].Tolerance == "exact" || Families[g].Verify is not null)
continue;
var floatWord = FloatWords(Resolve(Families[g].Glsl).ReturnType).ToArray();
for (var k = 0; k < Points && f < 0; k++)
{
var want = DevWords(Sides(fixture[g]!["cases"]![k])[1]);
for (var at = 0; at < want.Length && f < 0; at++)
if (floatWord[at] && float.IsInfinity(BitConverter.UInt32BitsToSingle(want[at])))
(f, c, w) = (g, k, at);
}
}
Assert.True(f >= 0, "no fixture family holds an infinite result");
using var faithful = Simulate();
Record(faithful);
using var finite = Simulate((_, item, words) =>
{
if (item.Index != f.ToString())
return words;
var altered = (uint[])words.Clone();
altered[c * item.WordsOut + w] = BitConverter.SingleToUInt32Bits(1.0f);
return altered;
});
var failures = Judge(finite, faithful).Failures;
Assert.Equal(DeviceRoutes.Length, failures.Count);
Assert.All(failures, line => Assert.Contains("NaN-class", line));
}
[Fact]
public void ARecordedRowForAnotherFamilyIsRefused()
{
using var device = Simulate();
Record(device);
var lines = File.ReadAllLines(device.Bounds).ToList();
var at = lines.FindIndex(l => l.StartsWith("lavapipe-glsl 5 "));
var row = lines[at].Split(' ');
row[2] = "float cultmath_other(float)";
lines[at] = string.Join(' ', row);
File.WriteAllLines(device.Bounds, lines);
var failures = Judge(device, device).Failures;
Assert.Single(failures);
Assert.Contains("another family", failures[0]);
}
[Fact]
public void ACheckFamilyIsJudgedByItsVerify()
{
var f = Array.FindIndex(Families, fam => fam.Verify is not null && !IsSampledForm(fam));
using var faithful = Simulate();
Record(faithful);
using var broken = Simulate((_, item, words) => item.Index == f.ToString() ? words.Select(_ => 0x7fc00000u).ToArray() : words);
var failures = Judge(broken, faithful).Failures;
Assert.Equal(DeviceRoutes.Length, failures.Count);
Assert.All(failures, line => Assert.Contains($"#{f} ", line));
}
[Fact]
public void AChangeInResultBitsFailsEvenWithinTheRecordedBound()
{
// Recorded on a device off in every case; judged on one off in a single case. The bound holds, the bits do not.
var f = BoundFamily();
using var recorded = Simulate((_, item, words) => item.Index == f.ToString() ? Alter(item, words, w => w ^ 4u) : words);
Record(recorded);
using var fewer = Simulate((_, item, words) => item.Index == f.ToString() ? Alter(item, words, w => w ^ 4u, 1) : words);
var failures = Judge(fewer, recorded).Failures;
Assert.Equal(DeviceRoutes.Length, failures.Count);
Assert.All(failures, line => { Assert.Contains($"#{f} ", line); Assert.Contains("result bits differ", line); });
}
[Fact]
public void AFamilyIsHeldToTheDeviceToleranceItsOwnTextStates()
{
// Recorded on the very device that violates it: only the stated tolerance can fail the family.
var f = Array.FindIndex(Families, fam => StatedDeviceLimit(fam.Tolerance) is not null);
Assert.True(f >= 0, "no family states a device tolerance");
using var far = Simulate((_, item, words) => item.Index == f.ToString() ? Alter(item, words, w => w ^ 0x00100000u) : words);
Record(far);
var failures = Judge(far, far).Failures;
Assert.Equal(DeviceRoutes.Length, failures.Count);
Assert.All(failures, line => Assert.Contains("stated tolerance", line));
}
[Theory]
[InlineData(24, true)]
[InlineData(40, false)]
public void TheStatedDeviceToleranceIsTheLimitInUlpsOfTheFixturesMagnitude(int ulps, bool holds)
{
// 2^-18 max(|v|, 1) is 32 u1. A fixture value in [1, 1.1) moved by k ulps is k/|v| u1 away, 0.91 k to k.
var f = Array.FindIndex(Families, fam => StatedDeviceLimit(fam.Tolerance) is not null);
Assert.Equal(32.0, StatedDeviceLimit(Families[f].Tolerance));
var fixture = FixtureFamilies()[f]!["cases"]!.AsArray();
var floatWord = FloatWords(Resolve(Families[f].Glsl).ReturnType).ToArray();
var at = -1;
for (var c = 0; c < fixture.Count && at < 0; c++)
{
var want = DevWords(Sides(fixture[c])[1]);
for (var w = 0; w < want.Length && at < 0; w++)
if (floatWord[w] && Math.Abs(BitConverter.UInt32BitsToSingle(want[w])) is >= 1.0f and < 1.1f)
at = c * want.Length + w;
}
Assert.True(at >= 0, "no fixture value of the family lies in [1, 1.1)");
using var device = Simulate((_, item, words) =>
{
if (item.Index != f.ToString())
return words;
var moved = (uint[])words.Clone();
moved[at] += (uint)ulps; // magnitude grows by k ulps whatever the sign
return moved;
});
Record(device);
var failures = Judge(device, device).Failures;
Assert.Equal(holds ? 0 : DeviceRoutes.Length, failures.Count);
}
[Fact]
public void AChangeInTheReferenceBitsOfASampledFormFailsItsFamily()
{
// One ulp in one reference value stays inside the form's enclosure, so only the recorded bits can see it.
var sampled = Array.FindIndex(Families, IsSampledForm);
using var faithful = Simulate();
Record(faithful);
using var moved = Simulate((_, item, words) => item.Index == "ref:" + sampled ? Alter(item, words, w => w + 1u, 1) : words);
var failures = Judge(moved, faithful).Failures;
Assert.Equal(DeviceRoutes.Length, failures.Count);
Assert.All(failures, line => { Assert.Contains($"#{sampled} ", line); Assert.Contains("result bits differ", line); });
}
}

60
packages/cultmath/tests/CultMath.Tests/GlslMirrorTests.Mutants.cs Normal file
View file

@ -0,0 +1,60 @@
using System.Reflection;
using System.Text.RegularExpressions;
using Xunit;
namespace CultMath.Tests;
public sealed partial class GlslMirrorTests
{
private const string Weighted = "cultmath_weighted_lattice_cells(";
private const string SmoothStep = "cultmath_lattice_smooth_step(";
// Edits of the shader text that each break one rule of the lattice loop. M7 (the exit one ring late) is absent: by the
// window-independence proof a later exit visits more cells and changes nothing, so no case can kill it.
private static readonly (string Name, string Function, string Old, string New, int Kills)[] LatticeMutants =
{
("M1 exit one ring early", Weighted, "0.5 * (float)(k - 1) >= max(f2", "0.5 * (float)k >= max(f2", 3),
("M2 exit ignores the smoothing band", Weighted, ">= max(f2, f1 + 2.0 * smoothing))", ">= f2)", 1),
("M3 no pre-normalisation of the normal", Weighted, "normal = stretchNormal / scale;", "normal = stretchNormal;", 1),
("M4 no unit length for the normal", Weighted, "normal = normal / length(normal);", "normal = normal * 1.0;", 1),
("M5 smooth step without its clamp", SmoothStep, "return min(next, f1 + smoothing);", "return next;", 1),
("M6 box pass skips its first row", Weighted, "for (int dz = -reach; dz <= reach; dz++)", "for (int dz = -reach + 1; dz <= reach; dz++)", 1),
};
/// <summary>
/// Each rule of the lattice loop is pinned by the committed fixture's own cases: the shader text with one rule broken (a
/// mutant mirror, the text a device runs) returns another result than the fixture on at least the stated number of
/// cases of the family that exercises it, the exit one ring early on at least three. The device judge holds the same
/// cases to their recorded bound, so a mutant these cases separate fails it too.
/// </summary>
[Fact]
public void TheFixtureCasesSeparateEveryLatticeMutantFromTheShader()
{
var families = FixtureFamilies();
foreach (var (name, function, old, replacement, kills) in LatticeMutants)
{
var (assembly, errors) = HlslSourceCompatibilityTests.CompileShaderMirror(mutate: text =>
{
Assert.True(text.Split(old).Length == 2, $"{name}: the shader text no longer holds exactly one \"{old}\"");
return text.Replace(old, replacement);
});
Assert.True(assembly is not null, name + ": " + string.Join(Environment.NewLine, errors));
var shaderType = assembly!.GetType("CultMathHlsl.HlslShader")!;
var shader = Activator.CreateInstance(shaderType);
var mirror = shaderType.GetMethod(function[..^1], BindingFlags.Instance | BindingFlags.NonPublic)!;
var f = Array.FindIndex(Families, family => family.Glsl.Contains(function));
var method = Resolve(Families[f].Glsl);
var killed = 0;
foreach (var entry in families[f]!["cases"]!.AsArray())
{
var sides = Sides(entry);
if (Hex(new[] { mirror.Invoke(shader, Arguments(method, sides[0]))! }) != sides[1])
killed++;
}
output.WriteLine($"{name}: separated on {killed} of {Points} cases of {Families[f].Glsl}");
Assert.True(killed >= kills, $"{name}: the {Families[f].Glsl} fixture cases separate it on {killed} of {Points}, fewer than {kills}");
}
}
}

57
packages/cultmath/tests/CultMath.Tests/GlslMirrorTests.Precision.cs Normal file
View file

@ -0,0 +1,57 @@
using System.Text.RegularExpressions;
using Xunit;
namespace CultMath.Tests;
public sealed partial class GlslMirrorTests
{
// Tripwire: lavapipe and desktop ANGLE ignore mediump and min16float, so a reduced-precision qualifier in the
// committed shaders would change the computation on a phone GPU and nowhere the conforming run can see it.
// Stated limits: a syntax guardrail only. It reads whole words in the committed .glsl and .hlsl under shaders/
// with comments removed; it does not see a qualifier spelled through a macro, an include or a generated
// shader, and it says nothing about a driver that relaxes highp itself. It names file and line, never text.
// Recorded limit (s2.tripwire-slashslash-star-hides): block comments are cut before line comments, so a qualifier between `// x /*` and `// */` passes.
private static readonly Regex ReducedPrecision = new(
@"\b(mediump|lowp|min(10|12|16)(float|int|uint)[1-4]?(x[1-4])?|half[1-4]?(x[1-4])?)\b|\bprecision\s+(?!highp\b)\w+", RegexOptions.Compiled);
private static IEnumerable<int> ReducedPrecisionLines(string source)
{
var text = Regex.Replace(source, @"/\*.*?\*/", m => new string('\n', m.Value.Count(c => c == '\n')), RegexOptions.Singleline);
var lines = text.Split('\n');
for (var i = 0; i < lines.Length; i++)
{
var code = lines[i].Split("//")[0];
if (ReducedPrecision.IsMatch(code))
yield return i + 1;
}
}
[Fact]
public void CommittedShadersCarryNoReducedPrecisionQualifier()
{
var found = new List<string>();
foreach (var file in Directory.EnumerateFiles(Path.Combine(Root, "shaders")).Where(f => f.EndsWith(".glsl") || f.EndsWith(".hlsl")).OrderBy(f => f, StringComparer.Ordinal))
found.AddRange(ReducedPrecisionLines(File.ReadAllText(file)).Select(line => $"{Path.GetFileName(file)}:{line}"));
Assert.True(found.Count == 0, "reduced-precision qualifier at " + string.Join(", ", found));
}
[Theory]
[InlineData("mediump float x;", true)]
[InlineData("lowp vec3 c;", true)]
[InlineData("precision mediump float;", true)]
[InlineData("precision highp float;", false)]
[InlineData("min16float x;", true)]
[InlineData("min10float2 x;", true)]
[InlineData("min16int i;", true)]
[InlineData("half3x3 m;", true)]
[InlineData("float half_width;", false)]
[InlineData("float x; // mediump is not used", false)]
[InlineData("/* half\n min16float */ float x;", false)]
public void TheTripwireReadsQualifiersAndNotComments(string source, bool trips) => Assert.Equal(trips, ReducedPrecisionLines(source).Any());
[Fact]
public void TheTripwireNamesTheLine()
{
Assert.Equal(new[] { 3 }, ReducedPrecisionLines("float a;\n/* c */\nmediump float b;\n").ToArray());
}
}

352
packages/cultmath/tests/CultMath.Tests/GlslMirrorTests.cs
View file

@ -16,13 +16,17 @@ namespace CultMath.Tests;
/// CULTMATH_WRITE_GLSL=1 set, the first and last tests write the committed files before comparing, which
/// is the one regeneration path.
/// </summary>
public sealed class GlslMirrorTests
public sealed partial class GlslMirrorTests
{
private readonly ITestOutputHelper output;
public GlslMirrorTests(ITestOutputHelper output) => this.output = output;
private static bool Write => Environment.GetEnvironmentVariable("CULTMATH_WRITE_GLSL") == "1";
// The fixture is written by Yggdrasil's mcr.microsoft.com/dotnet/sdk:10.0 image (Ubuntu 24.04) only: its platform families record
// that platform's libm, so a write from another OS would move them.
private static bool Write => Environment.GetEnvironmentVariable("CULTMATH_WRITE_GLSL") == "1"
? OperatingSystem.IsLinux() ? true : throw new InvalidOperationException("CULTMATH_WRITE_GLSL=1 writes glsl-parity.json, whose platform families belong to Yggdrasil's Ubuntu image (mcr.microsoft.com/dotnet/sdk:10.0): regenerate it there.")
: false;
private static string Root => HlslSourceCompatibilityTests.FindCultMathRoot();
@ -45,6 +49,54 @@ public sealed class GlslMirrorTests
}
}
// Counting stand-ins for the mirror's clamp, one per overload, declared in the mirror's class so they
// hide math.clamp for every function in it. Each counts a call whose minimum exceeds its maximum in any
// component and forwards to math.clamp. (The int vector overloads only forward: no mirror function
// clamps integer vectors with swapped bounds, and counting them would be unreachable.)
private const string CountingClampShadows = @"
public static int ClampMinAboveMax;
static float clamp(float x, float a, float b) { if (a > b) ClampMinAboveMax++; return math.clamp(x, a, b); }
static float2 clamp(float2 x, float2 a, float2 b) { if (a.x > b.x || a.y > b.y) ClampMinAboveMax++; return math.clamp(x, a, b); }
static float3 clamp(float3 x, float3 a, float3 b) { if (a.x > b.x || a.y > b.y || a.z > b.z) ClampMinAboveMax++; return math.clamp(x, a, b); }
static float4 clamp(float4 x, float4 a, float4 b) { if (a.x > b.x || a.y > b.y || a.z > b.z || a.w > b.w) ClampMinAboveMax++; return math.clamp(x, a, b); }
static int clamp(int x, int a, int b) { if (a > b) ClampMinAboveMax++; return math.clamp(x, a, b); }
static int2 clamp(int2 x, int2 a, int2 b) { if (a.x > b.x || a.y > b.y) ClampMinAboveMax++; return math.clamp(x, a, b); }
static int3 clamp(int3 x, int3 a, int3 b) { if (a.x > b.x || a.y > b.y || a.z > b.z) ClampMinAboveMax++; return math.clamp(x, a, b); }
static int4 clamp(int4 x, int4 a, int4 b) { if (a.x > b.x || a.y > b.y || a.z > b.z || a.w > b.w) ClampMinAboveMax++; return math.clamp(x, a, b); }
";
/// <summary>
/// The lowering keeps HLSL's statement order (CommittedGlslEqualsLowering pins the GLSL as that
/// lowering), and GLSL leaves clamp(x, minVal, maxVal) undefined for minVal above maxVal, so no clamp
/// reached by cultmath_integral_clamp_linear may see lo above hi, on any path and for any spelling of the
/// normalisation. The mirror is compiled with counting clamps and the function is run over the edge grid
/// (lo above hi, infinite bounds, zero-length spans, b = 0 and b != 0): the count of such clamp calls
/// stays 0 after every call. The cases are HlslSourceCompatibilityTests.IntegralClampLinearCases: the grid fixes
/// a in {-2.5, 0, 1.5}, so a normalisation keyed on a past 1.75 never clamps wrongly there; the committed
/// fixture's random cases reach it.
/// </summary>
[Fact]
public void IntegralClampLinearNeverClampsWithMinAboveMax()
{
var (assembly, errors) = HlslSourceCompatibilityTests.CompileShaderMirror(CountingClampShadows);
Assert.True(assembly is not null, string.Join(Environment.NewLine, errors));
var shaderType = assembly!.GetType("CultMathHlsl.HlslShader")!;
var shader = Activator.CreateInstance(shaderType);
var count = shaderType.GetField("ClampMinAboveMax")!;
var function = shaderType.GetMethod("cultmath_integral_clamp_linear", BindingFlags.Instance | BindingFlags.NonPublic)!;
var bad = new List<string>();
foreach (var c in HlslSourceCompatibilityTests.IntegralClampLinearCases)
{
count.SetValue(null, 0);
function.Invoke(shader, c.Cast<object>().ToArray());
if ((int)count.GetValue(null)! > 0)
bad.Add($"({string.Join(", ", c)})");
}
Assert.True(bad.Count == 0, $"{bad.Count} cases clamp with the minimum above the maximum:{Environment.NewLine}{string.Join(Environment.NewLine, bad.Take(12))}");
}
// Identifier boundaries: cultmath_lerp is a CultMath function, lerp( is HLSL.
private static readonly Regex HlslOnly = new(
@"\b(float|u?int|bool)[234]\b|\bfloat3x3\b|\blerp\(|\bfrac\(|\bsaturate\(|\brsqrt\(|\basuint\(|\basfloat\(|\batan2\(|static const|\((float|int|uint)\)|Texture2D|SamplerState|#include|precision (high|medium|low)p");
@ -288,7 +340,7 @@ public sealed class GlslMirrorTests
// The float32 allowance of one octave of af_snoise: 2^-14 + 2^-12 R + L 2^-20 (|c|_1 + R).
private static double SnoiseAllowance(double centre1, double reach) =>
Math.ScaleB(1.0, -14) + Math.ScaleB(reach, -12) + math.SNOISE_LIPSCHITZ * Math.ScaleB(centre1 + reach, -20);
math.SNOISE_READ_VALUE + math.SNOISE_READ_GRADIENT * reach + math.SNOISE_LIPSCHITZ * Math.ScaleB(centre1 + reach, -20);
// The Lipschitz form's width bound for one octave of the sampled-form check.
private static double SnoiseWidthBound(double centre1, double reach) => math.SNOISE_LIPSCHITZ * reach + SnoiseAllowance(centre1, reach);
@ -313,13 +365,17 @@ public sealed class GlslMirrorTests
return sum;
}
private static string AssertSnoiseForm(JsonArray cases, bool allowanceDecides = false) =>
AssertSampledForm(cases, "af_snoise", (_, p) => math.snoise(p),
private static float SnoiseReference(string[] tokens, float3 p) => math.snoise(p);
private static float FbmReference(string[] t, float3 p) => math.fbm_grad(p, (int)Convert.ToUInt32(t[7], 16), (float)Floats(t[8])[0], (float)Floats(t[9])[0]).w;
private static string AssertSnoiseForm(JsonArray cases, bool allowanceDecides = false, Func<string[], float3, float>? reference = null) =>
AssertSampledForm(cases, "af_snoise", reference ?? SnoiseReference,
(_, a) => SnoiseOctave(SnoiseWidthBound, a, 1.0), (_, a) => SnoiseOctave(SnoiseAllowance, a, 1.0), allowanceDecides);
private static string AssertFbmForm(JsonArray cases, bool allowanceDecides = false) =>
private static string AssertFbmForm(JsonArray cases, bool allowanceDecides = false, Func<string[], float3, float>? reference = null) =>
AssertSampledForm(cases, "af_fbm",
(t, p) => math.fbm_grad(p, (int)Convert.ToUInt32(t[7], 16), (float)Floats(t[8])[0], (float)Floats(t[9])[0]).w,
reference ?? FbmReference,
(t, a) => FbmOctaves(t, a, SnoiseWidthBound), (t, a) => FbmOctaves(t, a, SnoiseAllowance), allowanceDecides);
// AffineFrustumBall applied to C#'s own results.
@ -360,6 +416,74 @@ public sealed class GlslMirrorTests
return $"least enclosure slack {enclosure:E3} of the radius, least slack above R0 + d {lower:E3} of R0, below its widening {upper:F3} of 2^-19 (R0 + |c0|_1 + spread)";
}
private const string SpanAllowanceRead =
"Read at the case's point x, with S = |x|_1 and (e_n, e_g, e_h) = math.span_slack(math.span_rounding(1, x, zero stage, 0, 0, 0)), " +
"against the kernel's exact values in double: |n - n_exact| <= e_n and |g - g_exact|_2 <= e_g for snoise_grad, " +
"||H - H_exact||_F <= e_h for snoise_hessian, no tolerance. " +
"Report the largest fraction of each allowance per decade of S and the largest absolute error at S < 1e-2.";
// SpanAllowanceRead applied to a family's results (C#'s own, or a device's): every read is within the
// allowance math.span_slack gives at its point, the owner's own numbers and no restated literal. A small
// family (bands 1e-4 to 1e2) must also fail the check once the absolute term is removed in at least 16
// cases, and hold at least 48 cases with S <= 1e-2: the allowance's absolute term is what decides there.
private static string AssertSpanAllowance(JsonArray cases, bool hessian, bool small)
{
var absolute = math.span_slack(0.0f);
var fractions = new SortedDictionary<int, (double Value, double Gradient)>();
double smallestError = 0.0;
int absoluteDecides = 0, nearOrigin = 0;
foreach (var entry in cases)
{
var sides = ((string)entry!).Split(" -> ");
double[] x = Floats(sides[0]), result = Floats(sides[1]);
var s = Math.Abs(x[0]) + Math.Abs(x[1]) + Math.Abs(x[2]);
var slack = math.span_slack(math.span_rounding(1.0f, new float3((float)x[0], (float)x[1], (float)x[2]), default, new float3(0.0f, 0.0f, 0.0f), 0.0f, 0.0f));
double valueError = 0.0, gradientError, valueFraction = 0.0, gradientFraction;
bool needsAbsolute;
if (hessian)
{
var exact = NoiseBoundTests.HessianDouble(x[0], x[1], x[2]);
var sum = 0.0;
for (var i = 0; i < 9; i++)
sum += (result[i] - exact[i]) * (result[i] - exact[i]);
gradientError = Math.Sqrt(sum);
Assert.True(gradientError <= slack.z, $"snoise_hessian case {entry}: ||H - H_exact||_F is {gradientError:R}, past the allowance {slack.z:R}");
gradientFraction = gradientError / slack.z;
needsAbsolute = gradientError > slack.z - absolute.z;
}
else
{
var exact = NoiseBoundTests.GradDouble(x[0], x[1], x[2]);
valueError = Math.Abs(result[3] - NoiseBoundTests.ValueDouble(x[0], x[1], x[2]));
gradientError = Math.Sqrt((result[0] - exact[0]) * (result[0] - exact[0]) + (result[1] - exact[1]) * (result[1] - exact[1]) + (result[2] - exact[2]) * (result[2] - exact[2]));
Assert.True(valueError <= slack.x, $"snoise_grad case {entry}: |n - n_exact| is {valueError:R}, past the allowance {slack.x:R}");
Assert.True(gradientError <= slack.y, $"snoise_grad case {entry}: |g - g_exact|_2 is {gradientError:R}, past the allowance {slack.y:R}");
valueFraction = valueError / slack.x;
gradientFraction = gradientError / slack.y;
needsAbsolute = valueError > slack.x - absolute.x || gradientError > slack.y - absolute.y;
}
var decade = (int)Math.Floor(Math.Log10(s));
fractions.TryGetValue(decade, out var largest);
fractions[decade] = (Math.Max(largest.Value, valueFraction), Math.Max(largest.Gradient, gradientFraction));
if (needsAbsolute)
absoluteDecides++;
if (s < 1.0e-2)
smallestError = Math.Max(smallestError, Math.Max(valueError, gradientError));
if (s <= 1.0e-2)
nearOrigin++;
}
if (small)
{
Assert.True(absoluteDecides >= 16, $"only {absoluteDecides} of {cases.Count} cases fail once the absolute term is removed: the family cannot tell a device that needs it from one that does not");
Assert.True(nearOrigin >= 48, $"only {nearOrigin} of {cases.Count} cases have |x|_1 <= 1e-2");
}
var report = string.Join(", ", fractions.Select(f => hessian ? $"1e{f.Key}: {f.Value.Gradient:F3}" : $"1e{f.Key}: {f.Value.Value:F3}/{f.Value.Gradient:F3}"));
return $"largest fraction of {(hessian ? "e_h" : "e_n/e_g")} per decade of |x|_1: {report}; largest absolute error at |x|_1 < 1e-2 {smallestError:E3}; {absoluteDecides} of {cases.Count} cases need the absolute term";
}
// A committed case's arguments, decoded into the C# method's parameter types in Hex's field order.
private static object[] Arguments(MethodInfo method, string hex)
{
@ -377,6 +501,14 @@ public sealed class GlslMirrorTests
return (int)scalars.Dequeue();
if (type == typeof(uint))
return scalars.Dequeue();
if (type.IsArray)
{
var array = Array.CreateInstance(type.GetElementType()!, SiteArrays.Capacity);
for (var i = 0; i < array.Length; i++)
array.SetValue(FromScalars(type.GetElementType()!, scalars), i);
return array;
}
var value = Activator.CreateInstance(type)!;
foreach (var field in type.GetFields().Where(f => !f.IsStatic))
field.SetValue(value, FromScalars(field.FieldType, scalars));
@ -406,6 +538,8 @@ public sealed class GlslMirrorTests
private sealed record Family(string Glsl, string Tolerance, Func<System.Random, object[]> Arguments, string? Check = null, Platform Platform = Platform.None, Func<JsonArray, string>? Verify = null);
private static float Uniform(System.Random r, float lo, float hi) => lo + (hi - lo) * r.NextSingle();
private static float2 Point2(System.Random r) => new(Uniform(r, -50.0f, 50.0f), Uniform(r, -50.0f, 50.0f));
private static float3 Point3(System.Random r) => new(Uniform(r, -50.0f, 50.0f), Uniform(r, -50.0f, 50.0f), Uniform(r, -50.0f, 50.0f));
// A point on the simplex tie set, where snoise's x0 has equal components (NoiseGradTests.SnoiseIsContinuousOnTheSimplexDiagonal):
@ -527,7 +661,8 @@ public sealed class GlslMirrorTests
// Every family the site's WebGL2 readback evaluates, in a fixed order. A new family is appended, so the
// seeded stream and every earlier family's cases stay as they were. pcg3d and pcg4d are "exact"; the
// float families are ulp-bounded, with the bound measured on the device by the consumer, and
// iv_frustum_ball also carries an enclosure check. phacelle is in CultMath.Phacelle.glsl, so its
// iv_frustum_ball also carries an enclosure check; the four coordinate-band snoise_grad and snoise_hessian
// families carry an allowance check, applied to C#'s own results here and to a device's by the device judge. phacelle is in CultMath.Phacelle.glsl, so its
// evaluator concatenates that file after CultMath.glsl; the site, which does not vendor it, skips it.
private static readonly Family[] Families = new Family[]
{
@ -563,7 +698,7 @@ public sealed class GlslMirrorTests
return new object[] { slope, z0, z0 + Uniform(r, 0.01f, 50.0f), Uniform(r, 0.0001f, 0.05f), Uniform(r, 0.0f, 2.0f) };
}, FrustumBall, Verify: AssertFrustumBall),
new("vec2 cultmath_iv_exp(vec2)", "ulp-bounded, platform exp", r => new object[] { Interval(r) }, Platform: Platform.Ulp),
new("CultPhasor cultmath_phacelle(vec3, vec3, float, float)", "ulp-bounded, platform exp, sin, cos; across platforms |diff| <= 2^-20 max(|v|, 1)",
new("CultPhasor cultmath_phacelle(vec3, vec3, float, float)", "ulp-bounded, platform exp, sin, cos; across platforms |diff| <= 2^-20 max(|v|, 1); on a device |diff| <= 2^-18 max(|v|, 1)",
r => new object[] { Point3(r), Side(r), Uniform(r, 0.0f, 1.0f), 0.5f }, Platform: Platform.Scaled),
new("float cultmath_snoise(vec3)", "ulp-bounded, on the simplex tie set", r => new object[] { TiePoint3(r) }),
new("vec4 cultmath_snoise_grad(vec3)", "ulp-bounded, on the simplex tie set", r => new object[] { TiePoint3(r) }),
@ -638,8 +773,201 @@ public sealed class GlslMirrorTests
var lo = Uniform(r, -2.0f, 0.5f);
return new object[] { Uniform(r, -4.0f, 4.0f), r.Next(8) == 0 ? 0.0f : Uniform(r, -4.0f, 4.0f), lo, lo + Uniform(r, 0.0f, 3.0f), Uniform(r, -3.0f, 3.0f), Uniform(r, -3.0f, 3.0f) };
}),
// The edge cases the family above never draws: lo above hi (b = 0 and b != 0; the lowering must
// normalise before any clamp sees it) and infinite bounds (hi = +inf, lo = -inf, both).
new Family("float cultmath_integral_clamp_linear(float, float, float, float, float, float)", "ulp-bounded, lo above hi and infinite bounds",
r =>
{
var a = Uniform(r, -4.0f, 4.0f);
var slope = Uniform(r, -4.0f, 4.0f);
var (t0, t1) = (Uniform(r, -3.0f, 3.0f), Uniform(r, -3.0f, 3.0f));
return r.Next(5) switch
{
0 => new object[] { a, 0.0f, Uniform(r, 0.5f, 2.0f), Uniform(r, -2.0f, 0.4f), t0, t1 },
1 => new object[] { a, slope, Uniform(r, 0.5f, 2.0f), Uniform(r, -2.0f, 0.4f), t0, t1 },
2 => new object[] { a, slope, Uniform(r, -2.0f, 1.0f), float.PositiveInfinity, t0, t1 },
3 => new object[] { a, slope, float.NegativeInfinity, Uniform(r, -1.0f, 2.0f), t0, t1 },
_ => new object[] { a, slope, float.NegativeInfinity, float.PositiveInfinity, t0, t1 },
};
}),
// Zero-length spans (t1 = t0 three times in four) with a flat or sloped line, and infinite clamp bounds
// on either side or both: an infinite bound times a zero length must give +0, and a flat line clamped
// to an infinite bound over a nonzero span is infinite, never NaN.
new Family("float cultmath_integral_clamp_linear(float, float, float, float, float, float)", "ulp-bounded, zero-length spans and flat infinite clamps",
r =>
{
float Line() => r.Next(3) switch { 0 => 0.0f, 1 => -0.0f, _ => Uniform(r, -4.0f, 4.0f) };
float Bound() => r.Next(3) switch { 0 => float.NegativeInfinity, 1 => float.PositiveInfinity, _ => Uniform(r, -2.0f, 2.0f) };
var (a, b, lo, hi) = (Line(), Line(), Bound(), Bound());
var t0 = Uniform(r, -3.0f, 3.0f);
var t1 = r.Next(4) != 0 ? t0 : Uniform(r, -3.0f, 3.0f);
return new object[] { a, b, lo, hi, t0, t1 };
}),
// The span stages over their struct arguments: realistic stages and spans (SpanStageTests' own draw), the
// midpoint a Point3, one span in four a point.
new Family("SpanNoise cultmath_span_fbm(SpanPath, SpanStage, int, float, float)", "ulp-bounded", SpanArguments),
new Family("SpanNoise cultmath_span_ridged(SpanPath, SpanStage, int, float, float)", "ulp-bounded", SpanArguments),
// The kernel's reads over coordinate bands, the magnitude log-uniform in the band: the allowance of
// math.span_slack is checked against the exact kernel at every case (SpanAllowanceRead).
new Family("vec4 cultmath_snoise_grad(vec3)", "ulp-bounded, coordinate bands 1e-4 to 1e2", r => new object[] { BandPoint3(r, 1.0e-4, 1.0e2) }, SpanAllowanceRead, Verify: cases => AssertSpanAllowance(cases, hessian: false, small: true)),
new Family("vec4 cultmath_snoise_grad(vec3)", "ulp-bounded, coordinate bands 1e2 to 4e6", r => new object[] { BandPoint3(r, 1.0e2, 4.0e6) }, SpanAllowanceRead, Verify: cases => AssertSpanAllowance(cases, hessian: false, small: false)),
new Family("mat3 cultmath_snoise_hessian(vec3)", "ulp-bounded, coordinate bands 1e-4 to 1e2", r => new object[] { BandPoint3(r, 1.0e-4, 1.0e2) }, SpanAllowanceRead, Verify: cases => AssertSpanAllowance(cases, hessian: true, small: true)),
new Family("mat3 cultmath_snoise_hessian(vec3)", "ulp-bounded, coordinate bands 1e2 to 4e6", r => new object[] { BandPoint3(r, 1.0e2, 4.0e6) }, SpanAllowanceRead, Verify: cases => AssertSpanAllowance(cases, hessian: true, small: false)),
// Segment distance in 2D and 3D (one segment in eight degenerate), then the site cells: scores, fold step
// over states built by inserting 0-3 random scores (one in four with an exactly tied score), border, margin.
new Family("float cultmath_distance_to_segment(vec2, vec2, vec2)", "ulp-bounded", r =>
{
var start = Point2(r);
return new object[] { Point2(r), start, r.Next(8) == 0 ? start : Point2(r) };
}),
new Family("vec2 cultmath_segment_closest(vec2, vec2, vec2)", "ulp-bounded", r =>
{
var start = Point2(r);
return new object[] { Point2(r), start, r.Next(8) == 0 ? start : Point2(r) };
}),
new Family("float cultmath_distance_to_segment(vec3, vec3, vec3)", "ulp-bounded", r =>
{
var start = Point3(r);
return new object[] { Point3(r), start, r.Next(8) == 0 ? start : Point3(r) };
}),
new Family("vec3 cultmath_segment_closest(vec3, vec3, vec3)", "ulp-bounded", r =>
{
var start = Point3(r);
return new object[] { Point3(r), start, r.Next(8) == 0 ? start : Point3(r) };
}),
// The border step: owner and other random (one in six coincident), a running border either empty or a
// random (normal, distance); one in six other is a distinct site within 1e-3 of the owner (inside the 1e-2 guard a
// changed length test would trip), one in six coincident.
new Family("vec4 cultmath_power_border_step(vec3, vec4, vec4, vec4)", "ulp-bounded", r =>
{
var owner = Site(r, 0.1f, 20.0f);
var border = r.Next(2) == 0 ? new float4(0.0f, 0.0f, 0.0f, 1.0e30f) : new float4(Point3(r), Uniform(r, 0.0f, 50.0f));
var pick = r.Next(6);
var near = new float4(owner.xyz + new float3(Uniform(r, 1.0e-4f, 1.0e-3f), Uniform(r, -1.0e-3f, 1.0e-3f), Uniform(r, -1.0e-3f, 1.0e-3f)), Uniform(r, 0.1f, 20.0f));
return new object[] { Point3(r), owner, pick == 0 ? owner : pick == 1 ? near : Site(r, 0.1f, 20.0f), border };
}),
new Family("vec4 cultmath_power_score(vec3, vec4)", "ulp-bounded", r => new object[] { Point3(r), Site(r, 0.1f, 20.0f) }),
new Family("vec4 cultmath_multiplicative_score(vec3, vec4)", "ulp-bounded", r => new object[] { Point3(r), Site(r, 0.1f, 100.0f, logWeight: true) }),
new Family("CultSiteCells cultmath_site_cells_insert(CultSiteCells, vec4, int)", "ulp-bounded", r =>
{
var (cells, scores) = CellsState(r);
var score = r.Next(4) == 0 && scores.Count > 0 ? scores[r.Next(scores.Count)] : PowerOrMultiplicative(r);
return new object[] { cells, score, scores.Count };
}),
new Family("vec4 cultmath_site_cells_border(CultSiteCells)", "ulp-bounded", r =>
{
var (cells, _) = CellsState(r, minimum: r.Next(8) == 0 ? 0 : 2);
return new object[] { cells };
}),
new Family("float cultmath_gabriel_margin(vec3, vec3, vec3)", "ulp-bounded", r =>
{
var a = Point3(r);
return new object[] { a, r.Next(8) == 0 ? a : Point3(r), Point3(r) };
}),
// The lattice steps. The cell is an integer-valued vec2 (negative and large included), time and drift span
// zero and the accepted range; the weight's border covers 0, near the falloff, and the 1.0e30 of no body.
new Family("vec3 cultmath_lattice_site(vec2, float, float)", "ulp-bounded", r =>
new object[] { new float2(r.Next(-4000, 4000), r.Next(-4000, 4000)), r.Next(6) == 0 ? 0.0f : Uniform(r, -200.0f, 200.0f), r.Next(6) == 0 ? 0.0f : Uniform(r, -3.0f, 3.0f) }),
new Family("float cultmath_lattice_weight(float, float, float, float)", "ulp-bounded", r =>
{
var falloff = Uniform(r, 0.05f, 40.0f);
var border = r.Next(5) switch { 0 => 0.0f, 1 => 1.0e30f, 2 => falloff * Uniform(r, 0.9f, 1.1f), _ => Uniform(r, 0.0f, 3.0f * falloff) };
var boost = r.Next(5) switch { 0 => 0.0f, 1 => 4.0f, _ => Uniform(r, 0.0f, 4.0f) };
return new object[] { Uniform(r, 0.5f, 1.0f), border, boost, falloff };
}),
new Family("float cultmath_lattice_distance(vec2, vec2, float, vec2, float)", "ulp-bounded", r =>
{
var angle = Uniform(r, 0.0f, 6.2831855f);
var normal = r.Next(6) == 0 ? new float2(1.0f, 0.0f) : new float2(MathF.Cos(angle), MathF.Sin(angle));
var stretch = r.Next(5) switch { 0 => 1.0f, 1 => 2.0f, _ => Uniform(r, 1.0f, 2.0f) };
return new object[] { Point2(r), Point2(r), Uniform(r, 0.5f, 5.0f), normal, stretch };
}),
// The smooth step: d exactly at f1 + 2 smoothing one in four (the equal branch), near it one in four, the
// smoothing near 0 and at 1, m anywhere up to f1 + smoothing.
// One draw in four is the cap point: m = f1 + smoothing exactly and d = f1 + smoothing (1 + U(0.3, 0.9)) with smoothing in
// [0.5, 1], where next exceeds f1 + smoothing and the min clamps; a step that returns next reads another function there.
new Family("float cultmath_lattice_smooth_step(float, float, float, float)", "ulp-bounded", r =>
{
if (r.Next(4) == 0)
{
var cappedSmoothing = Uniform(r, 0.5f, 1.0f);
var cappedF1 = Uniform(r, 0.0f, 10.0f);
return new object[] { cappedF1 + cappedSmoothing, cappedF1 + cappedSmoothing * (1.0f + Uniform(r, 0.3f, 0.9f)), cappedF1, cappedSmoothing };
}
var smoothing = r.Next(5) switch { 0 => 1.0f, 1 => Uniform(r, 1.0e-3f, 1.0e-2f), _ => Uniform(r, 0.01f, 1.0f) };
var f1 = Uniform(r, 0.0f, 10.0f);
var edge = f1 + 2.0f * smoothing;
var d = r.Next(4) switch { 0 => edge, 1 => edge * Uniform(r, 0.99f, 1.01f), _ => Uniform(r, f1, edge + 1.0f) };
return new object[] { Uniform(r, f1, f1 + smoothing), d, f1, smoothing };
}),
// The site-array functions: a fixed-capacity array and a count, killed on the device (the lattice at rings up to 17).
new Family($"vec4 cultmath_power_border(vec3, vec4[{SiteArrays.Capacity}], int)", "ulp-bounded", SiteArrays.PowerBorderArguments),
new Family($"vec4 cultmath_weighted_lattice_cells(vec2, float, vec4[{SiteArrays.Capacity}], int, float, float, float, float, float, vec2, float)", "ulp-bounded", SiteArrays.LatticeArguments),
}).ToArray();
// A point of magnitude m, m log-uniform in [lo, hi]: one component at +-m, the other two uniform in [-m, m],
// the axis of the large one random.
private static float3 BandPoint3(System.Random r, double lo, double hi)
{
var m = lo * Math.Pow(hi / lo, r.NextDouble());
var v = new double[] { (r.NextDouble() * 2.0 - 1.0) * m, (r.NextDouble() * 2.0 - 1.0) * m, (r.NextDouble() * 2.0 - 1.0) * m };
v[r.Next(3)] = r.Next(2) == 0 ? m : -m;
return new float3((float)v[0], (float)v[1], (float)v[2]);
}
// A site float4(position.xyz, weight): the weight uniform in [lo, hi], or log-uniform when logWeight.
private static float4 Site(System.Random r, float lo, float hi, bool logWeight = false)
{
var p = Point3(r);
return new float4(p.x, p.y, p.z, logWeight ? lo * MathF.Pow(hi / lo, r.NextSingle()) : Uniform(r, lo, hi));
}
private static float4 PowerOrMultiplicative(System.Random r) =>
r.Next(2) == 0 ? math.power_score(Point3(r), Site(r, 0.1f, 20.0f)) : math.multiplicative_score(Point3(r), Site(r, 0.1f, 100.0f, logWeight: true));
// A fold state made by the production path: site_cells_insert over a few random scores (at least minimum).
private static (CultSiteCells Cells, List<float4> Scores) CellsState(System.Random r, int minimum = 0)
{
var scores = new List<float4>();
var cells = math.site_cells_empty();
for (var i = minimum + r.Next(4 - minimum); i > 0; i--)
{
var score = r.Next(4) == 0 && scores.Count > 0 ? scores[^1] : PowerOrMultiplicative(r);
cells = math.site_cells_insert(cells, score, scores.Count);
scores.Add(score);
}
return (cells, scores);
}
private static object[] SpanArguments(System.Random r)
{
var stage = SpanStageTests.Stage.Draw(r);
var halfLength = r.Next(4) == 0 ? 0.0f : new[] { 0.02f, 0.1f, 0.5f }[r.Next(3)];
var span = SpanStageTests.Span.Draw(r, Point3(r), halfLength, Uniform(r, 0.0f, 2.0f));
return new object[] { span.Path, stage.S, stage.Octaves, stage.Lacunarity, stage.Gain };
}
/// <summary>
/// The committed fixture's cases of the three cultmath_integral_clamp_linear families, decoded into the
/// C# arguments (a, b, lo, hi, t0, t1): the random cases the clamp-normalisation tests also run.
/// </summary>
internal static IReadOnlyList<float[]> IntegralClampLinearFixtureCases
{
get
{
const string clamp = "float cultmath_integral_clamp_linear(float, float, float, float, float, float)";
var method = Resolve(clamp);
return JsonNode.Parse(File.ReadAllText(FixturePath))!["functions"]!.AsArray()
.Where(family => (string)family!["glsl"]! == clamp)
.SelectMany(family => family!["cases"]!.AsArray())
.Select(entry => Arguments(method, ((string)entry!).Split(" -> ")[0]).Select(a => (float)a).ToArray())
.ToArray();
}
}
// The fixture text, each case's arguments given by family and point index.
private static string GoldenFixture(Func<int, int, object[]> arguments)
{
@ -682,6 +1010,9 @@ public sealed class GlslMirrorTests
{
["float"] = typeof(float), ["int"] = typeof(int), ["vec2"] = typeof(float2), ["vec3"] = typeof(float3),
["vec4"] = typeof(float4), ["ivec3"] = typeof(int3), ["ivec4"] = typeof(int4),
["SpanPath"] = typeof(SpanPath), ["SpanStage"] = typeof(SpanStage),
["CultSiteCells"] = typeof(CultSiteCells),
[$"vec4[{SiteArrays.Capacity}]"] = typeof(float4[]),
};
// The C# math method a GLSL signature names: cultmath_ dropped, parameter types mapped.
@ -690,6 +1021,7 @@ public sealed class GlslMirrorTests
var match = Regex.Match(glsl, @"^\w+ cultmath_(\w+)\(([^)]*)\)$");
var types = match.Groups[2].Value.Split(", ").Select(t => GlslTypes[t]).ToArray();
return typeof(math).GetMethod(match.Groups[1].Value, BindingFlags.Public | BindingFlags.Static, null, types, null)
?? typeof(SiteArrays).GetMethod(match.Groups[1].Value, BindingFlags.NonPublic | BindingFlags.Static, null, types, null)
?? throw new InvalidOperationException("no C# math counterpart for " + glsl);
}
@ -705,6 +1037,8 @@ public sealed class GlslMirrorTests
// the same read as GLSL's columns.
private static IEnumerable<object> Scalars(object value) => value is float or int or uint
? new[] { value }
: value is Array array
? array.Cast<object>().SelectMany(Scalars)
: value is float3x3 matrix
? new object[] { matrix._m00, matrix._m01, matrix._m02, matrix._m10, matrix._m11, matrix._m12, matrix._m20, matrix._m21, matrix._m22 }
: value.GetType().GetFields().Where(f => !f.IsStatic).SelectMany(f => Scalars(f.GetValue(value)!));

Some files were not shown because too many files have changed in this diff Show more