GameCult/CultLib
Watch
9
0
Fork
You've already forked CultLib
 0
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions

Land C2a: CultCache document variants, element ids and write ownership (3.0.0 Unreleased)

Browse source 
Merges hands/variants-c2a at 7605bd17. Verified at 463714c4 (code tree
identical): every C#, TS, Rust and Python suite green on Yggdrasil and the
old-reader refusal gate refusing in all four runtimes
(variants:cut_report:cut-land-c2a-main.h1, Soul variants:verdict:cut-land-c2a-main.s1).
No tag: tags, Unity plugin rebuild and the declared-range gate are the 3.0.0 release cut.
Non-building branch commits exist; bisect with --first-parent.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Meta Vens 2026-10-27 23:56:22 +01:00
commit b7a68884c6
213 changed files with 22543 additions and 3492 deletions
Download patch file Download diff file Expand all files Collapse all files

436
contracts/cultmesh/content-acceptance-vectors.json
View file

File diff suppressed because it is too large Load diff

10
contracts/cultmesh/content-vectors.cs-written.json
View file

@ -168,22 +168,22 @@
{
"label": "manifest",
"kind": "manifest",
"messagePackBase64": "mrVldXJla2EvY29udGVudC12ZWN0b3KlYXNzZXShMdlAZjRhZTJhN2M5NWY0ZDk2NjA1OGY3NWMxMzE4NWFhMGFiNmE5MWZjNWFmYWQ4ODYyOTZlZjJhZWU3NTVhMmFmMc4AFBLfvGFwcGxpY2F0aW9uL3gtY3VsdG5ldC12ZWN0b3LZITIwMjYtMDktMjJUMDA6MDA6MDAuMDAwMDAwMCswMDowMJaU2UAzMWExZjlkZWEwMTY5NTUxMDkyZDA1ZThiZjRhNDQ2MjI4YzhjM2ViNGM5YjcxM2M2NmFkY2I3ZmQ1M2M4OWJlAM4ABAAA2U9tZXNoOmNkbjpjaHVuazozMWExZjlkZWEwMTY5NTUxMDkyZDA1ZThiZjRhNDQ2MjI4YzhjM2ViNGM5YjcxM2M2NmFkY2I3ZmQ1M2M4OWJllNlAZTMxYmRjNjM1NzdmZTc2ZjY0NDEwMTRhYWFhMjIyNmQ5ZmZjZTExYjQ0ZDBkNTEwY2FkZTEzMmI3OWZkN2Q3Ns4ABAAAzgAEAADZT21lc2g6Y2RuOmNodW5rOmUzMWJkYzYzNTc3ZmU3NmY2NDQxMDE0YWFhYTIyMjZkOWZmY2UxMWI0NGQwZDUxMGNhZGUxMzJiNzlmZDdkNzaU2UBjN2YxYmRlOWUxZWZhN2QxMzg1ZDUxYjA3N2VhODdiMGMzODYwZDI4ZGE0M2Q1ZDZmMjIxZTg0NWM0YzIzNjBmzgAIAADOAAQAANlPbWVzaDpjZG46Y2h1bms6YzdmMWJkZTllMWVmYTdkMTM4NWQ1MWIwNzdlYTg3YjBjMzg2MGQyOGRhNDNkNWQ2ZjIyMWU4NDVjNGMyMzYwZpTZQGFhMTQ0NGJlNGI2NDA5OGY1ZGQ4NjNiNWIwYWM1NjE0ODBiM2ZiNWM0MWZjMDdjZjUwYTYyZWRjOTQxMjRiY2POAAwAAM4ABAAA2U9tZXNoOmNkbjpjaHVuazphYTE0NDRiZTRiNjQwOThmNWRkODYzYjViMGFjNTYxNDgwYjNmYjVjNDFmYzA3Y2Y1MGE2MmVkYzk0MTI0YmNjlNlAMjgzMTcxNTg1OWYwMWYzZjk2M2RkMGYxMTk0ZGJkYmI3NjA5OTk3NDIzMDE1ZDMxZjYwNzkxMWUxYmY4NjQwMs4AEAAAzgAEAADZT21lc2g6Y2RuOmNodW5rOjI4MzE3MTU4NTlmMDFmM2Y5NjNkZDBmMTE5NGRiZGJiNzYwOTk5NzQyMzAxNWQzMWY2MDc5MTFlMWJmODY0MDKU2UA1OTkzYWM2MDQwN2U4NjYwMTg3M2JjMzhlZTZmOGRiM2VhOTEyZTk4NzYxY2I1NThhYjk3OTVkYjMwNmI0NzRjzgAUAADNEt/ZT21lc2g6Y2RuOmNodW5rOjU5OTNhYzYwNDA3ZTg2NjAxODczYmMzOGVlNmY4ZGIzZWE5MTJlOTg3NjFjYjU1OGFiOTc5NWRiMzA2YjQ3NGOQgA=="
"messagePackBase64": "mrVldXJla2EvY29udGVudC12ZWN0b3KlYXNzZXShMdlAZjRhZTJhN2M5NWY0ZDk2NjA1OGY3NWMxMzE4NWFhMGFiNmE5MWZjNWFmYWQ4ODYyOTZlZjJhZWU3NTVhMmFmMc4AFBLfvGFwcGxpY2F0aW9uL3gtY3VsdG5ldC12ZWN0b3LZITIwMjYtMDktMjJUMDA6MDA6MDAuMDAwMDAwMCswMDowMJaV2UAzMWExZjlkZWEwMTY5NTUxMDkyZDA1ZThiZjRhNDQ2MjI4YzhjM2ViNGM5YjcxM2M2NmFkY2I3ZmQ1M2M4OWJlAM4ABAAA2U9tZXNoOmNkbjpjaHVuazozMWExZjlkZWEwMTY5NTUxMDkyZDA1ZThiZjRhNDQ2MjI4YzhjM2ViNGM5YjcxM2M2NmFkY2I3ZmQ1M2M4OWJloTCV2UBlMzFiZGM2MzU3N2ZlNzZmNjQ0MTAxNGFhYWEyMjI2ZDlmZmNlMTFiNDRkMGQ1MTBjYWRlMTMyYjc5ZmQ3ZDc2zgAEAADOAAQAANlPbWVzaDpjZG46Y2h1bms6ZTMxYmRjNjM1NzdmZTc2ZjY0NDEwMTRhYWFhMjIyNmQ5ZmZjZTExYjQ0ZDBkNTEwY2FkZTEzMmI3OWZkN2Q3NqYyNjIxNDSV2UBjN2YxYmRlOWUxZWZhN2QxMzg1ZDUxYjA3N2VhODdiMGMzODYwZDI4ZGE0M2Q1ZDZmMjIxZTg0NWM0YzIzNjBmzgAIAADOAAQAANlPbWVzaDpjZG46Y2h1bms6YzdmMWJkZTllMWVmYTdkMTM4NWQ1MWIwNzdlYTg3YjBjMzg2MGQyOGRhNDNkNWQ2ZjIyMWU4NDVjNGMyMzYwZqY1MjQyODiV2UBhYTE0NDRiZTRiNjQwOThmNWRkODYzYjViMGFjNTYxNDgwYjNmYjVjNDFmYzA3Y2Y1MGE2MmVkYzk0MTI0YmNjzgAMAADOAAQAANlPbWVzaDpjZG46Y2h1bms6YWExNDQ0YmU0YjY0MDk4ZjVkZDg2M2I1YjBhYzU2MTQ4MGIzZmI1YzQxZmMwN2NmNTBhNjJlZGM5NDEyNGJjY6Y3ODY0MzKV2UAyODMxNzE1ODU5ZjAxZjNmOTYzZGQwZjExOTRkYmRiYjc2MDk5OTc0MjMwMTVkMzFmNjA3OTExZTFiZjg2NDAyzgAQAADOAAQAANlPbWVzaDpjZG46Y2h1bms6MjgzMTcxNTg1OWYwMWYzZjk2M2RkMGYxMTk0ZGJkYmI3NjA5OTk3NDIzMDE1ZDMxZjYwNzkxMWUxYmY4NjQwMqcxMDQ4NTc2ldlANTk5M2FjNjA0MDdlODY2MDE4NzNiYzM4ZWU2ZjhkYjNlYTkxMmU5ODc2MWNiNTU4YWI5Nzk1ZGIzMDZiNDc0Y84AFAAAzRLf2U9tZXNoOmNkbjpjaHVuazo1OTkzYWM2MDQwN2U4NjYwMTg3M2JjMzhlZTZmOGRiM2VhOTEyZTk4NzYxY2I1NThhYjk3OTVkYjMwNmI0NzRjpzEzMTA3MjCQgA=="
},
{
"label": "manifest_out_of_order",
"kind": "manifest",
"messagePackBase64": "mrVldXJla2EvY29udGVudC12ZWN0b3KlYXNzZXShMdlAZjRhZTJhN2M5NWY0ZDk2NjA1OGY3NWMxMzE4NWFhMGFiNmE5MWZjNWFmYWQ4ODYyOTZlZjJhZWU3NTVhMmFmMc4AFBLfvGFwcGxpY2F0aW9uL3gtY3VsdG5ldC12ZWN0b3LZITIwMjYtMDktMjJUMDA6MDA6MDAuMDAwMDAwMCswMDowMJaU2UBhYTE0NDRiZTRiNjQwOThmNWRkODYzYjViMGFjNTYxNDgwYjNmYjVjNDFmYzA3Y2Y1MGE2MmVkYzk0MTI0YmNjzgAMAADOAAQAANlPbWVzaDpjZG46Y2h1bms6YWExNDQ0YmU0YjY0MDk4ZjVkZDg2M2I1YjBhYzU2MTQ4MGIzZmI1YzQxZmMwN2NmNTBhNjJlZGM5NDEyNGJjY5TZQDMxYTFmOWRlYTAxNjk1NTEwOTJkMDVlOGJmNGE0NDYyMjhjOGMzZWI0YzliNzEzYzY2YWRjYjdmZDUzYzg5YmUAzgAEAADZT21lc2g6Y2RuOmNodW5rOjMxYTFmOWRlYTAxNjk1NTEwOTJkMDVlOGJmNGE0NDYyMjhjOGMzZWI0YzliNzEzYzY2YWRjYjdmZDUzYzg5YmWU2UA1OTkzYWM2MDQwN2U4NjYwMTg3M2JjMzhlZTZmOGRiM2VhOTEyZTk4NzYxY2I1NThhYjk3OTVkYjMwNmI0NzRjzgAUAADNEt/ZT21lc2g6Y2RuOmNodW5rOjU5OTNhYzYwNDA3ZTg2NjAxODczYmMzOGVlNmY4ZGIzZWE5MTJlOTg3NjFjYjU1OGFiOTc5NWRiMzA2YjQ3NGOU2UBlMzFiZGM2MzU3N2ZlNzZmNjQ0MTAxNGFhYWEyMjI2ZDlmZmNlMTFiNDRkMGQ1MTBjYWRlMTMyYjc5ZmQ3ZDc2zgAEAADOAAQAANlPbWVzaDpjZG46Y2h1bms6ZTMxYmRjNjM1NzdmZTc2ZjY0NDEwMTRhYWFhMjIyNmQ5ZmZjZTExYjQ0ZDBkNTEwY2FkZTEzMmI3OWZkN2Q3NpTZQDI4MzE3MTU4NTlmMDFmM2Y5NjNkZDBmMTE5NGRiZGJiNzYwOTk5NzQyMzAxNWQzMWY2MDc5MTFlMWJmODY0MDLOABAAAM4ABAAA2U9tZXNoOmNkbjpjaHVuazoyODMxNzE1ODU5ZjAxZjNmOTYzZGQwZjExOTRkYmRiYjc2MDk5OTc0MjMwMTVkMzFmNjA3OTExZTFiZjg2NDAylNlAYzdmMWJkZTllMWVmYTdkMTM4NWQ1MWIwNzdlYTg3YjBjMzg2MGQyOGRhNDNkNWQ2ZjIyMWU4NDVjNGMyMzYwZs4ACAAAzgAEAADZT21lc2g6Y2RuOmNodW5rOmM3ZjFiZGU5ZTFlZmE3ZDEzODVkNTFiMDc3ZWE4N2IwYzM4NjBkMjhkYTQzZDVkNmYyMjFlODQ1YzRjMjM2MGaQgA=="
"messagePackBase64": "mrVldXJla2EvY29udGVudC12ZWN0b3KlYXNzZXShMdlAZjRhZTJhN2M5NWY0ZDk2NjA1OGY3NWMxMzE4NWFhMGFiNmE5MWZjNWFmYWQ4ODYyOTZlZjJhZWU3NTVhMmFmMc4AFBLfvGFwcGxpY2F0aW9uL3gtY3VsdG5ldC12ZWN0b3LZITIwMjYtMDktMjJUMDA6MDA6MDAuMDAwMDAwMCswMDowMJaV2UBhYTE0NDRiZTRiNjQwOThmNWRkODYzYjViMGFjNTYxNDgwYjNmYjVjNDFmYzA3Y2Y1MGE2MmVkYzk0MTI0YmNjzgAMAADOAAQAANlPbWVzaDpjZG46Y2h1bms6YWExNDQ0YmU0YjY0MDk4ZjVkZDg2M2I1YjBhYzU2MTQ4MGIzZmI1YzQxZmMwN2NmNTBhNjJlZGM5NDEyNGJjY6Y3ODY0MzKV2UAzMWExZjlkZWEwMTY5NTUxMDkyZDA1ZThiZjRhNDQ2MjI4YzhjM2ViNGM5YjcxM2M2NmFkY2I3ZmQ1M2M4OWJlAM4ABAAA2U9tZXNoOmNkbjpjaHVuazozMWExZjlkZWEwMTY5NTUxMDkyZDA1ZThiZjRhNDQ2MjI4YzhjM2ViNGM5YjcxM2M2NmFkY2I3ZmQ1M2M4OWJloTCV2UA1OTkzYWM2MDQwN2U4NjYwMTg3M2JjMzhlZTZmOGRiM2VhOTEyZTk4NzYxY2I1NThhYjk3OTVkYjMwNmI0NzRjzgAUAADNEt/ZT21lc2g6Y2RuOmNodW5rOjU5OTNhYzYwNDA3ZTg2NjAxODczYmMzOGVlNmY4ZGIzZWE5MTJlOTg3NjFjYjU1OGFiOTc5NWRiMzA2YjQ3NGOnMTMxMDcyMJXZQGUzMWJkYzYzNTc3ZmU3NmY2NDQxMDE0YWFhYTIyMjZkOWZmY2UxMWI0NGQwZDUxMGNhZGUxMzJiNzlmZDdkNzbOAAQAAM4ABAAA2U9tZXNoOmNkbjpjaHVuazplMzFiZGM2MzU3N2ZlNzZmNjQ0MTAxNGFhYWEyMjI2ZDlmZmNlMTFiNDRkMGQ1MTBjYWRlMTMyYjc5ZmQ3ZDc2pjI2MjE0NJXZQDI4MzE3MTU4NTlmMDFmM2Y5NjNkZDBmMTE5NGRiZGJiNzYwOTk5NzQyMzAxNWQzMWY2MDc5MTFlMWJmODY0MDLOABAAAM4ABAAA2U9tZXNoOmNkbjpjaHVuazoyODMxNzE1ODU5ZjAxZjNmOTYzZGQwZjExOTRkYmRiYjc2MDk5OTc0MjMwMTVkMzFmNjA3OTExZTFiZjg2NDAypzEwNDg1NzaV2UBjN2YxYmRlOWUxZWZhN2QxMzg1ZDUxYjA3N2VhODdiMGMzODYwZDI4ZGE0M2Q1ZDZmMjIxZTg0NWM0YzIzNjBmzgAIAADOAAQAANlPbWVzaDpjZG46Y2h1bms6YzdmMWJkZTllMWVmYTdkMTM4NWQ1MWIwNzdlYTg3YjBjMzg2MGQyOGRhNDNkNWQ2ZjIyMWU4NDVjNGMyMzYwZqY1MjQyODiQgA=="
},
{
"label": "manifest_metadata_order",
"kind": "manifest",
"messagePackBase64": "mrVldXJla2EvY29udGVudC12ZWN0b3KlYXNzZXShMdlAZjRhZTJhN2M5NWY0ZDk2NjA1OGY3NWMxMzE4NWFhMGFiNmE5MWZjNWFmYWQ4ODYyOTZlZjJhZWU3NTVhMmFmMc4AFBLfvGFwcGxpY2F0aW9uL3gtY3VsdG5ldC12ZWN0b3LZITIwMjYtMDktMjJUMDA6MDA6MDAuMDAwMDAwMCswMDowMJaU2UAzMWExZjlkZWEwMTY5NTUxMDkyZDA1ZThiZjRhNDQ2MjI4YzhjM2ViNGM5YjcxM2M2NmFkY2I3ZmQ1M2M4OWJlAM4ABAAA2U9tZXNoOmNkbjpjaHVuazozMWExZjlkZWEwMTY5NTUxMDkyZDA1ZThiZjRhNDQ2MjI4YzhjM2ViNGM5YjcxM2M2NmFkY2I3ZmQ1M2M4OWJllNlAZTMxYmRjNjM1NzdmZTc2ZjY0NDEwMTRhYWFhMjIyNmQ5ZmZjZTExYjQ0ZDBkNTEwY2FkZTEzMmI3OWZkN2Q3Ns4ABAAAzgAEAADZT21lc2g6Y2RuOmNodW5rOmUzMWJkYzYzNTc3ZmU3NmY2NDQxMDE0YWFhYTIyMjZkOWZmY2UxMWI0NGQwZDUxMGNhZGUxMzJiNzlmZDdkNzaU2UBjN2YxYmRlOWUxZWZhN2QxMzg1ZDUxYjA3N2VhODdiMGMzODYwZDI4ZGE0M2Q1ZDZmMjIxZTg0NWM0YzIzNjBmzgAIAADOAAQAANlPbWVzaDpjZG46Y2h1bms6YzdmMWJkZTllMWVmYTdkMTM4NWQ1MWIwNzdlYTg3YjBjMzg2MGQyOGRhNDNkNWQ2ZjIyMWU4NDVjNGMyMzYwZpTZQGFhMTQ0NGJlNGI2NDA5OGY1ZGQ4NjNiNWIwYWM1NjE0ODBiM2ZiNWM0MWZjMDdjZjUwYTYyZWRjOTQxMjRiY2POAAwAAM4ABAAA2U9tZXNoOmNkbjpjaHVuazphYTE0NDRiZTRiNjQwOThmNWRkODYzYjViMGFjNTYxNDgwYjNmYjVjNDFmYzA3Y2Y1MGE2MmVkYzk0MTI0YmNjlNlAMjgzMTcxNTg1OWYwMWYzZjk2M2RkMGYxMTk0ZGJkYmI3NjA5OTk3NDIzMDE1ZDMxZjYwNzkxMWUxYmY4NjQwMs4AEAAAzgAEAADZT21lc2g6Y2RuOmNodW5rOjI4MzE3MTU4NTlmMDFmM2Y5NjNkZDBmMTE5NGRiZGJiNzYwOTk5NzQyMzAxNWQzMWY2MDc5MTFlMWJmODY0MDKU2UA1OTkzYWM2MDQwN2U4NjYwMTg3M2JjMzhlZTZmOGRiM2VhOTEyZTk4NzYxY2I1NThhYjk3OTVkYjMwNmI0NzRjzgAUAADNEt/ZT21lc2g6Y2RuOmNodW5rOjU5OTNhYzYwNDA3ZTg2NjAxODczYmMzOGVlNmY4ZGIzZWE5MTJlOTg3NjFjYjU1OGFiOTc5NWRiMzA2YjQ3NGOQg6R6ZXRhoTGlYWxwaGGhMqNtaWShMw=="
"messagePackBase64": "mrVldXJla2EvY29udGVudC12ZWN0b3KlYXNzZXShMdlAZjRhZTJhN2M5NWY0ZDk2NjA1OGY3NWMxMzE4NWFhMGFiNmE5MWZjNWFmYWQ4ODYyOTZlZjJhZWU3NTVhMmFmMc4AFBLfvGFwcGxpY2F0aW9uL3gtY3VsdG5ldC12ZWN0b3LZITIwMjYtMDktMjJUMDA6MDA6MDAuMDAwMDAwMCswMDowMJaV2UAzMWExZjlkZWEwMTY5NTUxMDkyZDA1ZThiZjRhNDQ2MjI4YzhjM2ViNGM5YjcxM2M2NmFkY2I3ZmQ1M2M4OWJlAM4ABAAA2U9tZXNoOmNkbjpjaHVuazozMWExZjlkZWEwMTY5NTUxMDkyZDA1ZThiZjRhNDQ2MjI4YzhjM2ViNGM5YjcxM2M2NmFkY2I3ZmQ1M2M4OWJloTCV2UBlMzFiZGM2MzU3N2ZlNzZmNjQ0MTAxNGFhYWEyMjI2ZDlmZmNlMTFiNDRkMGQ1MTBjYWRlMTMyYjc5ZmQ3ZDc2zgAEAADOAAQAANlPbWVzaDpjZG46Y2h1bms6ZTMxYmRjNjM1NzdmZTc2ZjY0NDEwMTRhYWFhMjIyNmQ5ZmZjZTExYjQ0ZDBkNTEwY2FkZTEzMmI3OWZkN2Q3NqYyNjIxNDSV2UBjN2YxYmRlOWUxZWZhN2QxMzg1ZDUxYjA3N2VhODdiMGMzODYwZDI4ZGE0M2Q1ZDZmMjIxZTg0NWM0YzIzNjBmzgAIAADOAAQAANlPbWVzaDpjZG46Y2h1bms6YzdmMWJkZTllMWVmYTdkMTM4NWQ1MWIwNzdlYTg3YjBjMzg2MGQyOGRhNDNkNWQ2ZjIyMWU4NDVjNGMyMzYwZqY1MjQyODiV2UBhYTE0NDRiZTRiNjQwOThmNWRkODYzYjViMGFjNTYxNDgwYjNmYjVjNDFmYzA3Y2Y1MGE2MmVkYzk0MTI0YmNjzgAMAADOAAQAANlPbWVzaDpjZG46Y2h1bms6YWExNDQ0YmU0YjY0MDk4ZjVkZDg2M2I1YjBhYzU2MTQ4MGIzZmI1YzQxZmMwN2NmNTBhNjJlZGM5NDEyNGJjY6Y3ODY0MzKV2UAyODMxNzE1ODU5ZjAxZjNmOTYzZGQwZjExOTRkYmRiYjc2MDk5OTc0MjMwMTVkMzFmNjA3OTExZTFiZjg2NDAyzgAQAADOAAQAANlPbWVzaDpjZG46Y2h1bms6MjgzMTcxNTg1OWYwMWYzZjk2M2RkMGYxMTk0ZGJkYmI3NjA5OTk3NDIzMDE1ZDMxZjYwNzkxMWUxYmY4NjQwMqcxMDQ4NTc2ldlANTk5M2FjNjA0MDdlODY2MDE4NzNiYzM4ZWU2ZjhkYjNlYTkxMmU5ODc2MWNiNTU4YWI5Nzk1ZGIzMDZiNDc0Y84AFAAAzRLf2U9tZXNoOmNkbjpjaHVuazo1OTkzYWM2MDQwN2U4NjYwMTg3M2JjMzhlZTZmOGRiM2VhOTEyZTk4NzYxY2I1NThhYjk3OTVkYjMwNmI0NzRjpzEzMTA3MjCQg6R6ZXRhoTGlYWxwaGGhMqNtaWShMw=="
},
{
"label": "manifest_empty_body",
"kind": "manifest",
"messagePackBase64": "mrVldXJla2EvY29udGVudC12ZWN0b3KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQC8YXBwbGljYXRpb24veC1jdWx0bmV0LXZlY3RvctkhMjAyNi0wOS0yMlQwMDowMDowMC4wMDAwMDAwKzAwOjAwkZTZQGUzYjBjNDQyOThmYzFjMTQ5YWZiZjRjODk5NmZiOTI0MjdhZTQxZTQ2NDliOTM0Y2E0OTU5OTFiNzg1MmI4NTUAANlPbWVzaDpjZG46Y2h1bms6ZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NZCA"
"messagePackBase64": "mrVldXJla2EvY29udGVudC12ZWN0b3KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQC8YXBwbGljYXRpb24veC1jdWx0bmV0LXZlY3RvctkhMjAyNi0wOS0yMlQwMDowMDowMC4wMDAwMDAwKzAwOjAwkZXZQGUzYjBjNDQyOThmYzFjMTQ5YWZiZjRjODk5NmZiOTI0MjdhZTQxZTQ2NDliOTM0Y2E0OTU5OTFiNzg1MmI4NTUAANlPbWVzaDpjZG46Y2h1bms6ZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NaEwkIA="
}
]
}
}

10
contracts/cultmesh/content-vectors.rs-written.json
View file

@ -138,22 +138,22 @@
{
"label": "manifest",
"kind": "manifest",
"messagePackBase64": "mrVldXJla2EvY29udGVudC12ZWN0b3KlYXNzZXShMdlAZjRhZTJhN2M5NWY0ZDk2NjA1OGY3NWMxMzE4NWFhMGFiNmE5MWZjNWFmYWQ4ODYyOTZlZjJhZWU3NTVhMmFmMc4AFBLfvGFwcGxpY2F0aW9uL3gtY3VsdG5ldC12ZWN0b3LZITIwMjYtMDktMjJUMDA6MDA6MDAuMDAwMDAwMCswMDowMJaU2UAzMWExZjlkZWEwMTY5NTUxMDkyZDA1ZThiZjRhNDQ2MjI4YzhjM2ViNGM5YjcxM2M2NmFkY2I3ZmQ1M2M4OWJlAM4ABAAA2U9tZXNoOmNkbjpjaHVuazozMWExZjlkZWEwMTY5NTUxMDkyZDA1ZThiZjRhNDQ2MjI4YzhjM2ViNGM5YjcxM2M2NmFkY2I3ZmQ1M2M4OWJllNlAZTMxYmRjNjM1NzdmZTc2ZjY0NDEwMTRhYWFhMjIyNmQ5ZmZjZTExYjQ0ZDBkNTEwY2FkZTEzMmI3OWZkN2Q3Ns4ABAAAzgAEAADZT21lc2g6Y2RuOmNodW5rOmUzMWJkYzYzNTc3ZmU3NmY2NDQxMDE0YWFhYTIyMjZkOWZmY2UxMWI0NGQwZDUxMGNhZGUxMzJiNzlmZDdkNzaU2UBjN2YxYmRlOWUxZWZhN2QxMzg1ZDUxYjA3N2VhODdiMGMzODYwZDI4ZGE0M2Q1ZDZmMjIxZTg0NWM0YzIzNjBmzgAIAADOAAQAANlPbWVzaDpjZG46Y2h1bms6YzdmMWJkZTllMWVmYTdkMTM4NWQ1MWIwNzdlYTg3YjBjMzg2MGQyOGRhNDNkNWQ2ZjIyMWU4NDVjNGMyMzYwZpTZQGFhMTQ0NGJlNGI2NDA5OGY1ZGQ4NjNiNWIwYWM1NjE0ODBiM2ZiNWM0MWZjMDdjZjUwYTYyZWRjOTQxMjRiY2POAAwAAM4ABAAA2U9tZXNoOmNkbjpjaHVuazphYTE0NDRiZTRiNjQwOThmNWRkODYzYjViMGFjNTYxNDgwYjNmYjVjNDFmYzA3Y2Y1MGE2MmVkYzk0MTI0YmNjlNlAMjgzMTcxNTg1OWYwMWYzZjk2M2RkMGYxMTk0ZGJkYmI3NjA5OTk3NDIzMDE1ZDMxZjYwNzkxMWUxYmY4NjQwMs4AEAAAzgAEAADZT21lc2g6Y2RuOmNodW5rOjI4MzE3MTU4NTlmMDFmM2Y5NjNkZDBmMTE5NGRiZGJiNzYwOTk5NzQyMzAxNWQzMWY2MDc5MTFlMWJmODY0MDKU2UA1OTkzYWM2MDQwN2U4NjYwMTg3M2JjMzhlZTZmOGRiM2VhOTEyZTk4NzYxY2I1NThhYjk3OTVkYjMwNmI0NzRjzgAUAADNEt/ZT21lc2g6Y2RuOmNodW5rOjU5OTNhYzYwNDA3ZTg2NjAxODczYmMzOGVlNmY4ZGIzZWE5MTJlOTg3NjFjYjU1OGFiOTc5NWRiMzA2YjQ3NGOQgA=="
"messagePackBase64": "mrVldXJla2EvY29udGVudC12ZWN0b3KlYXNzZXShMdlAZjRhZTJhN2M5NWY0ZDk2NjA1OGY3NWMxMzE4NWFhMGFiNmE5MWZjNWFmYWQ4ODYyOTZlZjJhZWU3NTVhMmFmMc4AFBLfvGFwcGxpY2F0aW9uL3gtY3VsdG5ldC12ZWN0b3LZITIwMjYtMDktMjJUMDA6MDA6MDAuMDAwMDAwMCswMDowMJaV2UAzMWExZjlkZWEwMTY5NTUxMDkyZDA1ZThiZjRhNDQ2MjI4YzhjM2ViNGM5YjcxM2M2NmFkY2I3ZmQ1M2M4OWJlAM4ABAAA2U9tZXNoOmNkbjpjaHVuazozMWExZjlkZWEwMTY5NTUxMDkyZDA1ZThiZjRhNDQ2MjI4YzhjM2ViNGM5YjcxM2M2NmFkY2I3ZmQ1M2M4OWJloTCV2UBlMzFiZGM2MzU3N2ZlNzZmNjQ0MTAxNGFhYWEyMjI2ZDlmZmNlMTFiNDRkMGQ1MTBjYWRlMTMyYjc5ZmQ3ZDc2zgAEAADOAAQAANlPbWVzaDpjZG46Y2h1bms6ZTMxYmRjNjM1NzdmZTc2ZjY0NDEwMTRhYWFhMjIyNmQ5ZmZjZTExYjQ0ZDBkNTEwY2FkZTEzMmI3OWZkN2Q3NqYyNjIxNDSV2UBjN2YxYmRlOWUxZWZhN2QxMzg1ZDUxYjA3N2VhODdiMGMzODYwZDI4ZGE0M2Q1ZDZmMjIxZTg0NWM0YzIzNjBmzgAIAADOAAQAANlPbWVzaDpjZG46Y2h1bms6YzdmMWJkZTllMWVmYTdkMTM4NWQ1MWIwNzdlYTg3YjBjMzg2MGQyOGRhNDNkNWQ2ZjIyMWU4NDVjNGMyMzYwZqY1MjQyODiV2UBhYTE0NDRiZTRiNjQwOThmNWRkODYzYjViMGFjNTYxNDgwYjNmYjVjNDFmYzA3Y2Y1MGE2MmVkYzk0MTI0YmNjzgAMAADOAAQAANlPbWVzaDpjZG46Y2h1bms6YWExNDQ0YmU0YjY0MDk4ZjVkZDg2M2I1YjBhYzU2MTQ4MGIzZmI1YzQxZmMwN2NmNTBhNjJlZGM5NDEyNGJjY6Y3ODY0MzKV2UAyODMxNzE1ODU5ZjAxZjNmOTYzZGQwZjExOTRkYmRiYjc2MDk5OTc0MjMwMTVkMzFmNjA3OTExZTFiZjg2NDAyzgAQAADOAAQAANlPbWVzaDpjZG46Y2h1bms6MjgzMTcxNTg1OWYwMWYzZjk2M2RkMGYxMTk0ZGJkYmI3NjA5OTk3NDIzMDE1ZDMxZjYwNzkxMWUxYmY4NjQwMqcxMDQ4NTc2ldlANTk5M2FjNjA0MDdlODY2MDE4NzNiYzM4ZWU2ZjhkYjNlYTkxMmU5ODc2MWNiNTU4YWI5Nzk1ZGIzMDZiNDc0Y84AFAAAzRLf2U9tZXNoOmNkbjpjaHVuazo1OTkzYWM2MDQwN2U4NjYwMTg3M2JjMzhlZTZmOGRiM2VhOTEyZTk4NzYxY2I1NThhYjk3OTVkYjMwNmI0NzRjpzEzMTA3MjCQgA=="
},
{
"label": "manifest_out_of_order",
"kind": "manifest",
"messagePackBase64": "mrVldXJla2EvY29udGVudC12ZWN0b3KlYXNzZXShMdlAZjRhZTJhN2M5NWY0ZDk2NjA1OGY3NWMxMzE4NWFhMGFiNmE5MWZjNWFmYWQ4ODYyOTZlZjJhZWU3NTVhMmFmMc4AFBLfvGFwcGxpY2F0aW9uL3gtY3VsdG5ldC12ZWN0b3LZITIwMjYtMDktMjJUMDA6MDA6MDAuMDAwMDAwMCswMDowMJaU2UBhYTE0NDRiZTRiNjQwOThmNWRkODYzYjViMGFjNTYxNDgwYjNmYjVjNDFmYzA3Y2Y1MGE2MmVkYzk0MTI0YmNjzgAMAADOAAQAANlPbWVzaDpjZG46Y2h1bms6YWExNDQ0YmU0YjY0MDk4ZjVkZDg2M2I1YjBhYzU2MTQ4MGIzZmI1YzQxZmMwN2NmNTBhNjJlZGM5NDEyNGJjY5TZQDMxYTFmOWRlYTAxNjk1NTEwOTJkMDVlOGJmNGE0NDYyMjhjOGMzZWI0YzliNzEzYzY2YWRjYjdmZDUzYzg5YmUAzgAEAADZT21lc2g6Y2RuOmNodW5rOjMxYTFmOWRlYTAxNjk1NTEwOTJkMDVlOGJmNGE0NDYyMjhjOGMzZWI0YzliNzEzYzY2YWRjYjdmZDUzYzg5YmWU2UA1OTkzYWM2MDQwN2U4NjYwMTg3M2JjMzhlZTZmOGRiM2VhOTEyZTk4NzYxY2I1NThhYjk3OTVkYjMwNmI0NzRjzgAUAADNEt/ZT21lc2g6Y2RuOmNodW5rOjU5OTNhYzYwNDA3ZTg2NjAxODczYmMzOGVlNmY4ZGIzZWE5MTJlOTg3NjFjYjU1OGFiOTc5NWRiMzA2YjQ3NGOU2UBlMzFiZGM2MzU3N2ZlNzZmNjQ0MTAxNGFhYWEyMjI2ZDlmZmNlMTFiNDRkMGQ1MTBjYWRlMTMyYjc5ZmQ3ZDc2zgAEAADOAAQAANlPbWVzaDpjZG46Y2h1bms6ZTMxYmRjNjM1NzdmZTc2ZjY0NDEwMTRhYWFhMjIyNmQ5ZmZjZTExYjQ0ZDBkNTEwY2FkZTEzMmI3OWZkN2Q3NpTZQDI4MzE3MTU4NTlmMDFmM2Y5NjNkZDBmMTE5NGRiZGJiNzYwOTk5NzQyMzAxNWQzMWY2MDc5MTFlMWJmODY0MDLOABAAAM4ABAAA2U9tZXNoOmNkbjpjaHVuazoyODMxNzE1ODU5ZjAxZjNmOTYzZGQwZjExOTRkYmRiYjc2MDk5OTc0MjMwMTVkMzFmNjA3OTExZTFiZjg2NDAylNlAYzdmMWJkZTllMWVmYTdkMTM4NWQ1MWIwNzdlYTg3YjBjMzg2MGQyOGRhNDNkNWQ2ZjIyMWU4NDVjNGMyMzYwZs4ACAAAzgAEAADZT21lc2g6Y2RuOmNodW5rOmM3ZjFiZGU5ZTFlZmE3ZDEzODVkNTFiMDc3ZWE4N2IwYzM4NjBkMjhkYTQzZDVkNmYyMjFlODQ1YzRjMjM2MGaQgA=="
"messagePackBase64": "mrVldXJla2EvY29udGVudC12ZWN0b3KlYXNzZXShMdlAZjRhZTJhN2M5NWY0ZDk2NjA1OGY3NWMxMzE4NWFhMGFiNmE5MWZjNWFmYWQ4ODYyOTZlZjJhZWU3NTVhMmFmMc4AFBLfvGFwcGxpY2F0aW9uL3gtY3VsdG5ldC12ZWN0b3LZITIwMjYtMDktMjJUMDA6MDA6MDAuMDAwMDAwMCswMDowMJaV2UBhYTE0NDRiZTRiNjQwOThmNWRkODYzYjViMGFjNTYxNDgwYjNmYjVjNDFmYzA3Y2Y1MGE2MmVkYzk0MTI0YmNjzgAMAADOAAQAANlPbWVzaDpjZG46Y2h1bms6YWExNDQ0YmU0YjY0MDk4ZjVkZDg2M2I1YjBhYzU2MTQ4MGIzZmI1YzQxZmMwN2NmNTBhNjJlZGM5NDEyNGJjY6Y3ODY0MzKV2UAzMWExZjlkZWEwMTY5NTUxMDkyZDA1ZThiZjRhNDQ2MjI4YzhjM2ViNGM5YjcxM2M2NmFkY2I3ZmQ1M2M4OWJlAM4ABAAA2U9tZXNoOmNkbjpjaHVuazozMWExZjlkZWEwMTY5NTUxMDkyZDA1ZThiZjRhNDQ2MjI4YzhjM2ViNGM5YjcxM2M2NmFkY2I3ZmQ1M2M4OWJloTCV2UA1OTkzYWM2MDQwN2U4NjYwMTg3M2JjMzhlZTZmOGRiM2VhOTEyZTk4NzYxY2I1NThhYjk3OTVkYjMwNmI0NzRjzgAUAADNEt/ZT21lc2g6Y2RuOmNodW5rOjU5OTNhYzYwNDA3ZTg2NjAxODczYmMzOGVlNmY4ZGIzZWE5MTJlOTg3NjFjYjU1OGFiOTc5NWRiMzA2YjQ3NGOnMTMxMDcyMJXZQGUzMWJkYzYzNTc3ZmU3NmY2NDQxMDE0YWFhYTIyMjZkOWZmY2UxMWI0NGQwZDUxMGNhZGUxMzJiNzlmZDdkNzbOAAQAAM4ABAAA2U9tZXNoOmNkbjpjaHVuazplMzFiZGM2MzU3N2ZlNzZmNjQ0MTAxNGFhYWEyMjI2ZDlmZmNlMTFiNDRkMGQ1MTBjYWRlMTMyYjc5ZmQ3ZDc2pjI2MjE0NJXZQDI4MzE3MTU4NTlmMDFmM2Y5NjNkZDBmMTE5NGRiZGJiNzYwOTk5NzQyMzAxNWQzMWY2MDc5MTFlMWJmODY0MDLOABAAAM4ABAAA2U9tZXNoOmNkbjpjaHVuazoyODMxNzE1ODU5ZjAxZjNmOTYzZGQwZjExOTRkYmRiYjc2MDk5OTc0MjMwMTVkMzFmNjA3OTExZTFiZjg2NDAypzEwNDg1NzaV2UBjN2YxYmRlOWUxZWZhN2QxMzg1ZDUxYjA3N2VhODdiMGMzODYwZDI4ZGE0M2Q1ZDZmMjIxZTg0NWM0YzIzNjBmzgAIAADOAAQAANlPbWVzaDpjZG46Y2h1bms6YzdmMWJkZTllMWVmYTdkMTM4NWQ1MWIwNzdlYTg3YjBjMzg2MGQyOGRhNDNkNWQ2ZjIyMWU4NDVjNGMyMzYwZqY1MjQyODiQgA=="
},
{
"label": "manifest_metadata_order",
"kind": "manifest",
"messagePackBase64": "mrVldXJla2EvY29udGVudC12ZWN0b3KlYXNzZXShMdlAZjRhZTJhN2M5NWY0ZDk2NjA1OGY3NWMxMzE4NWFhMGFiNmE5MWZjNWFmYWQ4ODYyOTZlZjJhZWU3NTVhMmFmMc4AFBLfvGFwcGxpY2F0aW9uL3gtY3VsdG5ldC12ZWN0b3LZITIwMjYtMDktMjJUMDA6MDA6MDAuMDAwMDAwMCswMDowMJaU2UAzMWExZjlkZWEwMTY5NTUxMDkyZDA1ZThiZjRhNDQ2MjI4YzhjM2ViNGM5YjcxM2M2NmFkY2I3ZmQ1M2M4OWJlAM4ABAAA2U9tZXNoOmNkbjpjaHVuazozMWExZjlkZWEwMTY5NTUxMDkyZDA1ZThiZjRhNDQ2MjI4YzhjM2ViNGM5YjcxM2M2NmFkY2I3ZmQ1M2M4OWJllNlAZTMxYmRjNjM1NzdmZTc2ZjY0NDEwMTRhYWFhMjIyNmQ5ZmZjZTExYjQ0ZDBkNTEwY2FkZTEzMmI3OWZkN2Q3Ns4ABAAAzgAEAADZT21lc2g6Y2RuOmNodW5rOmUzMWJkYzYzNTc3ZmU3NmY2NDQxMDE0YWFhYTIyMjZkOWZmY2UxMWI0NGQwZDUxMGNhZGUxMzJiNzlmZDdkNzaU2UBjN2YxYmRlOWUxZWZhN2QxMzg1ZDUxYjA3N2VhODdiMGMzODYwZDI4ZGE0M2Q1ZDZmMjIxZTg0NWM0YzIzNjBmzgAIAADOAAQAANlPbWVzaDpjZG46Y2h1bms6YzdmMWJkZTllMWVmYTdkMTM4NWQ1MWIwNzdlYTg3YjBjMzg2MGQyOGRhNDNkNWQ2ZjIyMWU4NDVjNGMyMzYwZpTZQGFhMTQ0NGJlNGI2NDA5OGY1ZGQ4NjNiNWIwYWM1NjE0ODBiM2ZiNWM0MWZjMDdjZjUwYTYyZWRjOTQxMjRiY2POAAwAAM4ABAAA2U9tZXNoOmNkbjpjaHVuazphYTE0NDRiZTRiNjQwOThmNWRkODYzYjViMGFjNTYxNDgwYjNmYjVjNDFmYzA3Y2Y1MGE2MmVkYzk0MTI0YmNjlNlAMjgzMTcxNTg1OWYwMWYzZjk2M2RkMGYxMTk0ZGJkYmI3NjA5OTk3NDIzMDE1ZDMxZjYwNzkxMWUxYmY4NjQwMs4AEAAAzgAEAADZT21lc2g6Y2RuOmNodW5rOjI4MzE3MTU4NTlmMDFmM2Y5NjNkZDBmMTE5NGRiZGJiNzYwOTk5NzQyMzAxNWQzMWY2MDc5MTFlMWJmODY0MDKU2UA1OTkzYWM2MDQwN2U4NjYwMTg3M2JjMzhlZTZmOGRiM2VhOTEyZTk4NzYxY2I1NThhYjk3OTVkYjMwNmI0NzRjzgAUAADNEt/ZT21lc2g6Y2RuOmNodW5rOjU5OTNhYzYwNDA3ZTg2NjAxODczYmMzOGVlNmY4ZGIzZWE5MTJlOTg3NjFjYjU1OGFiOTc5NWRiMzA2YjQ3NGOQg6R6ZXRhoTGlYWxwaGGhMqNtaWShMw=="
"messagePackBase64": "mrVldXJla2EvY29udGVudC12ZWN0b3KlYXNzZXShMdlAZjRhZTJhN2M5NWY0ZDk2NjA1OGY3NWMxMzE4NWFhMGFiNmE5MWZjNWFmYWQ4ODYyOTZlZjJhZWU3NTVhMmFmMc4AFBLfvGFwcGxpY2F0aW9uL3gtY3VsdG5ldC12ZWN0b3LZITIwMjYtMDktMjJUMDA6MDA6MDAuMDAwMDAwMCswMDowMJaV2UAzMWExZjlkZWEwMTY5NTUxMDkyZDA1ZThiZjRhNDQ2MjI4YzhjM2ViNGM5YjcxM2M2NmFkY2I3ZmQ1M2M4OWJlAM4ABAAA2U9tZXNoOmNkbjpjaHVuazozMWExZjlkZWEwMTY5NTUxMDkyZDA1ZThiZjRhNDQ2MjI4YzhjM2ViNGM5YjcxM2M2NmFkY2I3ZmQ1M2M4OWJloTCV2UBlMzFiZGM2MzU3N2ZlNzZmNjQ0MTAxNGFhYWEyMjI2ZDlmZmNlMTFiNDRkMGQ1MTBjYWRlMTMyYjc5ZmQ3ZDc2zgAEAADOAAQAANlPbWVzaDpjZG46Y2h1bms6ZTMxYmRjNjM1NzdmZTc2ZjY0NDEwMTRhYWFhMjIyNmQ5ZmZjZTExYjQ0ZDBkNTEwY2FkZTEzMmI3OWZkN2Q3NqYyNjIxNDSV2UBjN2YxYmRlOWUxZWZhN2QxMzg1ZDUxYjA3N2VhODdiMGMzODYwZDI4ZGE0M2Q1ZDZmMjIxZTg0NWM0YzIzNjBmzgAIAADOAAQAANlPbWVzaDpjZG46Y2h1bms6YzdmMWJkZTllMWVmYTdkMTM4NWQ1MWIwNzdlYTg3YjBjMzg2MGQyOGRhNDNkNWQ2ZjIyMWU4NDVjNGMyMzYwZqY1MjQyODiV2UBhYTE0NDRiZTRiNjQwOThmNWRkODYzYjViMGFjNTYxNDgwYjNmYjVjNDFmYzA3Y2Y1MGE2MmVkYzk0MTI0YmNjzgAMAADOAAQAANlPbWVzaDpjZG46Y2h1bms6YWExNDQ0YmU0YjY0MDk4ZjVkZDg2M2I1YjBhYzU2MTQ4MGIzZmI1YzQxZmMwN2NmNTBhNjJlZGM5NDEyNGJjY6Y3ODY0MzKV2UAyODMxNzE1ODU5ZjAxZjNmOTYzZGQwZjExOTRkYmRiYjc2MDk5OTc0MjMwMTVkMzFmNjA3OTExZTFiZjg2NDAyzgAQAADOAAQAANlPbWVzaDpjZG46Y2h1bms6MjgzMTcxNTg1OWYwMWYzZjk2M2RkMGYxMTk0ZGJkYmI3NjA5OTk3NDIzMDE1ZDMxZjYwNzkxMWUxYmY4NjQwMqcxMDQ4NTc2ldlANTk5M2FjNjA0MDdlODY2MDE4NzNiYzM4ZWU2ZjhkYjNlYTkxMmU5ODc2MWNiNTU4YWI5Nzk1ZGIzMDZiNDc0Y84AFAAAzRLf2U9tZXNoOmNkbjpjaHVuazo1OTkzYWM2MDQwN2U4NjYwMTg3M2JjMzhlZTZmOGRiM2VhOTEyZTk4NzYxY2I1NThhYjk3OTVkYjMwNmI0NzRjpzEzMTA3MjCQg6R6ZXRhoTGlYWxwaGGhMqNtaWShMw=="
},
{
"label": "manifest_empty_body",
"kind": "manifest",
"messagePackBase64": "mrVldXJla2EvY29udGVudC12ZWN0b3KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQC8YXBwbGljYXRpb24veC1jdWx0bmV0LXZlY3RvctkhMjAyNi0wOS0yMlQwMDowMDowMC4wMDAwMDAwKzAwOjAwkZTZQGUzYjBjNDQyOThmYzFjMTQ5YWZiZjRjODk5NmZiOTI0MjdhZTQxZTQ2NDliOTM0Y2E0OTU5OTFiNzg1MmI4NTUAANlPbWVzaDpjZG46Y2h1bms6ZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NZCA"
"messagePackBase64": "mrVldXJla2EvY29udGVudC12ZWN0b3KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQC8YXBwbGljYXRpb24veC1jdWx0bmV0LXZlY3RvctkhMjAyNi0wOS0yMlQwMDowMDowMC4wMDAwMDAwKzAwOjAwkZXZQGUzYjBjNDQyOThmYzFjMTQ5YWZiZjRjODk5NmZiOTI0MjdhZTQxZTQ2NDliOTM0Y2E0OTU5OTFiNzg1MmI4NTUAANlPbWVzaDpjZG46Y2h1bms6ZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NaEwkIA="
}
]
}
}

8
contracts/cultmesh/content-vectors.ts-written.json
View file

@ -35,7 +35,7 @@
}
},
"recordKey": "mesh:cdn:artifact:pixel-heroes-hüt:unversioned:cf8a453a36981321977355448d24df49bcefee22be6e3730b9140e87c29013f9",
"messagePackBase64": "mrVQaXhlbCBIZXJvZXMvSMO8dCDimIOlYXNzZXSg2UBjZjhhNDUzYTM2OTgxMzIxOTc3MzU1NDQ4ZDI0ZGY0OWJjZWZlZTIyYmU2ZTM3MzBiOTE0MGU4N2MyOTAxM2Y5zgAL3jG/YXBwbGljYXRpb24veC1jdWx0bmV0LXRzLXZlY3RvctkhMjAyNi0xMC0xOVQwMDowMDowMC4wMDAwMDAwKzAwOjAwlpTZQDM3N2M3YjBjYzIyOWY0NzA1NjcwZDYzZDU0NGMyNzViNjYzZGY2ZThjODMxYjA5ODQ2YTVjNDhjY2EwOGI4NDgAzgACAADZT21lc2g6Y2RuOmNodW5rOjM3N2M3YjBjYzIyOWY0NzA1NjcwZDYzZDU0NGMyNzViNjYzZGY2ZThjODMxYjA5ODQ2YTVjNDhjY2EwOGI4NDiU2UAzNzdjN2IwY2MyMjlmNDcwNTY3MGQ2M2Q1NDRjMjc1YjY2M2RmNmU4YzgzMWIwOTg0NmE1YzQ4Y2NhMDhiODQ4zgACAADOAAIAANlPbWVzaDpjZG46Y2h1bms6Mzc3YzdiMGNjMjI5ZjQ3MDU2NzBkNjNkNTQ0YzI3NWI2NjNkZjZlOGM4MzFiMDk4NDZhNWM0OGNjYTA4Yjg0OJTZQDM3N2M3YjBjYzIyOWY0NzA1NjcwZDYzZDU0NGMyNzViNjYzZGY2ZThjODMxYjA5ODQ2YTVjNDhjY2EwOGI4NDjOAAQAAM4AAgAA2U9tZXNoOmNkbjpjaHVuazozNzdjN2IwY2MyMjlmNDcwNTY3MGQ2M2Q1NDRjMjc1YjY2M2RmNmU4YzgzMWIwOTg0NmE1YzQ4Y2NhMDhiODQ4lNlAMzc3YzdiMGNjMjI5ZjQ3MDU2NzBkNjNkNTQ0YzI3NWI2NjNkZjZlOGM4MzFiMDk4NDZhNWM0OGNjYTA4Yjg0OM4ABgAAzgACAADZT21lc2g6Y2RuOmNodW5rOjM3N2M3YjBjYzIyOWY0NzA1NjcwZDYzZDU0NGMyNzViNjYzZGY2ZThjODMxYjA5ODQ2YTVjNDhjY2EwOGI4NDiU2UAzNzdjN2IwY2MyMjlmNDcwNTY3MGQ2M2Q1NDRjMjc1YjY2M2RmNmU4YzgzMWIwOTg0NmE1YzQ4Y2NhMDhiODQ4zgAIAADOAAIAANlPbWVzaDpjZG46Y2h1bms6Mzc3YzdiMGNjMjI5ZjQ3MDU2NzBkNjNkNTQ0YzI3NWI2NjNkZjZlOGM4MzFiMDk4NDZhNWM0OGNjYTA4Yjg0OJTZQGEwYmI2OWY0NDg2MWViM2RiNTYwMjZhMWFiYmY5MWZmYTYyMTMyNTA4ZDVhMDZlMGQwZGRjNWNjMjZiZTMzZmPOAAoAAM4AAd4x2U9tZXNoOmNkbjpjaHVuazphMGJiNjlmNDQ4NjFlYjNkYjU2MDI2YTFhYmJmOTFmZmE2MjEzMjUwOGQ1YTA2ZTBkMGRkYzVjYzI2YmUzM2ZjkqFhoWKDojEwoXilYWxwaGGheaEyoXo="
"messagePackBase64": "mrVQaXhlbCBIZXJvZXMvSMO8dCDimIOlYXNzZXSg2UBjZjhhNDUzYTM2OTgxMzIxOTc3MzU1NDQ4ZDI0ZGY0OWJjZWZlZTIyYmU2ZTM3MzBiOTE0MGU4N2MyOTAxM2Y5zgAL3jG/YXBwbGljYXRpb24veC1jdWx0bmV0LXRzLXZlY3RvctkhMjAyNi0xMC0xOVQwMDowMDowMC4wMDAwMDAwKzAwOjAwlpXZQDM3N2M3YjBjYzIyOWY0NzA1NjcwZDYzZDU0NGMyNzViNjYzZGY2ZThjODMxYjA5ODQ2YTVjNDhjY2EwOGI4NDgAzgACAADZT21lc2g6Y2RuOmNodW5rOjM3N2M3YjBjYzIyOWY0NzA1NjcwZDYzZDU0NGMyNzViNjYzZGY2ZThjODMxYjA5ODQ2YTVjNDhjY2EwOGI4NDihMJXZQDM3N2M3YjBjYzIyOWY0NzA1NjcwZDYzZDU0NGMyNzViNjYzZGY2ZThjODMxYjA5ODQ2YTVjNDhjY2EwOGI4NDjOAAIAAM4AAgAA2U9tZXNoOmNkbjpjaHVuazozNzdjN2IwY2MyMjlmNDcwNTY3MGQ2M2Q1NDRjMjc1YjY2M2RmNmU4YzgzMWIwOTg0NmE1YzQ4Y2NhMDhiODQ4pjEzMTA3MpXZQDM3N2M3YjBjYzIyOWY0NzA1NjcwZDYzZDU0NGMyNzViNjYzZGY2ZThjODMxYjA5ODQ2YTVjNDhjY2EwOGI4NDjOAAQAAM4AAgAA2U9tZXNoOmNkbjpjaHVuazozNzdjN2IwY2MyMjlmNDcwNTY3MGQ2M2Q1NDRjMjc1YjY2M2RmNmU4YzgzMWIwOTg0NmE1YzQ4Y2NhMDhiODQ4pjI2MjE0NJXZQDM3N2M3YjBjYzIyOWY0NzA1NjcwZDYzZDU0NGMyNzViNjYzZGY2ZThjODMxYjA5ODQ2YTVjNDhjY2EwOGI4NDjOAAYAAM4AAgAA2U9tZXNoOmNkbjpjaHVuazozNzdjN2IwY2MyMjlmNDcwNTY3MGQ2M2Q1NDRjMjc1YjY2M2RmNmU4YzgzMWIwOTg0NmE1YzQ4Y2NhMDhiODQ4pjM5MzIxNpXZQDM3N2M3YjBjYzIyOWY0NzA1NjcwZDYzZDU0NGMyNzViNjYzZGY2ZThjODMxYjA5ODQ2YTVjNDhjY2EwOGI4NDjOAAgAAM4AAgAA2U9tZXNoOmNkbjpjaHVuazozNzdjN2IwY2MyMjlmNDcwNTY3MGQ2M2Q1NDRjMjc1YjY2M2RmNmU4YzgzMWIwOTg0NmE1YzQ4Y2NhMDhiODQ4pjUyNDI4OJXZQGEwYmI2OWY0NDg2MWViM2RiNTYwMjZhMWFiYmY5MWZmYTYyMTMyNTA4ZDVhMDZlMGQwZGRjNWNjMjZiZTMzZmPOAAoAAM4AAd4x2U9tZXNoOmNkbjpjaHVuazphMGJiNjlmNDQ4NjFlYjNkYjU2MDI2YTFhYmJmOTFmZmE2MjEzMjUwOGQ1YTA2ZTBkMGRkYzVjYzI2YmUzM2ZjpjY1NTM2MJKhYaFig6IxMKF4pWFscGhhoXmhMqF6"
},
{
"label": "ts empty body",
@ -72,7 +72,7 @@
}
},
"recordKey": "mesh:cdn:artifact:pixel-heroes-hüt:unversioned:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
"messagePackBase64": "mrVQaXhlbCBIZXJvZXMvSMO8dCDimIOlYXNzZXSg2UBlM2IwYzQ0Mjk4ZmMxYzE0OWFmYmY0Yzg5OTZmYjkyNDI3YWU0MWU0NjQ5YjkzNGNhNDk1OTkxYjc4NTJiODU1AL9hcHBsaWNhdGlvbi94LWN1bHRuZXQtdHMtdmVjdG9y2SEyMDI2LTEwLTE5VDAwOjAwOjAwLjAwMDAwMDArMDA6MDCRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAA2U9tZXNoOmNkbjpjaHVuazplM2IwYzQ0Mjk4ZmMxYzE0OWFmYmY0Yzg5OTZmYjkyNDI3YWU0MWU0NjQ5YjkzNGNhNDk1OTkxYjc4NTJiODU1kqFhoWKDojEwoXilYWxwaGGheaEyoXo="
"messagePackBase64": "mrVQaXhlbCBIZXJvZXMvSMO8dCDimIOlYXNzZXSg2UBlM2IwYzQ0Mjk4ZmMxYzE0OWFmYmY0Yzg5OTZmYjkyNDI3YWU0MWU0NjQ5YjkzNGNhNDk1OTkxYjc4NTJiODU1AL9hcHBsaWNhdGlvbi94LWN1bHRuZXQtdHMtdmVjdG9y2SEyMDI2LTEwLTE5VDAwOjAwOjAwLjAwMDAwMDArMDA6MDCRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAA2U9tZXNoOmNkbjpjaHVuazplM2IwYzQ0Mjk4ZmMxYzE0OWFmYmY0Yzg5OTZmYjkyNDI3YWU0MWU0NjQ5YjkzNGNhNDk1OTkxYjc4NTJiODU1oTCSoWGhYoOiMTCheKVhbHBoYaF5oTKheg=="
},
{
"label": "ts single chunk, versioned",
@ -93,7 +93,7 @@
}
},
"recordKey": "mesh:cdn:artifact:pixel-heroes-hüt:v1-beta:89f4ff56a25dd1db06a4ce6033603775d705fb96f30f8693733fef602a1ca532",
"messagePackBase64": "mrVQaXhlbCBIZXJvZXMvSMO8dCDimIOlYXNzZXSnVjEgQmV0YdlAODlmNGZmNTZhMjVkZDFkYjA2YTRjZTYwMzM2MDM3NzVkNzA1ZmI5NmYzMGY4NjkzNzMzZmVmNjAyYTFjYTUzMs0D6L9hcHBsaWNhdGlvbi94LWN1bHRuZXQtdHMtdmVjdG9y2SEyMDI2LTEwLTE5VDAwOjAwOjAwLjAwMDAwMDArMDA6MDCRlNlAODlmNGZmNTZhMjVkZDFkYjA2YTRjZTYwMzM2MDM3NzVkNzA1ZmI5NmYzMGY4NjkzNzMzZmVmNjAyYTFjYTUzMgDNA+jZT21lc2g6Y2RuOmNodW5rOjg5ZjRmZjU2YTI1ZGQxZGIwNmE0Y2U2MDMzNjAzNzc1ZDcwNWZiOTZmMzBmODY5MzczM2ZlZjYwMmExY2E1MzKQgA=="
"messagePackBase64": "mrVQaXhlbCBIZXJvZXMvSMO8dCDimIOlYXNzZXSnVjEgQmV0YdlAODlmNGZmNTZhMjVkZDFkYjA2YTRjZTYwMzM2MDM3NzVkNzA1ZmI5NmYzMGY4NjkzNzMzZmVmNjAyYTFjYTUzMs0D6L9hcHBsaWNhdGlvbi94LWN1bHRuZXQtdHMtdmVjdG9y2SEyMDI2LTEwLTE5VDAwOjAwOjAwLjAwMDAwMDArMDA6MDCRldlAODlmNGZmNTZhMjVkZDFkYjA2YTRjZTYwMzM2MDM3NzVkNzA1ZmI5NmYzMGY4NjkzNzMzZmVmNjAyYTFjYTUzMgDNA+jZT21lc2g6Y2RuOmNodW5rOjg5ZjRmZjU2YTI1ZGQxZGIwNmE0Y2U2MDMzNjAzNzc1ZDcwNWZiOTZmMzBmODY5MzczM2ZlZjYwMmExY2E1MzKhMJCA"
},
{
"label": "ts whitespace version, padded kind and mime",
@ -114,7 +114,7 @@
}
},
"recordKey": "mesh:cdn:artifact:pixel-heroes-hüt:unversioned:89f4ff56a25dd1db06a4ce6033603775d705fb96f30f8693733fef602a1ca532",
"messagePackBase64": "mrVQaXhlbCBIZXJvZXMvSMO8dCDimIOnIGJ1aWxkIKIgINlAODlmNGZmNTZhMjVkZDFkYjA2YTRjZTYwMzM2MDM3NzVkNzA1ZmI5NmYzMGY4NjkzNzMzZmVmNjAyYTFjYTUzMs0D6L0gIHRleHQvcGxhaW47IGNoYXJzZXQ9dXRmLTggINkhMjAyNi0xMC0xOVQwMDowMDowMC4wMDAwMDAwKzAwOjAwkZTZQDg5ZjRmZjU2YTI1ZGQxZGIwNmE0Y2U2MDMzNjAzNzc1ZDcwNWZiOTZmMzBmODY5MzczM2ZlZjYwMmExY2E1MzIAzQPo2U9tZXNoOmNkbjpjaHVuazo4OWY0ZmY1NmEyNWRkMWRiMDZhNGNlNjAzMzYwMzc3NWQ3MDVmYjk2ZjMwZjg2OTM3MzNmZWY2MDJhMWNhNTMykIA="
"messagePackBase64": "mrVQaXhlbCBIZXJvZXMvSMO8dCDimIOnIGJ1aWxkIKIgINlAODlmNGZmNTZhMjVkZDFkYjA2YTRjZTYwMzM2MDM3NzVkNzA1ZmI5NmYzMGY4NjkzNzMzZmVmNjAyYTFjYTUzMs0D6L0gIHRleHQvcGxhaW47IGNoYXJzZXQ9dXRmLTggINkhMjAyNi0xMC0xOVQwMDowMDowMC4wMDAwMDAwKzAwOjAwkZXZQDg5ZjRmZjU2YTI1ZGQxZGIwNmE0Y2U2MDMzNjAzNzc1ZDcwNWZiOTZmMzBmODY5MzczM2ZlZjYwMmExY2E1MzIAzQPo2U9tZXNoOmNkbjpjaHVuazo4OWY0ZmY1NmEyNWRkMWRiMDZhNGNlNjAzMzYwMzc3NWQ3MDVmYjk2ZjMwZjg2OTM3MzNmZWY2MDJhMWNhNTMyoTCQgA=="
}
]
}

3
contracts/cultnet/cultnet.witness-artifact-bundle.schema.json
View file

@ -59,6 +59,7 @@
],
"additionalProperties": false,
"properties": {
"id": { "type": "string", "minLength": 1 },
"role": { "type": "string", "minLength": 1 },
"schemaId": { "type": "string", "minLength": 1 },
"schemaVersion": { "type": "string", "minLength": 1 },
@ -74,6 +75,7 @@
],
"additionalProperties": false,
"properties": {
"id": { "type": "string", "minLength": 1 },
"role": { "type": "string", "minLength": 1 },
"uri": { "type": "string", "minLength": 1 },
"mediaType": { "type": "string", "minLength": 1 },
@ -92,6 +94,7 @@
],
"additionalProperties": false,
"properties": {
"id": { "type": "string", "minLength": 1 },
"stage": { "type": "string", "minLength": 1 },
"startedAt": { "type": "string", "minLength": 1 },
"completedAt": { "type": "string", "minLength": 1 },

441
docs/c2a-land-map.md Normal file
View file

@ -0,0 +1,441 @@
# C2a land map
What C2a (`hands/variants-c2a`, a5c7954e) must still prove to merge to main, measured
on 2026-10-22 against main 7a8933b0 and the 2.0.0 release candidate
`hands/cultlib-unity-release-2.0.0` (5b5d8573, which carries f91e30ed). The land cut
is the merge plus the gate. Everything else is classified below. Governing ruling:
`cultlib-gaps:ruling:release-now-c2a-bumps-format`.
## Cut sequence to main (2026-10-27, branch head 7c7cc799)
Landed since the facts below: `land-c2a` (h1), `land-c2a-merge-api-docs` (h1-h6), the 2.0.1 and
cultcache-rs 0.3.1 merges (5baa02fa, f54472e5), `land-c2a-gate` r4 (h1), `store-gate-one-owner`
(h1-h7, Soul s6) and `restamp-on-write` (h1-h2, 8ee66714..7c7cc799, Soul s2 holds). One static `Owns`
decides ownership for cache and cacheless writers, and the High finding
`cut-land-c2a-gate.s1.reflist-record-read-only-at-head` is Fixed by 7c7cc799. origin/main is 73e684b1:
21 commits past the merge base 02cf04b5, all docs and research (15 files, +1879, none under src/,
tests/ or packages/, none on a path the branch changed). What stands between the branch and main, in
order:
1. **claimant-one-owner** (spec r1 at 7c7cc799, Hands then Soul). Restamp's Soul s2 left four Lows
on the landing path. Which entry reads a persisted entry is chosen twice (`Claimant` at
CultCache.cs:703-750, `ClaimantEntry` at :797-799), and so a cacheless writer handed registry {A,B}
overwrites a record that a cache with {A,B} holds read-only (probe X4). The cut keeps one `Claimant`
precedence over a by-id/by-name lookup that the registry or a single entry supplies. A cacheless
writer with a registry then takes its claimant from that registry. The cut also pins the
unpublished-id case and fixes the stale test comment. It closes `claimant-selection-duplicated`,
`cacheless-inverts-registry-claimant`, `unpublished-id-guard-unpinned` and `stale-flag-copy-comment`.
It is kept apart from the gate so that Soul can falsify it alone. Its files (CultCache.cs, the
persistence contract, RestampOnWriteTests.cs) are disjoint from the gate's, so it can run before or
beside step 2. The second cut to land rebases.
2. **land-c2a-gate** (spec r7 at 7c7cc799, Hands h2 then Soul). The ref-list gate does the same write
at 2.0.1 and at head and compares schema ids. The committed vectors are pinned byte for byte under a
fixed clock, and every stage counts its cases. r7 drops r6's head-only restamp assertions, which
`RestampOnWriteTests.TheCommitted1060VectorsListRecordIsWrittenUnderTheHoldersOwnId` already pins.
It also drops the red-at-193cf1c3 run, because restamp is measured. The cut closes
`single-vector-hand-edit-passes`, `gate-green-with-no-refusal-probes` and follow-up
`gate-reads-1x-written-store`.
3. **Ledger** (Self, no Hands; can run now, beside steps 1 and 2). Each of these is in force though
its fix has landed:
- The seven `cut-land-c2a.s1.*` findings. `land-c2a-merge-api-docs` carried their pins (source reads
2026-10-27: GameCult.Caching grants InternalsVisibleTo only to `.Tests` and `.MessagePack`, the
residual being follow-up `caching-messagepack-ivt`; PreCut2StoreFormatTests pins "with a suffix
of"). Close each one against its commit, or defer it. `stale-unity-plugin-dlls` defers to the
Unity 3.0.0 release cut (the plugins are 2.0.1 builds, 1eeb1a01).
- `cut-c2a-write-set.s1.untouched-dropped-slot-resurrects` and follow-up `slot-reuse-contract-doc`.
Ruling `slot-reuse-documented` decided the behaviour, and the paragraph is in
cultcache-schema-compatibility.md:64-66.
- `cut-restamp-on-write.s1.websockets-tests-unreported`, Fixed by h2 (s2 promise
`websockets-tests-reported` holds).
- Follow-up `gate-bound-test-tighten`, carried by restamp r2 (50362447, a768b821). Close it if
restamp's Soul passes measured it; otherwise route it.
- After steps 1 and 2 hold: the four s2 Lows, the two remaining `cut-land-c2a-gate.s1` Lows and
`gate-reads-1x-written-store`.
4. **Consumer pins** (ruling `unpinned-consumers-at-land-3`, follow-up `consumer-pins-before-land`).
Brokkr, Aquarium, Ymir, AquaSynth and Mimir have no `CultLibRevision`/`VerifyCultLibRevision` guard,
and each builds against the sibling `..\CultLib` checkout. In its own repo, each adopts Aetheria's
guard at the 2.0.0 tag, with one Hands per repo, before the merge reaches main. The pins touch no
CultLib file, so they can start now and run in parallel with steps 1-3. Eve and Gjallar get no pin
(ruled). The step is done when each repo's guard fails a build against a CultLib checkout at
another revision and passes at the tag, and the follow-up is resolved.
5. **Main into the branch, then full verification.** After steps 1, 2 and 4, `git merge origin/main`
into `hands/variants-c2a`. It should be clean, because main's commits since 02cf04b5 are docs and
research only. Re-check that before merging (`git diff --stat 02cf04b5 origin/main -- src tests
packages unity`). Then run every suite at the merged head on Yggdrasil: Caching, Mesh, Networking,
Geometry, WebSockets, ConsumerBuild, InteropPeer, cultcache-ts/-rs/-py, cultnet-ts and cultmesh-ts.
Run `old-reader-refusal.sh gate` for csharp, typescript, python and rust. If main gained code by then,
a Soul pass covers the merge.
6. **Land.** The land is `git merge --no-ff`, never a squash, so the SHAs that every report and verdict cites stay reachable from main (skip the branch's non-building commits with `git bisect --first-parent`). Steps 1 and 2 closed at 723b7996 (claimant-one-owner, land-c2a-gate); step 5 closed at the merge commit 3157d4b3 (Soul `cut-land-c2a-main.s1` holds). Step 3 closed except `cut-land-c2a.s1.stale-unity-plugin-dlls` and `cut-land-c2a-gate.s2.mutation-table-reads-gate-as-zero`, which wait on the 3.0.0 release cut and the Eureka tools respectively; step 4 closed with `consumer-pins-before-land` (pins since 2026-10-22). Run `git merge --no-ff hands/variants-c2a` onto main, with an empty diff against the
verified head. There is no tag: tags, the Unity plugin rebuild and `declared-range-gate-own-version`
belong to the 3.0.0 release cut. Open follow-ups `c2a-load-slower` (load 2 to 2.3 times slower than
main on a synthetic Aetheria catalog) and `drift-write-ownership-leaves` do not say "before land".
`c2a-load-slower` still needs a reason or a fix before the 3.0.0 release, under
`cultlib-gaps:ruling:database-shaped-built-for-load`.
## Facts
**1. The merge.** A trial `git merge origin/main` on a5c7954e gives 12 conflicted files and
33 hunks of about 950 marker lines (CultCache.cs 17 hunks/196; cult-cache.test.ts 337;
single-file-messagepack-backing-store.ts 4/106; rs lib.rs 2/51; VariantTests 51; three
CHANGELOG add/adds; store-composition contract 32; Directory store 14; InteropPeer 21).
Main moved 4.8k lines in the cache runtimes since base 1a12bc2d. Three semantic collisions
are not textual:
- *Durable record identity has two owners.* Main added `CultStoredDocument.PersistedSchemaId`
and `CacheBackingStore.MarkWritten` (73d438db, afd799ba, 3f6dda8b). Conditional commit
compares the persisted schema id plus storedAt. C2a already owns this as
`StoredSchemaId` + `StoredVersion` (SHA-256 of the stored bytes, which names the schema
id and storedAt) set by `CultCache.Wrote`. After the merge, 9 references in
ConditionalCommitTests and the contract name main's owner. The version is the owner.
PersistedSchemaId and MarkWritten are deleted, and main's +309 test lines are ported as
pins against StoredVersion. 2.0.0 ships MarkWritten as protected API, and C2a also
removes `SerializeSnapshot`/`DeserializeSnapshot` overrides and changes `ConditionsHold`.
C2a is therefore a major release (3.0.0), which semver forces.
- *TS store commit moved.* Main's `store-file.ts` (0e6452c9, 0010ee92; 391 lines) now owns
commit, replace and the `<path>.lock`. C2a's v3 read and keep-v3-on-rewrite
(`STORE_FORMAT_ELEMENT_IDS`) lives in the old store file and must move into store-file.ts.
- *Rust decode door.* Main's cultcache-rs 0.3.0 grown decoder (`decode_stack`, fc630d3b)
must sit under C2a's `readable_store_format` and v3 `decode_store_file`.
**2. The gate already holds; it needs a test, not a format change.** C2a stamps
`cultcache.store.v3` (single file) or `...v5.directory-content-addressed-pages` exactly when
a record holds an id (`HeaderFor`, CultCache.cs:3543). It also marks when a copied record
decodes holding one (`CopiedHoldIds`, :4285). The 2.0 C# reader accepts only v1/v2
(CultDocumentMessagePackSerialization.cs:229) and directory v4 (Directory store :452).
Main's TS, Rust and Python readers accept only v1 (store-format.ts `isV1Snapshot`, lib.rs:1274,
stores.py:192). Probe on Yggdrasil at 5b5d8573 plus C2a's `v3-base.msgpack` (job
`gate1.log`): 4/4 pass. The single file is refused naming v3, and a later flush leaves it
byte-identical. A v5-stamped directory store is refused naming v5, and its 4 files are
byte-identical after a later add and flush. **The pin:** C2a already ships
`tests/vectors/document-variants-c2a/old-reader-refusal.sh`, which stages an old revision
plus probes on a scratch ref and runs them per runtime. The land cut sets its
`OLD_READER_REV` default to the tag `cultlib-unity-v2.0.0`. It also adds the never-rewritten
assertion and the v5 directory case to the C# probe, and runs all four runtimes at the
tag. If the tag is absent, the land waits for it. The tag, not a package, is the pin: the tag's
tree holds every runtime's released reader.
*Outside C2a's reach (release session):* in 2.0, `CultMesh.WriteSingleFileDocument`/
`WriteSingleFileDocumentPayload` (CultMeshSingleFileDocuments.cs:211-245 at 7a8933b0) build
a fresh one-record v1 snapshot and replace the file without reading it. A 2.0 Mesh write
to a C2a v3 path therefore discards it whole, ids and other records included, and no header
gate can stop a writer that never reads. C2a's version reads under the lock and refuses.
Fix it in 2.0.0 before tagging, or record it.
**5. Consumers.** Aetheria pins a CultLib revision (`VerifyCultLibRevision`) and Unity tags.
Ghostlight, Eve's Rust crates, Huginn and the Brokkr daemon pin cultcache-rs by git rev.
None of them moves when main moves. Two C# consumers build against an unpinned sibling
`..\CultLib` checkout: Eve `GameCult.Eve.Surface.csproj:16` and Brokkr's three csprojs.
Eve.Surface has `[MessagePackObject]` element arrays with no `[CultElementId]`
(`EveAssetCatalogEntry[]`, `EveEntitySoaBuffer[]`, `EveInputActionDocument[]`,
`IReadOnlyList<EveCommandTemplate>`, 0 ids in the repo), so C2a registration refuses its
documents as soon as the checkout pulls the land. Grep finds no object lists in Brokkr's
`[CultDocument]` files (not build-verified). Removed C2a API (`Options`, `AsSchemaAlias`) is
unused by Eve and Brokkr. Fork: `variants:question:unpinned-consumers-at-land`.
## Classification
| Item | Class | Why |
|---|---|---|
| c2a-format-version-gate | **a** | The gate test above. No new header: v3/v5 already refuse. |
| slot-reuse-contract-doc | **a** | The ruled consequence of the write set C2a lands. One paragraph in cultcache-schema-compatibility.md, absent at a5c7954e (grep). |
| foreign-ids-under-v1-header | **b** | Obsoleted by fb065f0d (write-time `CopiedHoldIds`, after the finding) for every C# writer with the type. The residual (non-C# or opaque writers) is what `non-csharp-writers-mark-ids-2` ruled to carry, and the contract states it (persistence-format.md:434-439). The gate changes neither side, so no fork remains. Resolve as Superseded by that ruling. |
| custom-resolver-element-ids | **b** | Fixed by b091d21f (19:52Z, after the 18:27Z follow-up): `ListElementsClaimedByAResolverAreRefused`, `AResolverClaimingTheListTypeOfAnObjectListIsRefused`. s6 holds on the claim loop. |
| codec-door-proven-mismatch | **b** | Done in h6 57fdf70c (s6 `only-proven-mismatch` Holds; Geometry 76/0). |
| consumer-element-id-sweep-r2 | **c** | Each repo's sweep is its own work (q3). Only the unpinned trackers matter at land, and that is the question below. Aetheria bumps a pin on purpose (C4). |
| schema-discovery-holder-recogniser | **c** | `TryGetDictionaryRefTypes` is identical on main (CultCache.cs:1288). It describes record-ref dictionaries and is not on the id path. Changing it moves schema hashes, which is its own epoch event. |
| copy-check-memo | **c** | Performance only (about 3x flush on unmarked stores of 20k records). Correct as is. |
| string-key-field-and-shadow-pins | **c** | No wrong behaviour at 64ec14f9. These are test pins and may ride the land cut if Hands touches the file. |
| declared-range-gate-own-version, range-check-in-ci | **c** | The script exists only on C2a and runs in no CI. It guards a TS publish, not a merge. Fix both before the first TS release from main after the land. |
| s6 Unproven: Unity/IL2CPP identity, other MessagePack versions | **c** | The merge reaches no Unity build. Unity sees C2a only at the 3.0.0 tag, so measure it in the Unity release cut (operator Unity check, C3). |
| multi-store-commit-atomicity, store-replace-cut, reader-refuses-duplicate-keys, soft-flush-all-runtimes, writer-dangling-link-tolerance, unique-index-cut, py-shared-write-lock, rust-lock-open-waits, ts-lock-test-hardening, c0/legacy/directory vectors, culture-test, wire-id-cuts, remaining-variant-cuts, golden-path, paged store | **c** | None says "before land". Each is pre-existing on main or C2a-independent. None lets a released reader strip ids or corrupt a C2a store. |
## The land cut's shape (transcribe once the question is ruled)
- **Base.** Merge main at or after the `cultlib-unity-v2.0.0` tag into `hands/variants-c2a`.
Versions stay main's. CHANGELOGs get Unreleased entries, and the Unity entry says major.
- **Deletes and collapses.** Delete `PersistedSchemaId`, `PersistedIdentity`,
`CultStoredDocument.MarkWritten` and `CacheBackingStore.MarkWritten` (main, about 60 lines),
and the store-composition contract's MarkWritten paragraph. C2a's `StoredVersion`/`Wrote`
is the one owner. In TS, delete C2a's header handling in
single-file-messagepack-backing-store.ts and move it into store-file.ts. In Rust, C2a's v3
decode goes through the grown decoder.
- **Adds.** Retarget old-reader-refusal.sh to the tag. Add the C# probe's never-rewritten
assertion and the v5 directory case (as in gate1.log). Add the slot-reuse paragraph.
Port main's ConditionalCommitTests to the version identity.
- **Verification.** Full suites: Caching, Mesh, Networking, Geometry, WebSockets,
ConsumerBuild, InteropPeer, cultcache-ts/-rs/-py, cultnet-ts and cultmesh-ts, all on
Yggdrasil. Run old-reader-refusal.sh for all four runtimes at the tag, and it must refuse.
Negative greps over the merged tree: `PersistedSchemaId|MarkWritten`, conflict markers.
Mutation: drop the v3/v5 stamp in `HeaderFor` and the gate probe must fail.
- **Estimate.** About 950 conflict lines resolved. Net -60 to -150 from the identity
collapse and TS de-duplication, +80 to +120 tests and docs. One Hands pass of 100-150
calls, then one Soul pass.
- **Precondition (question).** Eve.Surface, and Brokkr if confirmed, are pinned or swept
before the merge reaches main.
## Store gate: attachment is the attach's hold (store-gate-one-owner r2)
Spec `variants:cut_spec:cut-store-gate-one-owner.r2` (base ec94ab98), question
`variants:question:pulled-variant-codec`. It closes Soul s3's
`attach-window-cacheless-write` (Medium) and `codec-loan-pinned-by-reflection` (Low).
**Facts (source reads at ec94ab98 and a probe, 2026-10-25).**
- The window. AddBackingStore holds the cache's gate, then the store's gate through
`InStoreHold`, lends the registry and codec, sets `Loaded`, adds the store to `_stores`,
pulls, and only then sets `store.Cache` (CultCache.cs 2268-2312). `InStoreHold` lets any
call on the same store's gate re-enter it (2825-2831). So a journal the pull's Admit runs
on the attaching thread can call `store.WriteDocument`: `Gate(cacheless)` guesses null,
re-enters the store's gate, finds `Cache == null` and runs the write, and its `finally`
calls `UseRegistry(null, null)` (Soul s3 P1, full1.log lines 49-53). Other threads are not
in the window: they queue on the store's gate and see either the published cache or a
store that the failed attach put back.
- The codec loan has no reader on the attach path. A pulled variant's codec is read only
by `CultStoredDocument.HoldsIds` (543-557). Header decisions read `HoldsIds` only on
staged entries (single-file 4318, 4447 and 4472; directory store 362), and a staged entry
is always the document the cache pushed (`Push` sets `Entries[key] = entry`, 4211-4215).
A single-file `PullAll` returns without loading while the store is dirty (4157-4159), so
a pulled copy is never staged. A refused write unstages its keys before it adopts the
reload. The directory store refuses variants (ruling `q6-directory-and-cultnet-refuse`;
DirectoryMessagePackBackingStore.cs 139-144 and 642-643). Admit re-resolves every pulled
variant with the cache's codec (CultCache.cs 3259). Soul's N4 mutant (the attach lends
no codec) was caught only by the reflection test, and h4's M2 only by a bare-store test
that sets `Loaded` and calls `UseRegistry` by hand.
- Probe (scratch clone `imagination-store-gate-r2/clone`, commits 4663934e and c099b9e8 on
ec94ab98; Yggdrasil t1.log, t2.log and t3.log; 2026-10-25 around 20:50 CEST). The
probe did three things. `InStoreHold` refuses a cacheless call that re-enters the same
store's gate. The `_codec` field and the two-argument `UseRegistry` are deleted. The pull
builds variants with a null codec. Results:
- Mesh is 342/0/4.
- In the Caching suite, the only failures were two probe tests, both from authoring
errors: a key `AddAsync` minted, and an element id that was not 12 hex characters.
No existing test failed, `UserCodeInACachelessCallMayCallItsOwnStore` included.
- Soul's P1 as a test: the journal's write is refused with the new message. The attach
keeps its registry and the file holds only `a`. After the attach, a cache write, a
flush and `store.PullAll()` all succeed.
- A cacheless write nested in another on the same store is refused, the outer write
fails, and no file is written.
- Through a real attach, a store holding a name-override variant flushes an unrelated
commit with a `cultcache.store.v2` header. With no codec loan, the pulled copy does not
mark the store.
- Mutant `return cacheless` changed to `return false`, which is ec94ab98's behaviour,
fails both new gate tests. In that mutant the nested write succeeds and the file exists.
**Rationale.**
- Attachment has one fact again. While the attach runs, the store's gate is the attach's
hold, and `Cache` is published when that hold ends. A cacheless write is a write the
store owns outright, so it must be the outermost hold of its store's gate. One more
argument to the existing owner of per-thread gate facts (`InStoreHold`) refuses it. No
field mirrors `Cache`, and `Loaded` and `_registry` are not read. Precedent:
`ConcurrentHashMap.computeIfAbsent` refuses a mapping function that updates the same
map with `IllegalStateException("Recursive update")` since Java 9. That is from
memory, not fetched this pass; the 2026-10-23 prior-art note covers the documented
restriction in its counter-positions. Non-cacheless store calls from user code still
re-enter: `UserCodeInACachelessCallMayCallItsOwnStore` stays green.
- The late publish stays. It is what bounds `Gate` at two passes (s2's finding). Publishing
early would bring back the failure-rate-bounded retry.
- The codec loan goes: it is a value nothing on the attach path reads. A pulled variant
carries no codec and counts as holding ids (the documented conservative answer), and
the cache's resolved copy is what any header decision sees. This reverses the codec
half of Defaulted ruling `late-publish-needs-attach-codec`, so it goes to Self as
`variants:question:pulled-variant-codec`. The spec is written for the recommended option.
- Scar check: `git log -S` over CultCache.cs for `reentrant`, `outermost` and `_attaching`
finds nothing. `Cache?.Codec` appears in 1468a83a (h4) and 619e089f only. No earlier
re-entry refusal was built and deleted.
- Behaviour change, stated in the CHANGELOG: a cacheless write made from user code that
another cacheless write of the same store runs used to succeed, and its `finally` took
the outer call's registry. It is now refused.
## Store gate: Cache is the one attachment fact (store-gate-one-owner r3)
Spec `variants:cut_spec:cut-store-gate-one-owner.r3` (base 0fa836cb), superseding r2. It
closes Soul s4's `attach-window-held-store-writes` (Medium), `cacheless-refusal-order-unpinned`
(Low) and `interop-consumer-suite-unrun` (Low). This is the third time the attach window has
beaten the rule (s3 from the cacheless side, s4 from the held side), so r3 moves the
decision instead of patching the next entry point.
**Facts: every entry point that reads or decides attachment, at 0fa836cb.** Source reads of
CultCache.cs, DirectoryMessagePackBackingStore.cs and CultMeshSingleFileDocuments.cs. The
window column is a journal on the attaching thread, through a real attach of a store holding
`a`. Probe: scratch clone `imagination-store-gate-r3/clone`, commit b2330b6d (test R3Probes),
Yggdrasil `probe-base.log`, 2026-10-25 around 23:00 CEST. "Disagree" means that after the
attach, a commit of `later` and a flush, the attaching cache holds `a,later` and a fresh cache
over the same file reads something else.
| Entry point (line) | Fact it reads | In the attach window at 0fa836cb |
| --- | --- | --- |
| Single-file `PullAll` (4156) | `Gate`: `Cache` guessed unlocked, then re-read under the re-entered store gate | Runs as detached; its `Loaded` re-enters the cache and is refused as a journal write. Agrees. |
| Single-file `Push` (4212) | same | Runs as detached and stages `b`. Disagree: fresh reads `a,b,later`. |
| Single-file `Delete` (4223) | same | Runs as detached and stages a removal the cache never made (dirty after the attach). |
| Single-file `PushAll` (4239), and `Dispose` (3722) through it | same, then in-body `Cache == null` (4531, 4543, 4588, 4603, 4611, 4661) | Runs as a cacheless writer. Push then PushAll writes `b` to the file inside the attach (disagree). Delete then PushAll is refused with the cacheless writer's text. On a variant store it is refused as "a writer with no CultCache" (Soul s4). |
| Single-file `CommitBatch` (4256) | same | Runs as detached and commits `b`. Disagree. |
| Single-file `JudgeChecks` (4568), `RefuseRemovalOfUndeclared` and `StageChecks` (4558, 3915) | same | Run as detached. Agree in the probe, but under the wrong gate's rules. |
| Single-file `ReadStoreSnapshot` (4687) | same | Runs as detached and reads the file. |
| `WriteRecord` (4342), `WriteDocument` (4366) | `InStoreHold`'s per-thread hold (cacheless re-entry), before `Cache` | Refused with the re-entry text (h5). Agree. |
| Directory `PullAll` (48) | `Gate`, as above | Refused as a journal write. Agree. |
| Directory `Push` (147), `Delete` (160), `CommitBatch` (172) | same | Run as detached. Disagree: fresh reads `a,b,later` after Push or CommitBatch, and `later` after Delete. |
| Directory `PushAll` (235) | same | Runs as detached. Push then PushAll and Delete then PushAll disagree. |
| Directory `JudgeChecks` (269), `RefuseRemovalOfUndeclared` (259) | same | Run as detached. Agree. |
| In-body reads: `StoredAtOver` clock (4028), refusal text and `DurableAt` (4531, 4543), declares and cacheless refusal (4588, 4603, 4611), `Wrote` (4661) | `Cache`, under the gate `Gate` chose | On the attaching thread they read null while the thread holds the cache's gate, so a held call runs as a cacheless writer. |
| `AddBackingStore` (2259-2314) | writes `Cache` under both gates, after the pull | The window's author: the store is attached for the pull (registry lent, `Loaded` set, in `_stores`) while `Cache` is null. |
| Mesh `CultMeshSingleFileDocuments` writes and reads (99, 143, 174, 226) | none of the attached store's | `StoreAt` (239) builds a new store per call, so they never touch an attached store object. A file it shares with an attached store is the file lock's and the write set's business. |
| `IsDirty`, `HasStagedChecks`, `ForeignRecords`, `LastSchemaMigrationReports`, `HeaderFor`, `HeaderForWrite` | none | Ungated values or static decisions. They read no attachment fact. |
So during an attach, cacheless entry points decide attachment from the hold, and every held
entry point, plus every in-body read, decides it from `Cache`, which is still null. These are
two facts.
**Probe of the chosen shape** (commit 4695390c on b2330b6d; `proto-full.log`). `AddBackingStore`
publishes `Cache` before the pull and takes it back in the same hold if the pull fails. `Gate`
becomes straight-line code: guess, then the store's gate, where `Cache` is settled, then the
settled cache's gate. Results:
- All 21 probe cases are refused. The cacheless writes get the re-entry text and every other
case gets "A journal entered the cache it journals". After the flush, the attaching cache
and a fresh cache both read `a,later`.
- Caching: the only failures are the two tests that pin the late publish:
`AnAttachLendsTheRegistryAndPublishesTheCacheAfterThePull` (asserts no cache during the
pull) and `ACallWhoseGateChangesTwiceUnderItThrowsInsteadOfLooping` (asserts the old message
text; the tampering reader still throws). The churn, queued-reader and failed-attach tests
pass unchanged.
- Mesh and Networking are green.
- The cultcache-ts suite, which builds and drives the C# InteropPeer, passes on this shape:
175 tests, 171 pass, 0 fail, 4 skipped. The C#, Rust and Python interop tests ran, the C#
edit-in-place one included (`proto-ts2.log`, commit f5f02053). Recipe `clone/r3ts.sh`:
image node:24-bookworm, apt python3-msgpack, dotnet-install for 10.0, rustup minimal.
**Shapes weighed (rationale).**
- (a) Publish `Cache` before the pull, as a616431e did. Chosen. What s2 found was not that an
early publish is unbounded. It was that `Gate` re-read `Cache` *unlocked* after a stale
guess, so a churn of failing attaches could hand it a new stale guess every pass. The fix is
where `Gate` reads on its second step. Under the store's gate, `Cache` is settled: an attach
publishes and takes it back inside one hold of that gate, and no other thread can hold that
gate during the attach. A settled cache is permanent, because there is no detach. So `Gate`
needs at most three acquisitions, with no loop and no counter: the guess's gate, the store's
gate, and then the settled cache's gate, which cannot miss. This deletes a fact rather than
reconciling two. During the attach the store *is* attached, so every entry point, cacheless
or held, and every in-body read sees one answer. A journal's call is then refused by the rule
journals already have (AddJournal: "any hold on this cache while a journal runs throws"),
exactly as it is after the attach. Prior art: `ConcurrentHashMap.computeIfAbsent` installs a
ReservationNode in the bin *before* running the mapping function, so a recursive update on
the same thread finds the owner and throws, and on failure the reservation is removed (see
`docs/research/2026-10-23-store-lock-user-code-prior-art.md` item 4 on main: the re-check
after locking, and the ReservationNode on the empty-bin path). The validate-after-acquire in
`Gate` is the same item.
- (b) Keep the late publish and make the attach's hold the one fact for every entry point.
Rejected. `InStoreHold` knows that this thread holds the store's gate, not *whose* hold it is.
Refusing every same-store re-entry kills `UserCodeInACachelessCallMayCallItsOwnStore` (r2's
R5), so (b) needs an "attaching" marker beside `Cache`. That is the second fact that s3 and
s4 rule out, plus every in-body `Cache == null` read would need to consult it as well.
- (c) Run no user code on the attaching thread under the store's hold during a pull: defer the
pull's journals to after the hold (open calls). Rejected as insufficient on its own. Getters
and constructors still run during the pull's decode, so the window stays open to any user
code that is not a journal. It also moves journal timing, which is a documented contract
("right after an admission is applied").
- Scar check: `git log -S` over CultCache.cs finds `store.Cache = null` and `MaxStaleGuesses`
only in ba658965 and 1468a83a. The early publish was not removed for a correctness reason of
its own: 1468a83a removed it to bound `Gate` and to lend the codec, and the codec loan is gone
(`pulled-variant-codec`). No settled read under the store's gate was built before. The
precedent recipe (no root filter) finds no same decision.
- Not a product fork. A journal's store call during an attach now throws, as it already does
after the attach. Cacheless writes on the attaching thread stay refused, with the same text.
Getters and constructors during the pull see an attached store, as they do during any held
call. It changes the late-publish clause of Defaulted ruling `variants:ruling:pulled-variant-codec`,
so Self defaults a superseding ruling (recommended label `cache-published-for-the-pull`)
before dispatch.
- Residual, not in r3. `Push`, `Delete`, `PushAll` and `CommitBatch` are public, so user code
that is not a journal (another thread, or a getter during any held call) can stage records
behind an attached store's cache. Soul s4 shows this with `other-thread-push`: file `a,b`,
cache `a`. That is true before, during and after an attach, so it is not the window's
two-fact problem. Closing it changes public API (a store's writes taken only from its
cache), so it is a follow-up and an operator question for Self, not part of this cut.
## Body facts: claimant-one-owner r2 (Imagination, 2026-10-27, base fc74cf6f)
Why a revision: Soul beat the same rule three times (s1 claimant choice, s2 compatible ids, s3 shape). Each time a
cacheless write lent a registry let the registry's entry for the written schema id decide ownership beside the entry
the caller passed. Two descriptions of one type reached `Owns`; each fix (h2 schema-id compare, h3 subset of compatible
ids) compared one more facet of the two and left the rest open. r2 removes the second description from the lookup.
### Where the two descriptions met (source read, fc74cf6f)
- `OwnsOnDisk` cacheless arm (`CultCache.cs:4725-4740`): with a lent registry it called
`lent.Owns(id, catalog, declared, written)`, whose lookup is the registry's `_indexes` and whose `entryOf` is
`descriptor.ToCatalogEntry()` (`:824-825`). The claimant for the written schema id was the registry's descriptor,
so `ReadsWhole(persisted, claimantEntry)` (`:848`) judged shape against the registry's entry, and the claimant
precedence (`:756-806`) walked the registry's compatible ids. `written` (`:4734-4736`) then compared the two entries
by schema id plus a subset of compatible ids.
- `WriteDocument` (`:4495-4524`) takes its entry from the registry (`resolvedRegistry.GetRequired<TDocument>().ToCatalogEntry()`),
so its two descriptions are always identical. Only `WriteRecord(record, catalogEntry, holdsIds, registry)` (`:4471`) can
pass an entry that disagrees with the registry's entry for the same id.
- `CopiedHoldIds` (`:4600-4630`) also reads `AttachedRegistry`, to decide the v3 header from copied records. It is a
positive-only header decision, not ownership, and out of this cut.
### Consumer audit (2026-10-27)
- `WriteRecord` and `WriteDocument` are new in the Unreleased 3.0.0 entry; `main` has neither (`git show main:src/GameCult.Caching/CultCache.cs`).
- Per-repo `rg` over every git repo directly under `F:\Projects` for `.WriteRecord(`, `.WriteDocument(`, `WriteSingleFileDocument`
(Library, obj, bin, PackageCache excluded): hits only in CultLib and in vendored CultLib copies under Heimdall and StreamPixels
(`vendor\CultLib`, older pins without these methods). No consumer outside CultLib calls either method.
- Inside CultLib at fc74cf6f: Mesh `WriteSingleFileDocument` calls `WriteDocument(key, document, storedAt, registry)`;
Mesh's raw-payload path calls `WriteRecord(..., registry: null)` (`CultMeshSingleFileDocuments.cs:129, 226`). The only
`WriteRecord` calls with a registry are tests: `CachelessSingleFileTests.cs:453, 660` (a registry `{IdDeck}` that does
not hold the written entry `One` at all, used for the copied-record header), `RestampOnWriteTests.cs:595, 599`, and
Soul's probes. Packages: no TS, Python or Rust counterpart takes a registry (cultnet-rs `write_record` is an example helper).
- Consequence: the registry's documented role in `WriteRecord` is to describe the other records in the store. A shape
that derives the written entry from the registry (or refuses an entry the registry does not hold) breaks
`CachelessSingleFileTests.cs:660` and removes that role.
### Candidate shapes weighed
| Shape | One description? | Public API | Kills s2/s3 | Cost |
|---|---|---|---|---|
| A. Lent registry's entry decides; refuse a passed entry that disagrees | yes, after a door check | signature kept; new refusal | yes | an equality check over a redundant parameter; breaks the registry-describes-others use (`:660`) when the registry lacks the written type unless it falls back to the passed entry, which is two descriptions again |
| B. Passed entry is the only description of its schema id; the registry supplies other types only | yes, by construction | signature kept; doc only | yes (probed) | one lookup class gains the registry; one registry overload deleted |
| C. Overloads: `WriteRecord(record, catalogEntry, holdsIds)` and `WriteRecord(record, registry, holdsIds)` | yes, by type | changes a 3.0.0 signature | yes | loses the registry-describes-others use |
| D. Delete `WriteRecord`'s registry parameter | yes | changes a 3.0.0 signature | yes | loses the copied-record header decision for raw writes |
Picked B. It is the only shape that keeps every documented use of the registry, needs no public signature change,
and deletes the comparison instead of extending it. It is ordinary scoping (an inner binding shadows the outer one
for its name), applied to schema ids.
### Probes (Starfire, local filtered runs, 2026-10-27)
Scratch clone of fc74cf6f at `<scratchpad>/imagination-claimant-r2/clone`, Soul's `SoulClaimantProbe.cs`,
`SoulClaimant2Probe.cs`, `SoulClaimant3Probe.cs` copied into `tests/GameCult.Caching.Tests`.
Base fc74cf6f (`probe-base.log`):
- P7 (registry {N lists own, A, B}; passed entry N with the list varied; record k under O, named like N): lent R / no
registry C for rows b, b2, e, g, h, k, l, l2; lent C for a, c, d, f, i, j. Confirms `subset-refuses-own-drift`.
- P9 (registry {NW: Name, Value, Colour}; passed NW minus Colour): lent C (restamped, Colour lost), no registry R.
Confirms `lent-shape-widens-passed-entry`.
- P6 (bare X1 entry, registry {X1} listing OX, record under OX): lent R, no registry R, cache C. h3 closed the id axis.
- P4: 12 states x 8 registries, violations=0. P1d: a cache {N,L} commits N over L's listed old id (`cache-replaces-other-types-record`).
Prototype B (scratch commit 349ea3f3 on the clone, not for landing; `probe-protoB2.log`): `CatalogEntryLookup` takes
the written entries and an optional registry and drops every registry descriptor whose schema id a written entry has;
the registry's `Owns(schemaId, catalog, declared, mayClaim)` overload is deleted; the cacheless arm always asks
`CultDocumentRegistry.Owns(id, catalog, declared, new CatalogEntryLookup(writes, AttachedRegistry), entry => entry,
claimant => writes has claimant's schema id)`.
- P7: lent == no registry on all 14 rows (all C).
- P9: lent R, no registry R, wide entry C.
- P6: unchanged (R, R, C). P4: identical table to base, violations=0. P1a-f, P2a-d: identical to base.
- `RestampOnWriteTests`, `CachelessSingleFileTests` and the probes: 139/139 passed locally.
- Yggdrasil, 349ea3f3, `ygg-protoB-full.log`: GameCult.Caching.Tests 1180 passed / 0 failed / 2 skipped;
GameCult.Mesh.Tests 342 / 0 / 4.
Negative greps, each tested at fc74cf6f (each matches there, and none may match after the cut, src and tests):
`claimant.CompatibleSchemaIds.All(` (1, the subset), `lent.Owns(` (1), `Func<CultSchemaCatalogEntry, bool> mayClaim) =>`
(1, the deleted overload), `CatalogEntryLookup(writes)` (3, one-argument construction). CHANGELOG and docs:
`never widens what the passed entry declares` (1), `with no compatible id the passed entry does not list` (1).
### Not settled by r2
- `cache-replaces-other-types-record` (s2, PreExisting, cache arm): r2 touches only the cacheless lookup, so it does
not settle whether a cache may replace another registered type's record at a shared key.
- Whether a registry-less `WriteDocument` should consult `CultDocumentRegistry.Shared` for claimants at all (s1
`shared-decides-registryless-write`, closed Fixed by documentation in h2). r2 keeps today's answer.
- `variants:finding:cut-restamp-on-write.s2.cacheless-inverts-registry-claimant` is still in force although r1 was
mapped to close it; Self should close or re-triage it after r2's Soul pass.

149
docs/document-variants-cut.md
View file

@ -91,6 +91,121 @@ Probes run 2026-10-12 from `F:\Projects\CultLib-c2afix` with git only (no builds
`unstored-same-tick-unpinned`) caught at head. `git grep RawBytes de401056 -- src tests`: no
match. Those findings were never closed in the mind.
### Body at 2026-10-15: element ids and the MessagePack walk (Modeling, from the Imagination and Soul passes)
Probes ran on Yggdrasil (`mcr.microsoft.com/dotnet/sdk:10.0`, MessagePack 3.1.7 in every csproj) as probe
tests that were never committed: at `6afa8ed9` for I1 to I4, at `494ad9ab` for r4 and r5, at `6e70626b` for r6.
Logs and sources are under `F:\Projects\eureka-handoff-aux-2026-10-15\scratchpad-self-2026-10-14b-eureka\`
(`imag\`, `imag-r4\`, `imag-r5\`; Soul's `soul-c2a-id-holes-s5\` is in that session's scratchpad). The typed
record of each fact is in the mind: `variants:cut_spec:cut-c2a-element-id-holes.r5` and `.r6` (`first`),
verdicts `cut-c2a-element-id-holes.s5` and `.s6`, spec `cut-c2a-ids-header.r1` (`first`). This section keeps
what was run and what it showed.
**What an id is on disk and in memory.**
- An id is an ordinary keyed string in the payload. A list element whose `[CultElementId]` member holds
`a1b2c3d4e5f6` and one whose plain `Tag` holds the same value encode to identical bytes
(`92A1649192AC61316232633364346535663607`; `c2a-ids-header` r1 probe, base `2774a7fe`). No reader can tell
from a record's bytes that it holds ids; only the store header says so.
- `StubArm`, a union arm that is not an object type, is never minted (`imag\imag1.log`, I1): for a document of
`StubArm` and `Unit8`, `Plan` counts 1, `StubArm.Id` stays `''`, `Unit8.Id` is minted, `Inspect` reports 0
problems.
- Resolver over-mark (I2): a consumer `[assembly: CultCacheFormatterResolver]` claims a keyed arm with a
formatter that writes only `N`. The cache mints an id in memory, `Holds` is true and the header is stamped
`v3`, but the payload (`92A263319192009104`) carries no id, and a reopen mints a different one (`8f5e929888ca`
in memory, `a1ea2b683d98` after reload). The header claims ids the disk lacks.
- `[MessagePackFormatter]` on an element type is honoured (I3): the attribute formatter wrote
`92A26131919104`, no id, and `Inspect` reported 0 problems at that base. Registration refuses it from r5 on
(`declares [MessagePackFormatter]`).
- Python store (I4): `py-store/leaf-record.cc` (header `v1`, ids `000000000021/22`) loads in C# with
`HoldsIds = true`, `IdsInMemoryOnly = false`. A C# write that adds an unrelated record copies it forward,
the header stays `v1`, and the ids survive reload. cultcache-py, -ts and -rs have no declaration of which
member is an id, so they keep a `v3` they found and never decide one.
- Header narrowing (`c2a-ids-header` r1): per-record `StoredHoldsIds`, set from the type that decoded a
record, is falsified by the identical-bytes probe above. The durable header owns "these stored bytes hold
ids" at store granularity. A write that copies any record forward keeps a marked header; only a write that
encodes every record decides by content. `StoredHoldsIds` and `KnownIdFree` are deleted.
- Object list versus value list (r4, `imag-r4\probe1.log`, `probe2.log`): ids sit on object-list elements only
(ruling `q7-object-list-elements-only`). `Dictionary`, `ConcurrentDictionary`, `SortedDictionary`,
`IReadOnlyDictionary`, and `ILookup` of keyed arms round-trip with the id at
the element's own slot (e.g. `ConcDict` `81A16B92AC30313233343536373839616200`: key, then `[id, 0]`). A list
or map of scalars has no element to carry one and never enters the id path.
**How MessagePack decides what it writes** (r5 `imag-r5\probe1.log`, `probe3.log`; Soul s5 `job2.log`).
- Member discovery, `MessagePack.Internal.ObjectSerializationInfo.CreateOrNull(Type, bool forceStringKey,
bool contractless, bool allowPrivate)`, is internal, and so is `StandardResolver.Resolvers`. The public
surface allows one test: `StandardResolver`'s formatter for `T` is reference-equal to the formatter one
public singleton hands out for `T`, each caching one instance per type. `DynamicObjectResolver.Instance`
matches keyed classes and structs, `DynamicUnionResolver.Instance` `[Union]` types,
`DynamicGenericResolver.Instance` List, arrays, maps, KeyValuePair, tuples, Lazy, Memory, ArraySegment and
Nullable, and `ImmutableCollectionResolver.Instance` (namespace `MessagePack.ImmutableCollection`)
`ImmutableArray` and `ImmutableList`. Asking `DynamicObjectResolver` for a collection type throws
`TypeInitializationException`, so a family probe swallows exceptions.
- Kind facts (`probe3`): a `[MessagePackObject]` class implementing only `IEnumerable<T>` or
`IReadOnlyList<T>` (`KBag`, `KNode`, `KRo`) is written by key (object resolver). One that is also
`ICollection<T>`, a `List<T>` subclass or a `Dictionary` subclass (`KC2`, `KColl`, `KList2<T>`, `KDict`) is
written as a collection (generic resolver) and its `[Key]` members are dropped.
- Member facts (`probe3`; `ap` is `AllowPrivate`, `R` written, `W` read back). A member is written when it
has an integer `[Key]`, no `[IgnoreMember]`, and is a public field or a property with a public getter. It is
read back when it is a public non-readonly field, a property with a public setter (`init` counts), or a
`[SerializationConstructor]` parameter. At `ap=False`: `PG` (private getter) `R=False W=True`, `PS`
(private setter) `R=True W=False`, `GetOnly` `R=True W=False`, `InitOnly` both. `IdPriv`
(`Id { get; private set; }`) is written and not read back (wire `91AC62...`, back `<null>`), and `IdCtor` is
`W=False`. At `ap=True` every one of these, `priv` included, is `R=True W=True`, except `GetOnly`
(`W=False`).
- AllowPrivate attribute form (Soul s5 `SoulS5Probe.C`, `ApTrueArm`): on a type declared
`[MessagePackObject(AllowPrivate = true)]`, `StandardResolver` writes private `[Key]` members and a
private-setter id round-trips. The oracle test calls `CreateOrNull(type, false, false, false)` and reports
none of them, so the walk's member set misses that form.
- Collection written as object (r6 `probe2.log`): a `[DataContract]` class implementing only `IEnumerable<T>`
over an id-carrying arm (`DcSeq`) is a collection in the walk, yet `StandardResolver`'s formatter is
`DynamicObjectResolver`'s (wire `91 91 91 92 AC..`, written as an object). A `[DataContract]` `List<T>`
subclass (`DcColl`) is written by the generic resolver, and the object resolver hands out a different
instance for it. For `DcBad` (two members on one `Order`) `StandardResolver` uses the generic resolver, and
Soul s5 measured both `DcColl` and `DcBad` passing the door.
- Geometry (r6 `probe2.log`, `probe3.log`): Geometry has a direct MessagePack 3.1.7 `PackageReference`, so the
source generator runs, and its assembly carries `[GeneratedAssemblyMessagePackResolverAttribute(
typeof(GeneratedMessagePackResolver), 3, 1)]`. For `CultGeometryDomainNode`, `DomainDocument` and
`FeatureClaim`, `StandardResolver` returns `GeneratedMessagePackResolver`'s formatter and every public
singleton returns a different instance or null. For `CultGeometryDomainNode[]` and `FeatureClaim[]` it is
the generated resolver's `ArrayFormatter`, not `DynamicGenericResolver`'s. No Geometry id-path type matches
a family singleton. Caching.Tests has no generator (analyzers do not flow through a `ProjectReference`), so
its keyed types match `DynamicObjectResolver`.
- Generated assemblies (Soul s5, fresh generator-enabled assembly `SoulGen`, `job2.log` GEN rows): a keyed
`List` or `Dictionary` subclass (`GKColl`, `GKDict`) is written by key, wire `["c",[7]]`, with its elements
dropped by MessagePack; the walk reads it by key too (Plan 0). A generated-assembly `[DataContract]`
enumerable (`GDcSeq`) is still written by `DynamicObjectResolver`, because the generator emits no
DataContract formatters.
- Null standard (Soul s6 finding `null-standard-untyped`): for a non-generic interface or abstract class that
enumerates arms, `StandardResolver` returns null instead of throwing, and the door neither refuses nor
types it.
- Wire pin (Soul s5, s6 `SoulS5Pin`): a deterministic document with 12 minted ids, encoded by the codec and
decoded independently with `MessagePackSerializer.ConvertToJson`, carries every id. The ids are
byte-identical at `494ad9ab` and `57fdf70c`, `0f8964b58a08` and `7eb4fb13d530` among them.
- Not measured: Unity/IL2CPP, where the package bundles its own `MessagePack.dll` (version not read) and the
dynamic resolvers cannot emit, and MessagePack versions other than 3.1.7.
**Whole-list holders at `57fdf70c`** (Imagination r2 for `c2a-id-closed-world`, 2026-10-16; probe tests in a
scratch commit `dc629459` that was never pushed, Yggdrasil `sdk:10.0`; logs `job1.log` and `job2.log` in the
Imagination scratchpad `imagination-c2a-closed-world-r2`).
- Shapes the walk does not own already behave as whole values. A keyed document holding
`ImmutableArray<PElem>`, `(List<PElem>, int)`, `KeyValuePair<string, List<PElem>>`, a non-enumerable
`[DataContract]` holder, or a string-keyed holder (`[Key("items")]`) gives `Inspect` problems 0, an empty
IdPath, `CanHoldIds` false, `Plan` 0 (random and deterministic), and `Holds` false with ids set.
- The marker and the copy check do not count them. Committed with ids `aaaaaaaaaaaa`/`bbbbbbbbbbbb`, the store
header stays `cultcache.store.v1`. A fresh cache that writes another record over that file (the copy-forward
check) keeps `v1`. Reload returns the ids exactly as written, and unset ids stay null after write and reload.
- Variants replace them whole, and that is the C2a resolver's behaviour for every list. A variant `Set` of the
`ImmutableArray` slot resolved to `[9:eeeeeeeeeeee]` with the base untouched, header `v2`, `HoldsIds` false.
A path step `[1:aaaaaaaaaaaa]` is refused: "this runtime resolves top-level Set overrides only". No C2a gap.
The gap belongs to C2b, in the cut sequence below.
- AllowPrivate is read from the most derived `[MessagePackObject]`. `ApDerived` (AllowPrivate over a base
without it) is written `[[],[],4,[]]`: base-chain privates and a private-getter property, with
`ObjectSerializationInfo` at `allowPrivate: true` = `[baseStash, own, N, Pg]`. `NoApDerived` (plain over an
AllowPrivate base) is written `[nil, 5]`. The walk read only `[N]` for both.
- A `[DataContract]` `List<T>` subclass is written by `NonGenericListFormatter`, and its record does not read
back ("System.Object[] is not of type PElem"), whatever the walk does. That is a MessagePack limit, not an
id fact.
## 2. Model page (step 0b)
| Kind | Named by | Over time | Decided by |
@ -99,16 +214,16 @@ Probes run 2026-10-12 from `F:\Projects\CultLib-c2afix` with git only (no builds
| Base reference | the base's key, in the record's variant slot | changed only by an explicit rebase; a base cannot be deleted while it has variants, unless the same batch rebases or removes them | author; the cache refuses a cycle, a missing base, a different concrete type, a global type |
| Override entry | `(op, path, id, value)`; op is Set, Insert or Remove | added by editing; removed by clearing; survives base edits while its path resolves | author; the cache refuses entries a base edit strands (Q2); an entry whose member left the type soft-drifts with a warning (Q4 c) |
| Member path | a sequence of `(slot, elementId)` steps; `slot` is the MessagePack `[Key]`; `elementId` is `""` unless the step enters an object-list element | a type change can invalidate it | the type's owner (slot authority); ids by the element's creator |
| Element id | a `[CultElementId]` string member on an object-list element type; `""` means unset | minted by the cache on write when unset; copied unchanged into variants; a moved element is a new element | cache mints it; it is never retargeted |
| Element id | a `[CultElementId]` string member on the element type of an object list in a shape the walk owns (arrays, `IEnumerable<T>`, `IDictionary`/`IReadOnlyDictionary` values, `[MessagePackObject]`/`[Union]` types with integer keys), one MessagePack writes and reads back (integer `[Key]`, no `[IgnoreMember]`; public, or any visibility under the most derived `AllowPrivate`); `""` means unset. **Exception (q3 amended 2026-10-16):** a shape the walk does not own is one whole value: `ImmutableArray`, `ArraySegment`, `Memory`, `KeyValuePair`, tuples, `Lazy`, unkeyed or `[DataContract]` classes, string-keyed types, a type or member with `[MessagePackFormatter]`, and a non-public member without `AllowPrivate`. Its elements carry no ids: a `[CultElementId]` inside is a plain string | minted by the cache on write when unset; copied unchanged into variants; a moved element is a new element. Inside a whole value: never minted, never checked, never counted for `v3` | cache mints it; it is never retargeted. `CultElementIds.Classify` alone decides owned versus whole |
| Resolved view | `CultStoredDocument.Document` for a variant key | recomputed inside the hold that admits the variant, or any base on its chain | the cache only; never persisted, never written back (Q1) |
| Index and name entries | as today, from the resolved view | a dependent's entries are dropped and re-added in the same `Apply` as its base change | the cache; a variant must override the `[CultName]` member (rule R6) |
| Watches | as today; each re-resolved dependent emits `Updated` with its own `Sequence` | published in the same hold as the base change | the cache |
| Compare-exchange | `(schemaId, storedAt)` of the stored delta (ruling 2) | a base change does not stamp its variants | the cache; the resolved view is not an identity |
| Persisted format version | store header `formatVersion` | `cultcache.store.v2` when the store holds at least one variant; `v1` otherwise (Q5) | the writing store, from content |
| Persisted format version | store header `formatVersion` | `cultcache.store.v2` when the store holds at least one variant; `v1` otherwise (Q5); a marked `v3` (`v5` directory) stays while a write copies any record forward, and only a write that encodes every record drops it by content | the writing store, from content; the durable header alone says the bytes hold element ids |
| Single-file store file | the path; one MessagePack array `[header, catalog, records]` | replaced whole and atomically; refused when unreadable; see "Store authorities" | each runtime's one reader decides what is a store; the writer derives the header from it |
| Store lock and staging files | `<file>.lock`; a unique temp beside the store | the lock is taken by C# and Rust writers (CultMesh included, through `ReplaceDurable`); staging is removed on a failed write | the store, never CultMesh or a consumer |
| Variant slot on the wire (CultNet) | **empty cell**: no CultNet message carries a delta today | C5 | ruling 4 decides deltas; the message shape is C5's design |
| Dictionary entries, scalar lists, arrays | **empty cell**: no stable identity exists | whole-member Set only (Q7) | Q7 |
| Dictionary entries, scalar lists, arrays, whole-list holders | **empty cell**: no stable identity exists | whole-member Set only (Q7; whole-list holders by q3 amended) | Q7, q3 amended |
| Template (non-item) bases | **empty cell**: parked by ruling 6 | none | not in these cuts |
## 3. Where the Body contradicts the target
@ -220,20 +335,21 @@ Dictionary entries, scalar lists and arrays are replaced whole.
- **Recommend yes.** Aetheria's cases (`Hardpoints` whole; `Behaviors[id]...Max`) fit. Keyed
dictionary addressing can be added later without a format change. Affects C2.
## Cut sequence (2026-10-12)
## Cut sequence (2026-10-15)
Mapped now (specs in the mind): `c2a-declared-apply` r2, the C# commit-time fix batch on
`hands/variants-c2a` (the stash, finished, with tests and contract text), and its sibling
`main-into-c2a` r2, main merged into C2a on `hands/variants-c2a-main` in its own worktree. Later
cuts, one line each, mapped only when Self is about to dispatch them:
Landed since 2026-10-12: `c2a-declared-apply` (r3, Soul s5) and `main-into-c2a` (r2, Soul s2); the
reconcile step is done, `hands/variants-c2a` is at the reconciled head `6afa8ed9`. Element ids part 1,
`c2a-element-id-holes` (r6), is held by Soul s5/s6 on `hands/variants-c2a-element-id-holes`. Mapped
now: part 2, `c2a-id-closed-world` r2 (2026-10-16, under the amended q3), not blocked. Its open fork
`variants:question:c2a-whole-list-opt-in` holds under every option. Later cuts, one line each, mapped
only when Self is about to dispatch them:
1. **reconcile**: merge `hands/variants-c2a` into `hands/variants-c2a-main` after both Soul passes
hold; Self moves `hands/variants-c2a` there. Waits on both mapped cuts.
1. **reconcile**: done at `6afa8ed9`.
2. **parity-ts**: TypeScript write set, one record per key, undeclared records carried and refused,
unheld removals judged at commit. Waits on reconcile.
unheld removals judged at commit. Waits on element ids parts 1 and 2 merged into C2a.
3. **parity-rs**: Rust `push_all` and `put_prepared_batch` apply a write set, ownership by
declaration, the same commit-time checks. Waits on reconcile; can run beside parity-ts.
4. **parity-py**: Python write-set apply. Waits on reconcile and on `cultlib-gaps` cut
declaration, the same commit-time checks. Waits as parity-ts; can run beside it.
4. **parity-py**: Python write-set apply. Waits as parity-ts, and on `cultlib-gaps` cut
`py-msgpack-default` landing on main (then a main merge into C2a first).
5. **write-set-vectors**: one cross-runtime manifest, read and acted on by all four runtimes;
carries follow-ups `c0-bad-canonical-in-manifest`, `legacy-vectors-all-runtimes`,
@ -242,6 +358,11 @@ cuts, one line each, mapped only when Self is about to dispatch them:
with an empty diff against the verified head; no tag. Waits on write-set-vectors, Soul holding,
and `asura:cut_spec:cut-semver-measured` landed. Asura Cut 3 bases on its merge commit.
Element-id part 2: `c2a-id-closed-world` r2 decides deferred findings `s5.member-set-misses-allowprivate`, `s5.enumerating-rule-unpinned`, `s6.null-standard-untyped`, `s6.two-recognisers-one-rule`, `s4.unrecognised-holders-fail-open` and follow-up `c2a-datacontract-holder-unwalked` (see its `first`).
- **marker-for-whole-lists** (only if `c2a-whole-list-opt-in` is ruled `marker-when-ids-declared` or `marker-for-any-keyed-object`): a CarriesElements predicate, a refusal unless the member is marked, and a public attribute. It is additive after part 2.
- **C2b constraint** (owed to `variants:follow_up:remaining-variant-cuts`): nested path resolution must refuse a step `[slot:elementId]` that enters a whole value, naming the holder. A whole value is any shape `CultElementIds.Classify` calls Leaf, so C2b asks Classify and keeps no recogniser of its own. Today's C2a resolver refuses every non-top-level step, so no store can hold such an override yet.
Rationale. The 2026-10-02 order (parity, vectors, then merge main) was right when main was close.
Main has since moved 453 commits and rebuilt exactly what the parity cuts change: cultcache-ts
payload ownership, cultmesh-ts, cultcache-rs. Parity mapped before the merge would be mapped
@ -1484,7 +1605,7 @@ The one door per runtime, and who walks through it:
- **Inputs.** The bytes at the path. **Outputs.** The decoded store with header and catalog; or a typed refusal (`CultStoreUnreadableException` in C#; `CultCacheStoreUnreadable{path, kind}` in Rust, kind `Undecodable` or `UnsupportedFormat`; `StoreUnreadableError` in TS and Python). A refusal names a header only as `cultcache.store.v<digits>`.
- **Derived state.** The write header is never decided by the writer: it is the reader's verdict (header read, whole-store flag, whether ids or variants are held). A rewrite of a file that cannot be read cannot happen, because the reader throws first.
- **Forbidden writers.** A second header classifier. Deleted: Rust `leading_header`, `read_array_header`, `read_string`; cultnet-ts's own decode of the store header; the TS async `readStore`; Rust `StoreUnreadableError` (negative grep `StoreUnreadableError` in `packages/cultcache-rs`: no match); Python `_decode_v1_snapshot`.
- **Shared paths.** cultnet-ts reaches the header decision only through `readSingleFileStore` (AUTH:4, :321) and checks only record count, schema id and type. cultnet-ts therefore declares cultcache-ts `^0.15.0`, the first release that exports it.
- **Shared paths.** cultnet-ts reaches the header decision only through `readSingleFileStore` (AUTH:4, :321) and checks only record count, schema id and type. cultnet-ts therefore needs a cultcache-ts release that exports it, which is the release that ships with this change; the manifests move to that version then (c0-ships-with-c2a), not before.
- **Cut line.** `merge-r1` h3 (Rust) and h4 (TS, cultnet-ts); spec `cut-merge-r1.r3` authority map.
- **Pinned by.** `tests/vectors/document-variants-c2a/readability/manifest.txt`, walked by `StoreReadabilityTests.OpenReadsExactlyTheFilesTheVectorsSayItReads` (C#), `CultMeshSingleFileReadabilityTests.AWriteReplacesAFileExactlyWhenItReads` (Mesh), `a_file_is_replaced_exactly_when_it_reads` (RS:5132), "SingleFileMessagePackBackingStore replaces a file exactly when it reads" (TS test :2321) and `test_single_file_replaces_a_file_exactly_when_it_reads` (Python). AUTH cases: `packages/cultnet-ts/test/idunn-runtime-authority.test.ts`. Declared divergences in the manifest: `legacy-envelopes.bin` (C# refuses; the others read and replace it) and `variant-slot-v3.bin` (C# reads; the others refuse).
- **Known gaps.** `variants:follow_up:c0-bad-canonical-in-manifest` (`current-catalog-bad-canonical.msgpack` is not in the manifest; TS reads it, Rust refuses). `variants:follow_up:legacy-vectors-all-runtimes` (the three `legacy-catalog-*` c0 vectors have a verdict only in C# tests; all four runtimes refuse them, probed). `variants:follow_up:range-check-in-ci` (`npm run test:pack:ts` builds every admitted tag and checks exports, but `.github/workflows/publish-packages.yml` does not run it).

272
docs/document-variants-id-sweep.md Normal file
View file

@ -0,0 +1,272 @@
# Element-id sweep list (C2a)
Data for the per-consumer sweep in docs/document-variants-cut.md, C2. Every object type below sits in a list (`List<T>`, `T[]`, `IEnumerable<T>`, and the like) reachable from a registered document type, and needs one string member with `[Key(n)]` and `[CultElementId]` before that consumer can take a CultLib that requires ids. A union base takes the member on the base; its subclasses inherit it, so put the slot above the subclasses' highest key.
Method: a static scan of `F:\Projects\<consumer>` C# files for types marked `[CultDocument(`, walked through keyed members, base classes, dictionary values and `[Union]` subtypes. Read-only; no consumer was edited. Worktree copies (`Aetheria-catalog`, `CultLib-*`, and so on) are excluded. Struct elements are values and need no id. Non-C# consumers carry the id as an ordinary member and are not listed. The scan is text-based: a document registered without a literal `[CultDocument(` attribute, or a type whose members lack an integer `[Key]`, is missed, so the sweep pass should confirm each repo by registering its types once.
Lines: `Type file:line (via Owner.Member)`, paths relative to the consumer root.
## Aetheria (13 element types)
- BehaviorData Assets/Scripts/ServerShared/Behaviors/Behaviors.cs:190 (via ConsumableItemData.Behaviors)
- EntityPack Assets/Scripts/ServerShared/EntitySerializer.cs:171 (via ZonePack.Entities)
- ProductRole Assets/Scripts/ServerShared/FactionProduct.cs:38 (via FactionProductData.Roles)
- InputLayoutRow Assets/Scripts/ServerShared/InputLayout.cs:32 (via InputLayout.Rows)
- InputLayoutColumn Assets/Scripts/ServerShared/InputLayout.cs:52 (via InputLayoutKeyRow.Columns)
- ItemRole Assets/Scripts/ServerShared/ItemData.cs:325 (via CraftedItemData.Roles)
- AudioStat Assets/Scripts/ServerShared/ItemData.cs:441 (via EquippableItemData.AudioStats)
- HardpointData Assets/Scripts/ServerShared/ItemData.cs:553 (via HullData.Hardpoints)
- StatTerm Assets/Scripts/ServerShared/ItemData.cs:630 (via PerformanceStat.Terms)
- LoadoutSlot Assets/Scripts/ServerShared/Loadout.cs:23 (via Loadout.Slots)
- RoleFill Assets/Scripts/ServerShared/Provenance.cs:84 (via Lot.Roles)
- SavedActionBarBinding Assets/Scripts/ServerShared/SavedGame.cs:168 (via SavedGame.ActionBarBindings)
- Asteroid Assets/Scripts/ServerShared/ZoneData.cs:133 (via AsteroidBeltData.Asteroids)
## AetheriaEve (115 element types)
- AetheriaCorporationAllegiance Aetheria.State/Documents/AetheriaCorporation.cs:67 (via AetheriaCorporation.Allegiances)
- AetheriaShapeCell Aetheria.State/Documents/AetheriaItemDefinition.cs:178 (via AetheriaItemDefinition.ShapeCells)
- AetheriaItemHardpoint Aetheria.State/Documents/AetheriaItemDefinition.cs:188 (via AetheriaItemDefinition.Hardpoints)
- AetheriaCurveKey Aetheria.State/Documents/AetheriaItemDefinition.cs:222 (via AetheriaItemDefinition.ThermalPerformanceCurveKeys)
- AetheriaItemAudioStat Aetheria.State/Documents/AetheriaItemDefinition.cs:238 (via AetheriaItemDefinition.AudioStats)
- AetheriaBehaviorPayload Aetheria.State/Documents/AetheriaItemDefinition.cs:267 (via AetheriaItemDefinition.BehaviorPayloads)
- AetheriaBehaviorField Aetheria.State/Documents/AetheriaItemDefinition.cs:286 (via AetheriaBehaviorPayload.Fields)
- AetheriaBehaviorMapEntry Aetheria.State/Documents/AetheriaItemDefinition.cs:296 (via AetheriaBehaviorValue.MapEntries)
- AetheriaBehaviorValue Aetheria.State/Documents/AetheriaItemDefinition.cs:306 (via AetheriaBehaviorValue.Children)
- AetheriaLegacyCatalogFile Aetheria.State/Documents/AetheriaLegacyCatalogQuarantine.cs:41 (via AetheriaLegacyCatalogQuarantine.NameFiles)
- AetheriaMigrationCount Aetheria.State/Documents/AetheriaMigrationLedger.cs:35 (via AetheriaMigrationLedger.Counts)
- AetheriaLegacyCatalogEntrySummary Aetheria.State/Documents/AetheriaMigrationLedger.cs:45 (via AetheriaMigrationLedger.LegacyCatalogEntries)
- AetheriaStoryFileHash Aetheria.State/Documents/AetheriaRuntimeStateDocuments.cs:42 (via AetheriaPlayerSettings.StoryFileHashes)
- AetheriaInputBindingOverride Aetheria.State/Documents/AetheriaRuntimeStateDocuments.cs:85 (via AetheriaPlayerInputSettings.BindingOverrides)
- AetheriaEntityLoadout Aetheria.State/Documents/AetheriaRuntimeStateDocuments.cs:119 (via AetheriaEntityLoadout.Children)
- AetheriaLoadoutItemSlot Aetheria.State/Documents/AetheriaRuntimeStateDocuments.cs:186 (via AetheriaEntityLoadout.Equipment)
- AetheriaCargoBayLoadout Aetheria.State/Documents/AetheriaRuntimeStateDocuments.cs:199 (via AetheriaEntityLoadout.CargoContents)
- AetheriaFactionZoneState Aetheria.State/Documents/AetheriaRuntimeStateDocuments.cs:268 (via AetheriaRunState.HomeZones)
- AetheriaAgentTaskState Aetheria.State/Documents/AetheriaRuntimeStateDocuments.cs:275 (via AetheriaRunState.AgentTasks)
- AetheriaDroppedPickupSnapshot Aetheria.State/Documents/AetheriaRuntimeStateDocuments.cs:346 (via AetheriaZoneState.DroppedPickups)
- AetheriaOrbitSnapshot Aetheria.State/Documents/AetheriaRuntimeStateDocuments.cs:362 (via AetheriaZoneState.Orbits)
- AetheriaBodySnapshot Aetheria.State/Documents/AetheriaRuntimeStateDocuments.cs:384 (via AetheriaZoneState.Bodies)
- AetheriaBodyResource Aetheria.State/Documents/AetheriaRuntimeStateDocuments.cs:449 (via AetheriaBodySnapshot.Resources)
- AetheriaAsteroidSnapshot Aetheria.State/Documents/AetheriaRuntimeStateDocuments.cs:459 (via AetheriaBodySnapshot.Asteroids)
- AetheriaAsteroidMiningAccumulatorSnapshot Aetheria.State/Documents/AetheriaRuntimeStateDocuments.cs:484 (via AetheriaAsteroidSnapshot.MiningAccumulators)
- AetheriaColor Aetheria.State/Documents/AetheriaRuntimeStateDocuments.cs:545 (via AetheriaGasGiantVisualState.Colors)
- AetheriaFogFieldEmitterSnapshot Aetheria.State/Documents/AetheriaRuntimeStateDocuments.cs:691 (via AetheriaEntitySnapshot.FogFieldEmitters)
- AetheriaLoadoutGenerationSelection Aetheria.State/Documents/AetheriaRuntimeStateDocuments.cs:723 (via AetheriaLoadoutGenerationReceipt.Selections)
- AetheriaEntityContactSnapshot Aetheria.State/Documents/AetheriaRuntimeStateDocuments.cs:734 (via AetheriaEntitySnapshot.Contacts)
- AetheriaEntityItemSlot Aetheria.State/Documents/AetheriaRuntimeStateDocuments.cs:751 (via AetheriaEntitySnapshot.Equipment)
- AetheriaWeaponGroupSnapshot Aetheria.State/Documents/AetheriaRuntimeStateDocuments.cs:782 (via AetheriaEntitySnapshot.WeaponGroups)
- AetheriaEntityStatGrid Aetheria.State/Documents/AetheriaRuntimeStateDocuments.cs:789 (via AetheriaEntitySnapshot.StatGrids)
- AetheriaActiveConsumableSnapshot Aetheria.State/Documents/AetheriaRuntimeStateDocuments.cs:805 (via AetheriaEntitySnapshot.ActiveConsumables)
- AetheriaConsumableBehaviorStateSnapshot Aetheria.State/Documents/AetheriaRuntimeStateDocuments.cs:828 (via AetheriaActiveConsumableSnapshot.BehaviorStates)
- AetheriaBehaviorProgressSnapshot Aetheria.State/Documents/AetheriaRuntimeStateDocuments.cs:844 (via AetheriaEntitySnapshot.BehaviorProgress)
- AetheriaWeaponStateSnapshot Aetheria.State/Documents/AetheriaRuntimeStateDocuments.cs:863 (via AetheriaEntitySnapshot.WeaponStates)
- AetheriaBehaviorStateSnapshot Aetheria.State/Documents/AetheriaRuntimeStateDocuments.cs:924 (via AetheriaEntitySnapshot.BehaviorStates)
- AetheriaFactionRelationshipState Aetheria.State/Documents/AetheriaRuntimeStateDocuments.cs:1093 (via AetheriaRunState.FactionRelationships)
- AetheriaItemRarityTier Aetheria.State/Documents/AetheriaTradeValuePolicy.cs:29 (via AetheriaTradeValuePolicy.Tiers)
- AetheriaRuntimeAgentTaskCommit Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeAgentTasks.cs:35 (via AetheriaRuntimeRunCheckpointCommit.AgentTasks)
- AetheriaRuntimeAssetManifestEntry Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeAssetDocuments.cs:102 (via AetheriaRuntimeAssetManifestDocument.Assets)
- AetheriaRuntimeCurveKey Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeCatalogSnapshot.cs:446 (via AetheriaRuntimeShotReceiptCommit.GuidanceCurve)
- AetheriaRuntimeCorporation Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeCatalogSnapshot.cs:620 (via AetheriaRuntimeCatalogSnapshot.Corporations)
- AetheriaRuntimeNameFile Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeCatalogSnapshot.cs:669 (via AetheriaRuntimeCatalogSnapshot.NameFiles)
- AetheriaRuntimeLoadoutTemplateSnapshot Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeCatalogSnapshot.cs:796 (via AetheriaRuntimeLoadoutTemplatesDocument.Templates)
- AetheriaRuntimeDaemonCommandBoundaryEntry Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeDaemonDocuments.cs:462 (via AetheriaRuntimeDaemonCommandBoundaryDocument.Commands)
- AetheriaRuntimeDaemonSoaIdentityDocument Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeDaemonSoaDocuments.cs:143 (via AetheriaRuntimeDaemonSoaViewDocument.Identities)
- AetheriaRuntimeDaemonRenderGroupDocument Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeDaemonSoaDocuments.cs:156 (via AetheriaRuntimeDaemonSoaViewDocument.RenderGroups)
- AetheriaRuntimeDaemonSoaBufferDocument Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeDaemonSoaDocuments.cs:222 (via AetheriaRuntimeDaemonSoaViewDocument.Buffers)
- AetheriaRuntimeDaemonSoaColumnDocument Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeDaemonSoaDocuments.cs:253 (via AetheriaRuntimeDaemonSoaViewDocument.Columns)
- AetheriaRuntimeDaemonSoaDirtyRangeDocument Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeDaemonSoaDocuments.cs:284 (via AetheriaRuntimeDaemonSoaViewDocument.DirtyRanges)
- AetheriaRuntimeEveSurfaceAdvertisement Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeEveSurfaceCatalog.cs:11 (via AetheriaRuntimeDaemonProviderAdvertisementDocument.EveSurfaces)
- AetheriaRuntimeEvePluginRequirement Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeEveSurfaceCatalog.cs:58 (via AetheriaRuntimeEveSurfaceAdvertisement.RequiresPlugins)
- AetheriaRuntimeZoneTargetRow Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeGameViewportDocuments.cs:410 (via AetheriaRuntimeZoneContactsDocument.Targets)
- AetheriaRuntimeZoneContactRow Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeGameViewportDocuments.cs:441 (via AetheriaRuntimeZoneContactsDocument.Contacts)
- AetheriaRuntimeStationStockItem Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeGameViewportDocuments.cs:551 (via AetheriaRuntimeStationRefitEntityOption.CargoItems)
- AetheriaRuntimeStationRefitEntityOption Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeGameViewportDocuments.cs:585 (via AetheriaRuntimeStationRefitDocument.AvailableEntities)
- AetheriaRuntimeStationDockingBayRow Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeGameViewportDocuments.cs:620 (via AetheriaRuntimeStationRefitDocument.DockingBays)
- AetheriaRuntimeStationLoadoutRestoreOption Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeGameViewportDocuments.cs:655 (via AetheriaRuntimeStationRefitDocument.LoadoutRestoreOptions)
- AetheriaRuntimeStationCargoTargetRow Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeGameViewportDocuments.cs:674 (via AetheriaRuntimeStationRefitDocument.CargoTargets)
- AetheriaRuntimeSectorMapZone Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeGameViewportDocuments.cs:765 (via AetheriaRuntimeSectorMapDocument.Zones)
- AetheriaRuntimeSectorMapLink Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeGameViewportDocuments.cs:802 (via AetheriaRuntimeSectorMapDocument.Links)
- AetheriaRuntimeZoneRenderAdjacentZone Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeGameViewportDocuments.cs:1197 (via AetheriaRuntimeZoneRenderDocument.AdjacentZones)
- AetheriaRuntimeZoneRenderWormholeExit Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeGameViewportDocuments.cs:1210 (via AetheriaRuntimeZoneRenderDocument.WormholeExits)
- AetheriaRuntimeZoneRenderBodyPose Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeGameViewportDocuments.cs:1229 (via AetheriaRuntimeZoneRenderDocument.BodyPoses)
- AetheriaRuntimeZoneRenderAsteroidBeltPose Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeGameViewportDocuments.cs:1260 (via AetheriaRuntimeZoneRenderDocument.AsteroidBeltPoses)
- AetheriaRuntimeZoneRenderAsteroidInstancePose Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeGameViewportDocuments.cs:1286 (via AetheriaRuntimeZoneRenderAsteroidBeltPose.InstancePoses)
- AetheriaRuntimeViewportObject Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeGameViewportDocuments.cs:1382 (via AetheriaRuntimeGameViewportDocument.Objects)
- AetheriaRuntimeInventoryItem Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeGameViewportDocuments.cs:1458 (via AetheriaRuntimeViewportObject.Inventory)
- AetheriaRuntimeGravityInfluence Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeGameViewportDocuments.cs:1493 (via AetheriaRuntimeGameViewportDocument.GravityInfluences)
- AetheriaRuntimeBodyView Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeGameViewportDocuments.cs:1530 (via AetheriaRuntimeGameViewportDocument.Bodies)
- AetheriaHangarShip Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeHangarDocuments.cs:119 (via AetheriaHangarState.Ships)
- AetheriaHangarItemStack Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeHangarDocuments.cs:129 (via AetheriaHangarState.Inventory)
- AetheriaHangarCurrency Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeHangarDocuments.cs:136 (via AetheriaHangarState.Currencies)
- AetheriaDeploymentReceipt Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeHangarDocuments.cs:155 (via AetheriaHangarState.Deployments)
- AetheriaProgressionVerseOption Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeProgressionSourceDocuments.cs:45 (via AetheriaProgressionSourceDocument.AvailableVerses)
- AetheriaRuntimeStoryFileHashDocument Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeSettingsDocuments.cs:87 (via AetheriaRuntimePlayerSettingsDocument.StoryFileHashes)
- AetheriaRuntimeInputBindingOverrideDocument Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeSettingsDocuments.cs:108 (via AetheriaRuntimePlayerSettingsDocument.BindingOverrides)
- AetheriaRuntimeEntityLoadoutCommit Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeSnapshotDocuments.cs:78 (via AetheriaRuntimeEntityLoadoutCommit.Children)
- AetheriaRuntimeLoadoutItemSlotCommit Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeSnapshotDocuments.cs:143 (via AetheriaRuntimeEntitySnapshotCommit.Equipment)
- AetheriaRuntimeCargoBayLoadoutCommit Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeSnapshotDocuments.cs:159 (via AetheriaRuntimeEntitySnapshotCommit.CargoContents)
- AetheriaRuntimeGameEventCommit Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeSnapshotDocuments.cs:490 (via AetheriaRuntimeRunCheckpointCommit.GameEvents)
- AetheriaRuntimeFactionZoneCommit Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeSnapshotDocuments.cs:509 (via AetheriaRuntimeRunCheckpointCommit.HomeZones)
- AetheriaRuntimeShotReceiptCommit Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeSnapshotDocuments.cs:539 (via AetheriaRuntimeRunCheckpointCommit.ShotReceipts)
- AetheriaRuntimePickupContactReceiptCommit Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeSnapshotDocuments.cs:591 (via AetheriaRuntimeRunCheckpointCommit.PickupContactReceipts)
- AetheriaRuntimeDamageCellCommit Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeSnapshotDocuments.cs:603 (via AetheriaRuntimeShotReceiptCommit.DamageCells)
- AetheriaRuntimeCorporationSurveyCommit Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeSnapshotDocuments.cs:614 (via AetheriaRuntimeRunCheckpointCommit.CorporationSurveys)
- AetheriaRuntimeFactionRelationshipCommit Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeSnapshotDocuments.cs:623 (via AetheriaRuntimeRunCheckpointCommit.FactionRelationships)
- AetheriaRuntimeZoneSnapshotCommit Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeSnapshotDocuments.cs:636 (via AetheriaRuntimeRunCheckpointCommit.Zones)
- AetheriaRuntimePhysicalPayloadCommit Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeSnapshotDocuments.cs:699 (via AetheriaRuntimeZoneSnapshotCommit.PhysicalPayloads)
- AetheriaRuntimeDroppedPickupCommit Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeSnapshotDocuments.cs:771 (via AetheriaRuntimeZoneSnapshotCommit.DroppedPickups)
- AetheriaRuntimeOrbitSnapshotCommit Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeSnapshotDocuments.cs:805 (via AetheriaRuntimeZoneSnapshotCommit.Orbits)
- AetheriaRuntimeBodySnapshotCommit Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeSnapshotDocuments.cs:830 (via AetheriaRuntimeZoneSnapshotCommit.Bodies)
- AetheriaRuntimeBodyResourceCommit Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeSnapshotDocuments.cs:901 (via AetheriaRuntimeBodySnapshotCommit.Resources)
- AetheriaRuntimeAsteroidCommit Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeSnapshotDocuments.cs:911 (via AetheriaRuntimeBodySnapshotCommit.Asteroids)
- AetheriaRuntimeAsteroidMiningAccumulatorCommit Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeSnapshotDocuments.cs:936 (via AetheriaRuntimeAsteroidCommit.MiningAccumulators)
- AetheriaRuntimeColorCommit Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeSnapshotDocuments.cs:1008 (via AetheriaRuntimeGasGiantVisualCommit.Colors)
- AetheriaRuntimeEntitySnapshotCommit Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeSnapshotDocuments.cs:1024 (via AetheriaRuntimeZoneSnapshotCommit.Entities)
- AetheriaRuntimeFogFieldEmitterCommit Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeSnapshotDocuments.cs:1198 (via AetheriaRuntimeEntitySnapshotCommit.FogFieldEmitters)
- AetheriaRuntimeEquipmentStateCommit Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeSnapshotDocuments.cs:1240 (via AetheriaRuntimeEntitySnapshotCommit.EquipmentStates)
- AetheriaRuntimeLoadoutGenerationSelectionCommit Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeSnapshotDocuments.cs:1264 (via AetheriaRuntimeLoadoutGenerationReceiptCommit.Selections)
- AetheriaRuntimeEntityContactCommit Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeSnapshotDocuments.cs:1275 (via AetheriaRuntimeEntitySnapshotCommit.Contacts)
- AetheriaRuntimeEntityStatGridCommit Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeSnapshotDocuments.cs:1291 (via AetheriaRuntimeEntitySnapshotCommit.StatGrids)
- AetheriaRuntimeActiveConsumableCommit Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeSnapshotDocuments.cs:1307 (via AetheriaRuntimeEntitySnapshotCommit.ActiveConsumables)
- AetheriaRuntimeConsumableBehaviorStateCommit Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeSnapshotDocuments.cs:1330 (via AetheriaRuntimeActiveConsumableCommit.BehaviorStates)
- AetheriaRuntimeBehaviorProgressCommit Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeSnapshotDocuments.cs:1355 (via AetheriaRuntimeEntitySnapshotCommit.BehaviorProgress)
- AetheriaRuntimeWeaponStateCommit Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeSnapshotDocuments.cs:1374 (via AetheriaRuntimeEntitySnapshotCommit.WeaponStates)
- AetheriaRuntimeBehaviorStateCommit Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeSnapshotDocuments.cs:1444 (via AetheriaRuntimeEntitySnapshotCommit.BehaviorStates)
- AetheriaRuntimeStarbridgeStationStockItem Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeStarbridgeDocuments.cs:71 (via AetheriaRuntimeStarbridgeScenarioDocument.StationStock)
- AetheriaRuntimeStarbridgeWaveDefinition Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeStarbridgeDocuments.cs:83 (via AetheriaRuntimeStarbridgeScenarioDocument.Waves)
- AetheriaRuntimeStarbridgeWaveForecast Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeStarbridgeDocuments.cs:94 (via AetheriaRuntimeStarbridgeSessionSummaryDocument.WaveForecast)
- AetheriaRuntimeStarbridgeRuntimeRole Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeStarbridgeDocuments.cs:104 (via AetheriaRuntimeStarbridgeScenarioDocument.RuntimeRoles)
- AetheriaRuntimeAuthorityRuntimeRole Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeVerseAuthorityPolicy.cs:194 (via AetheriaRuntimeVerseAuthorityPolicyDocument.RuntimeRoles)
- AetheriaRuntimeAuthorityRule Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeVerseAuthorityPolicy.cs:201 (via AetheriaRuntimeVerseAuthorityPolicyDocument.Rules)
- AetheriaRuntimeArenaSeat Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeVerseAuthorityPolicy.cs:255 (via AetheriaRuntimeArenaRosterDocument.Seats)
## AquaSynth (0 element types)
None.
## Aquarium (1 element types)
- AquariumFaceVoiceEndpointState src/Aquarium.Epiphany/State/AquariumLiveState.cs:61 (via AquariumLiveState.FaceVoiceEndpoints)
## Brokkr (4 element types)
- BrokkrGameObjectSnapshot surfaces/unity/Packages/com.gamecult.brokkr/Runtime/BrokkrHostSnapshot.cs:42 (via BrokkrHostSnapshot.sceneObjects)
- BrokkrComponentSnapshot surfaces/unity/Packages/com.gamecult.brokkr/Runtime/BrokkrHostSnapshot.cs:62 (via BrokkrGameObjectSnapshot.components)
- BrokkrSerializedPropertySnapshot surfaces/unity/Packages/com.gamecult.brokkr/Runtime/BrokkrHostSnapshot.cs:73 (via BrokkrComponentSnapshot.properties)
- BrokkrAssetSnapshot surfaces/unity/Packages/com.gamecult.brokkr/Runtime/BrokkrHostSnapshot.cs:84 (via BrokkrHostSnapshot.assets)
## Delvehold (0 element types)
None.
## Eve (17 element types)
- EveAssetCatalogEntry packages/org.gamecult.eve.surface/Runtime/EveAssetCatalogDocument.cs:52 (via EveAssetCatalogDocument.Assets)
- EveAssetVariant packages/org.gamecult.eve.surface/Runtime/EveAssetCatalogDocument.cs:89 (via EveAssetCatalogEntry.Variants)
- EveEntitySoaBuffer packages/org.gamecult.eve.surface/Runtime/EveEntitySoaViewDocument.cs:33 (via EveEntitySoaViewDocument.Buffers)
- EveEntitySoaColumn packages/org.gamecult.eve.surface/Runtime/EveEntitySoaViewDocument.cs:41 (via EveEntitySoaViewDocument.Columns)
- EveEntitySoaDirtyRange packages/org.gamecult.eve.surface/Runtime/EveEntitySoaViewDocument.cs:55 (via EveEntitySoaViewDocument.DirtyRanges)
- EveEntityIdentity packages/org.gamecult.eve.surface/Runtime/EveEntitySoaViewDocument.cs:64 (via EveEntitySoaViewDocument.Identities)
- EveEntityRenderGroup packages/org.gamecult.eve.surface/Runtime/EveEntitySoaViewDocument.cs:77 (via EveEntitySoaViewDocument.RenderGroups)
- EveInputActionDocument packages/org.gamecult.eve.surface/Runtime/EveInputCapabilityDocument.cs:22 (via EveInputCapabilityDocument.Actions)
- EveInputProfileDocument packages/org.gamecult.eve.surface/Runtime/EveInputCapabilityDocument.cs:51 (via EveInputCapabilityDocument.DefaultProfiles)
- EveInputBindingDocument packages/org.gamecult.eve.surface/Runtime/EveInputCapabilityDocument.cs:59 (via EveInputProfileDocument.Bindings)
- EveProviderWitness packages/org.gamecult.eve.surface/Runtime/EveProviderDocuments.cs:249 (via EveProviderAdvertisementDocument.Witnesses)
- EveAdvertisedSurface packages/org.gamecult.eve.surface/Runtime/EveProviderDocuments.cs:265 (via EveProviderAdvertisementDocument.Surfaces)
- EveAdvertisedCommand packages/org.gamecult.eve.surface/Runtime/EveProviderDocuments.cs:333 (via EveProviderAdvertisementDocument.Commands)
- EveSurfaceComponent packages/org.gamecult.eve.surface/Runtime/EveSurfaceDocument.cs:106 (via EveSurfaceComponent.Children)
- EveEmbeddedDocumentSlot packages/org.gamecult.eve.surface/Runtime/EveSurfaceDocument.cs:230 (via EveSurfaceComponent.EmbeddedDocuments)
- EveStyleToken packages/org.gamecult.eve.surface/Runtime/EveSurfaceDocument.cs:277 (via EveSurfaceTree.Styles)
- EveCommandTemplate packages/org.gamecult.eve.surface/Runtime/EveSurfaceDocument.cs:295 (via EveSurfaceDocument.Commands)
## EvePlugins (1 element types)
- EveFieldsSplatLayer plugins/eve-plugin-fields/unity/org.gamecult.eve.plugin-fields/Runtime/EveFieldsDocuments.cs:38 (via EveFieldsSplatsDocument.Layers)
## Ghostlight (17 element types)
These types are a vendored copy of Eve (`vendor/eve`); they change when Eve does and the pin bumps.
- EveAssetCatalogEntry vendor/eve/packages/org.gamecult.eve.surface/Runtime/EveAssetCatalogDocument.cs:52 (via EveAssetCatalogDocument.Assets)
- EveAssetVariant vendor/eve/packages/org.gamecult.eve.surface/Runtime/EveAssetCatalogDocument.cs:89 (via EveAssetCatalogEntry.Variants)
- EveEntitySoaBuffer vendor/eve/packages/org.gamecult.eve.surface/Runtime/EveEntitySoaViewDocument.cs:33 (via EveEntitySoaViewDocument.Buffers)
- EveEntitySoaColumn vendor/eve/packages/org.gamecult.eve.surface/Runtime/EveEntitySoaViewDocument.cs:41 (via EveEntitySoaViewDocument.Columns)
- EveEntitySoaDirtyRange vendor/eve/packages/org.gamecult.eve.surface/Runtime/EveEntitySoaViewDocument.cs:55 (via EveEntitySoaViewDocument.DirtyRanges)
- EveEntityIdentity vendor/eve/packages/org.gamecult.eve.surface/Runtime/EveEntitySoaViewDocument.cs:64 (via EveEntitySoaViewDocument.Identities)
- EveEntityRenderGroup vendor/eve/packages/org.gamecult.eve.surface/Runtime/EveEntitySoaViewDocument.cs:77 (via EveEntitySoaViewDocument.RenderGroups)
- EveInputActionDocument vendor/eve/packages/org.gamecult.eve.surface/Runtime/EveInputCapabilityDocument.cs:22 (via EveInputCapabilityDocument.Actions)
- EveInputProfileDocument vendor/eve/packages/org.gamecult.eve.surface/Runtime/EveInputCapabilityDocument.cs:51 (via EveInputCapabilityDocument.DefaultProfiles)
- EveInputBindingDocument vendor/eve/packages/org.gamecult.eve.surface/Runtime/EveInputCapabilityDocument.cs:59 (via EveInputProfileDocument.Bindings)
- EveProviderWitness vendor/eve/packages/org.gamecult.eve.surface/Runtime/EveProviderDocuments.cs:249 (via EveProviderAdvertisementDocument.Witnesses)
- EveAdvertisedSurface vendor/eve/packages/org.gamecult.eve.surface/Runtime/EveProviderDocuments.cs:265 (via EveProviderAdvertisementDocument.Surfaces)
- EveAdvertisedCommand vendor/eve/packages/org.gamecult.eve.surface/Runtime/EveProviderDocuments.cs:333 (via EveProviderAdvertisementDocument.Commands)
- EveSurfaceComponent vendor/eve/packages/org.gamecult.eve.surface/Runtime/EveSurfaceDocument.cs:106 (via EveSurfaceComponent.Children)
- EveEmbeddedDocumentSlot vendor/eve/packages/org.gamecult.eve.surface/Runtime/EveSurfaceDocument.cs:230 (via EveSurfaceComponent.EmbeddedDocuments)
- EveStyleToken vendor/eve/packages/org.gamecult.eve.surface/Runtime/EveSurfaceDocument.cs:277 (via EveSurfaceTree.Styles)
- EveCommandTemplate vendor/eve/packages/org.gamecult.eve.surface/Runtime/EveSurfaceDocument.cs:295 (via EveSurfaceDocument.Commands)
## Gjallar (3 element types)
- GjallarAdvertisedSchema src/Gjallar/VerseState.cs:462 (via GjallarProviderAdvertisementRecord.Schemas)
- GjallarAdvertisedWitness src/Gjallar/VerseState.cs:470 (via GjallarProviderAdvertisementRecord.Witnesses)
- GjallarAdvertisedSurface src/Gjallar/VerseState.cs:480 (via GjallarProviderAdvertisementRecord.Surfaces)
## Heimdall (12 element types)
These types are a vendored copy of CultLib (`vendor/CultLib`); C2a already adds the ids in CultLib, so they resolve when the pin bumps. The witness bundle types are the fork reported with C2a.
- CultGeometryDomainNode vendor/CultLib/src/GameCult.Geometry/CultGeometryDocuments.cs:61 (via CultGeometryDomainNode.Children)
- CultGeometryFeatureClaim vendor/CultLib/src/GameCult.Geometry/CultGeometryDocuments.cs:106 (via CultGeometryDomainNode.Claims)
- CultGeometryContributionRow vendor/CultLib/src/GameCult.Geometry/CultGeometryDocuments.cs:262 (via CultGeometrySelectedCutManifest.Diagnostics)
- CultMeshBodyDescriptor vendor/CultLib/src/GameCult.Mesh/CultMeshBodies.cs:35 (via CultMeshBodyPublicationDocument.Representations)
- CultMeshCdnChunkRef vendor/CultLib/src/GameCult.Mesh/CultMeshCdn.cs:73 (via CultMeshCdnArtifactManifest.Chunks)
- CultMeshDiscoveryCandidateDocument vendor/CultLib/src/GameCult.Mesh/CultMeshDiscoveryService.cs:418 (via CultMeshDiscoveryStateDocument.Candidates)
- CultMeshEntityPrefabComponent vendor/CultLib/src/GameCult.Mesh/CultMeshEntityPrefabs.cs:81 (via CultMeshEntityPrefabNode.Components)
- CultMeshEntityPrefabAssetRef vendor/CultLib/src/GameCult.Mesh/CultMeshEntityPrefabs.cs:96 (via CultMeshEntityPrefabPackage.Assets)
- CultMeshEntityPrefabNode vendor/CultLib/src/GameCult.Mesh/CultMeshEntityPrefabs.cs:127 (via CultMeshEntityPrefabPackage.Nodes)
- CultWitnessContractPin vendor/CultLib/src/GameCult.Networking/CultWitnessArtifactBundle.cs:105 (via CultWitnessArtifactBundle.Contracts)
- CultWitnessArtifactEntry vendor/CultLib/src/GameCult.Networking/CultWitnessArtifactBundle.cs:136 (via CultWitnessArtifactBundle.Artifacts)
- CultWitnessTimingEntry vendor/CultLib/src/GameCult.Networking/CultWitnessArtifactBundle.cs:179 (via CultWitnessArtifactBundle.TimingWitnesses)
## Mimir (0 element types)
None.
## StreamPixels (20 element types)
`CultGeometryDomainNode` is the vendored CultLib copy; C2a already adds its id.
- ActivationCondition apps/overlay-unity/Assets/Scripts/Assembly-CSharp/ActivationCondition.cs:18 (via CharacterConditions.Conditions)
- BattleEffect apps/overlay-unity/Assets/Scripts/Assembly-CSharp/BattleEffect.cs:8 (via BattleAction.Effects)
- GearStatEffect apps/overlay-unity/Assets/Scripts/Assembly-CSharp/GearStatEffect.cs:7 (via GearData.StatEffects)
- OverlaySceneIntent apps/overlay-unity/Assets/Scripts/Assembly-CSharp/ServiceLink/OverlayFrameDocuments.cs:33 (via OverlaySceneIntentWindow.Intents)
- PlayedIntentRecord apps/overlay-unity/Assets/Scripts/Assembly-CSharp/ServiceLink/PlayedIntentLedger.cs:25 (via OverlayPlayedIntents.Entries)
- StatModifier apps/overlay-unity/Assets/Scripts/Assembly-CSharp/StatModifier.cs:7 (via GearStatEffect.Modifiers)
- StatusEffect apps/overlay-unity/Assets/Scripts/Assembly-CSharp/StatusEffect.cs:10 (via StatusData.Effects)
- WeaponDamageTypeMultiplier apps/overlay-unity/Assets/Scripts/Assembly-CSharp/WeaponDamageTypeMultiplier.cs:7 (via WeaponData.DamageMultipliers)
- CultGeometryDomainNode vendor/CultLib/src/GameCult.Geometry/CultGeometryDocuments.cs:61 (via CultGeometryDomainNode.Children)
- CultGeometryFeatureClaim vendor/CultLib/src/GameCult.Geometry/CultGeometryDocuments.cs:106 (via CultGeometryDomainNode.Claims)
- CultGeometryContributionRow vendor/CultLib/src/GameCult.Geometry/CultGeometryDocuments.cs:262 (via CultGeometrySelectedCutManifest.Diagnostics)
- CultMeshBodyDescriptor vendor/CultLib/src/GameCult.Mesh/CultMeshBodies.cs:35 (via CultMeshBodyPublicationDocument.Representations)
- CultMeshCdnChunkRef vendor/CultLib/src/GameCult.Mesh/CultMeshCdn.cs:73 (via CultMeshCdnArtifactManifest.Chunks)
- CultMeshDiscoveryCandidateDocument vendor/CultLib/src/GameCult.Mesh/CultMeshDiscoveryService.cs:418 (via CultMeshDiscoveryStateDocument.Candidates)
- CultMeshEntityPrefabComponent vendor/CultLib/src/GameCult.Mesh/CultMeshEntityPrefabs.cs:81 (via CultMeshEntityPrefabNode.Components)
- CultMeshEntityPrefabAssetRef vendor/CultLib/src/GameCult.Mesh/CultMeshEntityPrefabs.cs:96 (via CultMeshEntityPrefabPackage.Assets)
- CultMeshEntityPrefabNode vendor/CultLib/src/GameCult.Mesh/CultMeshEntityPrefabs.cs:127 (via CultMeshEntityPrefabPackage.Nodes)
- CultWitnessContractPin vendor/CultLib/src/GameCult.Networking/CultWitnessArtifactBundle.cs:105 (via CultWitnessArtifactBundle.Contracts)
- CultWitnessArtifactEntry vendor/CultLib/src/GameCult.Networking/CultWitnessArtifactBundle.cs:136 (via CultWitnessArtifactBundle.Artifacts)
- CultWitnessTimingEntry vendor/CultLib/src/GameCult.Networking/CultWitnessArtifactBundle.cs:179 (via CultWitnessArtifactBundle.TimingWitnesses)
## Ymir (1 element types)
- YmirTransportLoweringDocument src/Ymir.Core/YmirServicePublicationDocument.cs:63 (via YmirProviderAdvertisementDocument.CommandLowerings)
## CultLib itself
C2a adds ids to: `CultGeometryDomainNode`, `CultGeometryFeatureClaim`, `CultGeometryContributionRow` (`src/GameCult.Geometry/CultGeometryDocuments.cs`); `CultMeshBodyDescriptor` (`src/GameCult.Mesh/CultMeshBodies.cs:35`); `CultMeshDiscoveryCandidateDocument` (`CultMeshDiscoveryService.cs:418`); `CultMeshEntityPrefabAssetRef`, `CultMeshEntityPrefabComponent`, `CultMeshEntityPrefabNode` (`CultMeshEntityPrefabs.cs`); `CultWitnessContractPin`, `CultWitnessArtifactEntry`, `CultWitnessTimingEntry` (`src/GameCult.Networking/CultWitnessArtifactBundle.cs`).
`CultMeshCdnChunkRef` (`src/GameCult.Mesh/CultMeshCdn.cs:73`) has a derived id, `[CultElementId(nameof(Offset))]`: the decimal text of its offset, never random, so packing the same body encodes the same manifest bytes. `contracts/cultmesh/content-vectors.*.json` and the Rust runtime carry it. A consumer with content-addressed elements declares the same way.

59
docs/research/2026-10-22-consumer-pinning-at-breaking-land.md Normal file
View file

@ -0,0 +1,59 @@
# Consumer pinning at a breaking land: prior art and internal state
Serves fork `variants:question:unpinned-consumers-at-land` (campaign `variants`, session `self-2026-10-22-eureka-c2a`).
Eyes pass, 2026-10-22. Facts and evidence pointers only; no recommendation.
Question as admitted: Eve.Surface and Brokkr build against an unpinned sibling `..\CultLib` source checkout. C2a (refuse element types with no id) lands on main. Options: pin-then-land, sweep-then-land, land-and-break.
## Sources
| # | Source | URL | Status |
|---|---|---|---|
| S1 | Abseil, About / philosophy | https://abseil.io/about/philosophy | fetched |
| S2 | Abseil, compatibility guidelines | https://abseil.io/about/compatibility | fetched |
| S3 | Software Engineering at Google, ch. 21 Dependency Management | https://abseil.io/resources/swe-book/html/ch21.html | fetched |
| S4 | Software Engineering at Google, ch. 16 Version Control and Branch Management | https://abseil.io/resources/swe-book/html/ch16.html | fetched (does not cover live at head) |
| S5 | Russ Cox, Minimal Version Selection | https://research.swtch.com/vgo-mvs | fetched |
| S6 | Cargo book, Cargo.toml vs Cargo.lock | https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html | fetched |
| S7 | npm docs, package-lock.json | https://docs.npmjs.com/cli/v10/configuring-npm/package-lock-json | fetched |
| S8 | Chromium src, dependencies doc (DEPS, gclient, roll-dep) | https://chromium.googlesource.com/chromium/src/+/main/docs/dependencies.md | fetched |
| S9 | Pro Git, Submodules | https://git-scm.com/book/en/v2/Git-Tools-Submodules | fetched |
| S10 | chromium infra dependencies.md | https://chromium.googlesource.com/infra/infra/+/main/docs/dependencies.md | 404, not retrieved |
| L1 | `F:\Projects\Aetheria\Directory.Build.targets`, `Directory.Build.props`, git history | local files | read |
| L2 | `F:\Projects\Aetheria\docs\cultcache-migration-cut.md` (Q6 table, lines ~565-580; pin guard notes ~1900-1925) | local file | read |
| L3 | `F:\Projects\Eve` and `F:\Projects\Brokkr` csprojs and git file lists | local files | read |
| L4 | voidbot `search_history` (two queries), `search_sources` (one query) | MCP | queried |
Anything not in a "fetched" row above is not claimed from the web.
## 1. Established practice: who pays for a breaking upstream change, and when
- **Abseil / live at head (S1, S2).** Recommends users track the latest code. Says it will provide a tool to perform an API refactor for well-behaved code (S1). Requires of downstream: show how you call it, stay within the compatibility guidelines; otherwise fork or use periodic supported branches (about every 6 months, patched 2 years) (S1). Says every change is potentially breaking for someone; breaks are announced in commit messages and LTS notes, and users are asked to report undetected breaks on GitHub (S2). Users should be able to temporarily pin to an earlier commit if a newer change is a problem (S2). S1/S2 do not mention Rosie, the Beyonce rule, or caller-fixing; the fetched pages describe tooling and a feedback channel only.
- **Live at Head as described in SWE at Google ch. 21 (S3).** Providers are expected to test changes against downstream dependents before committing; breaking changes wait until dependents are updated or an automated migration tool exists. Consumers must keep meaningful tests. Stated preconditions: unit tests across dependencies run in CI, a visible reverse-dependency graph, substantial compute, no pinning of dependencies, provider acceptance of the downstream-testing duty. Called "costly but effective" inside Google and "largely unproven at scale." Notes that under SemVer a major bump costs maintainers little and many indirect users a lot. Ch. 16 (S4) does not address live at head; it states large-scale changes across trunk are costly and tooling-supported.
- **Go MVS (S5).** Selects the oldest version satisfying all declared minimums. Publishing a new version changes nobody's build; a consumer gets it only when a requirement names it, directly or through another module. Upgrades are explicit edits to `go.mod` by the top-level module. Compatibility rule: newer versions inside one major version are expected to work where older did; a major version is a different module identity. Cost of a breaking change falls on whoever opts into the new version, at the time they opt in.
- **Cargo (S6).** Git dependency without `rev` takes the default-branch tip at first build and fixes that commit in `Cargo.lock`; changes arrive only on `cargo update`. A `rev` in `Cargo.toml` pins explicitly; the Cargo book calls hand-managed SHAs "tedious and error prone" and prefers the lockfile. Consumer pays at `cargo update` time.
- **npm (S7).** `package-lock.json` records the exact tree; installs stay on it until the lockfile changes and is committed. It is not published with a package. The page gives no timeline for when updates reach a consumer.
- **Chromium DEPS (S8).** `DEPS` stores dependency URLs and commit hashes (often in `vars`). Developers roll with `gclient setdep`, `roll-dep`, or manual `git update-index`; some projects (Skia) are autorolled. The page says manual rolling after an upstream change is good practice to confirm the roll compiles and tests pass. It does not describe revert procedure. Cost is paid by the roller at roll time.
- **Git submodules (S9).** The superproject records one commit SHA (gitlink, mode 160000). Upstream commits never move the pin; it moves when someone updates the checkout and commits the new SHA in the superproject. Collaborators receive the pin via pull and must run `git submodule update`.
## 2. Internal precedent
- **Aetheria pin (L1).** `Directory.Build.props` holds `CultLibRevision` (currently `45c2f4006f5f7dab357426cabd3197e8d0a17f66`) and `CultMathRevision` (`9c1b06cd703bfa44352050e819b0d0659cf5978a`). `Directory.Build.targets` target `VerifyCultLibRevision` (BeforeTargets `ResolveProjectReferences`) errors if the sibling checkout's `git rev-parse HEAD` differs from the pin, or if `git status --porcelain` is non-empty; escape hatch `SkipCultLibRevisionCheck=true`. File is 39 lines; the pin is one pair of properties per pinned package.
- **Why added (L1 git).** Commit `4f4beb0f` (2026-09-14, "Pin CultLib for headless builds"): "Directory.Build.targets carries Delvehold's VerifyCultLibRevision guard, checking for CultMath.csproj: HEAD must equal the pin and the worktree must be clean." Commit `e91ba369` (2026-09-24, "Pin CultLib per package, as Unity's manifest does"): headless build must compile the revisions Unity consumes; a single pin would have compiled six commits of newer CultCache under headless tests while Unity stays on its tag. The Aetheria cut doc (L2, ~line 1905) calls the guard "the only working pin for a sibling checkout." Aetheria's `Packages/manifest.json` pins Unity packages by git tag.
- **Pin guard already in Delvehold (L4/search_sources hit, `Delvehold:Directory.Build.targets`)** and a variant in AetheriaEve (archived, `Aetheria.State.Dependencies.targets`, also checks a Unity package tree hash). Delvehold's guard checks `GameCult.Mesh.csproj` exists, HEAD equals `CultLibRevision`, tree clean. Date of Delvehold's first guard: not determined.
- **Consumer table (L2, Q6 in Aetheria `docs/cultcache-migration-cut.md`).** Aquarium, Ymir, AquaSynth, Mimir, Gjallar: ".NET sibling, unpinned", compile against main at once. Brokkr, Eve, EveUnity, EvePlugins: "Unity package / sibling", pin "various". Delvehold: sibling with revision guard at `334e60f`, 52 behind, "Not until it re-pins." Heimdall: submodule `b6b1d9c`. Rust consumers (Idunn, Epiphany, Odin, Ghostlight, Muninn, Ratatoskr, CodexConnector): Cargo by commit.
- **Operator discussion (L4).** voidbot `search_history` ("pin CultLib revision vs track main for consumers"; "Brokkr Eve sibling CultLib checkout build pin") returned no relevant operator statement on pinning versus tracking; hits were unrelated Discord chatter. Nearest operator words found in the Discord index: Metacrat, 2026-05-09, "Oh, anyone using CultLib will still be paying the blood price, you can count on that" (message 1502671320406687894; context not fetched, so its subject is not confirmed). No operator quote on pinning versus tracking was found in memory files grepped (`C:\Users\Meta\.claude\projects\F--Projects\memory`, pattern pin/track main/unpinned/sibling) beyond a retired Idunn pin line and Odin/Muninn pin-branch coordination notes.
- Aetheria cut doc operator mentions near pins are about content (Schematic-shape drawer, 2026-09-14; Studio grouping, 2026-09-17), not pinning policy.
## 3. Current state of the two consumers (L3)
- **Eve.** `packages/org.gamecult.eve.surface/GameCult.Eve.Surface.csproj:16` `CultLibRoot` defaults to `$(MSBuildThisFileDirectory)..\..\..\CultLib`; lines 20-21 ProjectReference `$(CultLibRoot)\src\GameCult.Caching\...` and `...\GameCult.Mesh\...`. `tests/GameCult.Eve.Surface.Serialization.Tests/*.csproj:19` references `$(CultLibRoot)\src\GameCult.Mesh\...` (and Eve.Surface by relative path). `git ls-files` shows only `packages/org.gamecult.eve.surface/Directory.Build.props` (sets only BaseOutputPath/BaseIntermediateOutputPath/MSBuildProjectExtensionsPath, no revision). No `Directory.Build.targets`, no `global.json`, no `.gitmodules`, no revision property or check. Eve also holds Cargo.toml/Cargo.lock (Rust side, not examined).
- **Brokkr.** `brokkr-command/Brokkr.Command.csproj:7`, `brokkr-contracts.Tests/Brokkr.Contracts.Tests.csproj:7`, `brokkr-contracts/Brokkr.Contracts.csproj:8`: `CultLibRoot` defaults to `..\..\CultLib`. ProjectReferences: Caching, Caching.MessagePack, Mesh, Networking (by project). No `Directory.Build.*`, no `.gitmodules` in `git ls-files`; no revision property or check.
- Neither repo contains `CultLibRevision`, a `VerifyCultLibRevision` target, or any HEAD check. Both build whatever `F:\Projects\CultLib` currently holds (including dirty or other-branch states).
- Fork text claims (not re-verified here): Eve.Surface has 0 `[CultElementId]`; Brokkr `[CultDocument]` files show no object lists by grep.
## Gaps
- S10 (Chromium infra) 404; Chromium autoroller behavior beyond S8 not fetched.
- Rosie / Beyonce rule not found in fetched pages; not cited from memory.
- No operator statement on pin-versus-track found in voidbot or the memory grep.

7
docs/runtime-parity-scope.md
View file

@ -15,6 +15,9 @@ Cross-runtime parity has three layers:
without translation folklore.
Every store reader refuses, by name, a header or record slot it does not
understand (vectors: `tests/vectors/document-variants-c0`).
Element ids: a store that can hold them declares `cultcache.store.v3`; TypeScript,
Python and Rust read it and keep the header, and readers older than ids refuse it
(vectors: `tests/vectors/document-variants-c2a`).
Document variants: C# resolves them (`cultcache.store.v2`); every other runtime,
CultMesh single-file reads, the directory store and CultNet refuse a variant by name
(vector written by C#: `tests/vectors/document-variants-c1`).
@ -41,8 +44,8 @@ to speak CultNet or participate in CultMesh.
| C# | Reference implementation and production game/server substrate | CultCache typed documents, CultNet schema-v0, LiteNetLib production path with adapter profile, shared transport profiles, shared RUDP socket transport, C# multi-peer RUDP listener and schema service host, C# interop RUDP serve/dial path, connected RUDP CultMesh client helpers, RUDP-backed schema-client selection for `rudp://` service endpoints, RUDP reconnect policy/controller and reconnect-loop helper, LiteNetLib client reconnect-controller adoption, schema-client transport port for service request/response clients, CultMesh node/session/catalog/stream/simulation/RUDP surfaces, authorized peer lookup | No need to abandon LiteNetLib before the CultNet adapter fully owns production service adoption |
| Python | Package runtime, local daemon endpoint, raw interop peer, typed state participant | CultCache/CultNet/CultMesh package surface, branded CultMesh node/schema/shard/Verse/peer/authority/stream/RUDP ergonomics, authorized peer lookup, shard catalogs/logs, schema-transport factory for raw client request/subscription flows, local TCP and RUDP CultMesh server listeners, Python interop RUDP serve/dial path, session/prediction/simulation fact helpers, RUDP codec/session/socket transport, RUDP reconnect policy/controller and reconnect-loop helper, schema-v0 MessagePack over RUDP evidence, benchmark evidence against C# cache paths | Universal C# throughput, full C# daemon/server stack, and RUDP as the only public service pipe |
| TypeScript | Node/browser-adjacent projection and interop runtime | CultCache parity tests, CultNet schema-v0 peer, advertised schema/shard catalog wire contracts plus request/sync helpers, transport profiles, TCP-framed transport port and peer helper, shared RUDP socket transport, RUDP interop serve/dial path, RUDP reconnect policy/controller, RUDP reconnect-loop helper, branded CultMesh node/schema/shard/Verse/peer/authority/stream/RUDP ergonomics, authorized peer lookup | Full C# game session/server stack |
| Rust | Low-level cache/transport/runtime substrate | CultCache/CultNet interop lane, schema and shard catalog helpers, typed same-schema alias hydration from raw snapshots, transport profiles, TCP-framed port, shared RUDP codec/session/socket transport, Rust interop RUDP serve/dial path, connected RUDP CultMesh client helpers, RUDP reconnect policy/controller and reconnect-loop helper, schema-v0 MessagePack over RUDP evidence, small CultMesh facade for schema registry, shard catalog, peer cards, authority leases, authorized peer lookup, RUDP endpoint parsing, and peer-card/authorized-peer RUDP client construction | Full CultMesh game-session/server facade or production game-session server. `packages/cultmesh-rs` now holds a Rust stream and session surface -- node, document sets, RUDP document publication, stream descriptors, negotiation, stream catalog, and shared-memory frame rings -- but no interop lane exercises it, so it carries no cross-runtime parity claim |
| Kotlin | JVM/Android client substrate | Typed MessagePack cache, one-call typed snapshot document sync through local same-schema aliases, built-in schema catalog advertisement, schema/shard catalogs with shared schema-message transport fetch/sync helpers, authority leases, authorized peer lookup, stream catalog, branded `CultMesh` facade, WebSocket transport profile/connection helper, RUDP client/server sugar, connected RUDP CultMesh client helpers, Kotlin interop RUDP serve/dial path, RUDP reconnect policy/controller and reconnect-loop helper, schema-v0 MessagePack over RUDP evidence | C# SoA store (parked at `parked/cultcache-soa`), full C# server daemon, or Unity hot-loop memory layout |
| Rust | Low-level cache/transport/runtime substrate | CultCache/CultNet interop lane, schema and shard catalog helpers, typed hydration of raw snapshots into the receiver's own registered type for each schema id, transport profiles, TCP-framed port, shared RUDP codec/session/socket transport, Rust interop RUDP serve/dial path, connected RUDP CultMesh client helpers, RUDP reconnect policy/controller and reconnect-loop helper, schema-v0 MessagePack over RUDP evidence, small CultMesh facade for schema registry, shard catalog, peer cards, authority leases, authorized peer lookup, RUDP endpoint parsing, and peer-card/authorized-peer RUDP client construction | Full CultMesh game-session/server facade or production game-session server. `packages/cultmesh-rs` now holds a Rust stream and session surface -- node, document sets, RUDP document publication, stream descriptors, negotiation, stream catalog, and shared-memory frame rings -- but no interop lane exercises it, so it carries no cross-runtime parity claim |
| Kotlin | JVM/Android client substrate | Typed MessagePack cache, one-call typed snapshot document sync into the receiver's own registered type for the schema, built-in schema catalog advertisement, schema/shard catalogs with shared schema-message transport fetch/sync helpers, authority leases, authorized peer lookup, stream catalog, branded `CultMesh` facade, WebSocket transport profile/connection helper, RUDP client/server sugar, connected RUDP CultMesh client helpers, Kotlin interop RUDP serve/dial path, RUDP reconnect policy/controller and reconnect-loop helper, schema-v0 MessagePack over RUDP evidence | C# SoA store (parked at `parked/cultcache-soa`), full C# server daemon, or Unity hot-loop memory layout |
## Evidence Gates

2
package.json
View file

@ -10,7 +10,7 @@
"scripts": {
"build:ts": "node scripts/run-typescript-workspaces.mjs build",
"test:ts": "node scripts/run-typescript-workspaces.mjs test",
"test:pack:ts": "node scripts/test-typescript-package-closure.mjs",
"test:pack:ts": "node scripts/test-typescript-package-closure.mjs && node scripts/test-declared-dependency-ranges.mjs",
"test:eve-browser-network": "node scripts/verify-eve-browser-network.mjs"
},
"devDependencies": {

15
packages/cultcache-py/CHANGELOG.md Normal file
View file

@ -0,0 +1,15 @@
# Changelog
## Unreleased
### Breaking
- A single-file store written as `cultcache.store.v3` is read and written: a record that holds an element id (an owned list element with an `[CultElementId]` member, carried through every write) marks the store v3, and a v3 store is carried through a rewrite with its ids. A runtime from before this change refuses a v3 store by name.
- A write over a file the reader refuses (`StoreUnreadableError`, from `push`, `push_all` and `delete`) refuses and leaves the file as it is. A flush used to replace such a file. A write set cannot merge with a disk it cannot read, so the file is repaired or removed by its owner before the store writes again.
- Added `StoreUnreadableError` (a `ValueError`) for a store file this runtime cannot read, and `SchemaConflictError` (a `ValueError`) for a
write the catalog cannot describe. A store is exactly three slots; a schema is never recovered from a record's payload.
- The single-file writer derives the catalog from the records it writes: one entry per carried schema id, the registered entry over an
arrived one, written as chosen and never merged. A cache stamps a record with its registered schema's id.
- Every registration refusal is now a `SchemaConflictError`, a `ValueError`, where it was a `CultCacheError`, a `RuntimeError`: registering a document type or a schema name twice no longer raises what `except CultCacheError` or `except RuntimeError` catches.
- Two documents that own one schema id are refused at registration (`SchemaConflictError`). 2.0.1 accepted them. A document may still list another's id as compatible.
- A loaded record resolves by its schema id first: the document that owns the id, else the one document that lists it as compatible; several listers and no owner refuse the record with `SchemaConflictError`. The record's type name resolves it only when its schema id names no registered document. 2.0.1 resolved by the record's type name alone.

4
packages/cultcache-py/src/cultcache_py/__init__.py
View file

@ -17,7 +17,7 @@ from .documents import (
define_union,
)
from .leaves import SlotCodec
from .stores import JsonLinesBackingStore, SingleFileMessagePackBackingStore
from .stores import JsonLinesBackingStore, SchemaConflictError, SingleFileMessagePackBackingStore, StoreUnreadableError
__all__ = [
"BackingStore",
@ -32,8 +32,10 @@ __all__ = [
"JsonLinesBackingStore",
"MAX_PAYLOAD_DEPTH",
"MAX_SLOT_KEY",
"SchemaConflictError",
"SingleFileMessagePackBackingStore",
"SlotCodec",
"StoreUnreadableError",
"UnionArm",
"leaves",
"database_entry_field",

58
packages/cultcache-py/src/cultcache_py/cache.py
View file

@ -7,6 +7,7 @@ from typing import Any, Callable, Generic, TypeVar
from .backing_store import BackingStore, CultCacheEnvelope
from .documents import DocumentDefinition, extract_value
from .stores import SchemaConflictError
T = TypeVar("T")
F = TypeVar("F", bound=Callable[..., Any])
@ -138,11 +139,28 @@ class CultCache:
self._register_document_type(document)
def _register_document_type(self, document: DocumentDefinition[Any]) -> None:
# Every registration refusal is a SchemaConflictError naming the new document's schema id and both schema names.
entry = document.catalog_entry()
schema_name = entry.schema_name
def refuse(registered: DocumentDefinition[Any], what: str) -> SchemaConflictError:
return SchemaConflictError(
entry.schema_id,
[registered.catalog_entry().schema_name, schema_name],
"",
message=f"{what} is already registered to document type {registered.type!r} and cannot also be claimed by "
f"{document.type!r}.",
)
if document.type in self._state.documents:
raise CultCacheError(f"Document type already registered: {document.type}")
schema_name = document.catalog_entry().schema_name
raise refuse(self._state.documents[document.type], f"Document type {document.type!r}")
if schema_name in self._state.documents_by_schema_name:
raise CultCacheError(f"Document schema name already registered: {schema_name}")
raise refuse(self._state.documents_by_schema_name[schema_name], f"Schema name {schema_name!r}")
# One document owns each schema id. Others may list it as compatible beside its owner: a record under the id resolves to
# its owner, and to a lister only when nothing owns it. Neither depends on the order documents were registered in.
for registered in self._state.documents.values():
if registered.catalog_entry().schema_id == entry.schema_id:
raise refuse(registered, f"Schema id {entry.schema_id!r}")
name_extractors = dict(self._state.name_extractors)
index_extractors = {type: dict(indexes) for type, indexes in self._state.index_extractors.items()}
if document.name is not None:
@ -280,6 +298,7 @@ class CultCache:
if envelope.key != GLOBAL_KEY:
loaded_legacy_global_keys[envelope.type] = envelope.key
envelope = replace(envelope, key=GLOBAL_KEY)
envelope = self._stamped(document, envelope)
value = document.decode_payload(envelope.payload)
loaded_values.setdefault(envelope.type, {})[envelope.key] = value
loaded_envelopes.setdefault(envelope.type, {})[envelope.key] = envelope
@ -372,6 +391,7 @@ class CultCache:
def put_envelope(self, document: DocumentDefinition[T], envelope: CultCacheEnvelope) -> T:
self._assert_registered(document)
self._check_envelope(document, envelope)
envelope = self._stamped(document, envelope)
value = document.decode_payload(envelope.payload)
self._write(document, [(envelope, value)], batch=False)
return value
@ -380,12 +400,21 @@ class CultCache:
def put_envelopes(self, document: DocumentDefinition[T], envelopes: list[CultCacheEnvelope]) -> list[T]:
self._assert_registered(document)
values: list[T] = []
stamped: list[CultCacheEnvelope] = []
for envelope in envelopes:
self._check_envelope(document, envelope)
stamped.append(self._stamped(document, envelope))
values.append(document.decode_payload(envelope.payload))
self._write(document, list(zip(envelopes, values)), batch=True)
self._write(document, list(zip(stamped, values)), batch=True)
return values
@staticmethod
def _stamped(document: DocumentDefinition[Any], envelope: CultCacheEnvelope) -> CultCacheEnvelope:
"""The record under its registered schema's id, whatever id the envelope arrived under, so the catalog entry a store
writes is the one its schema id names."""
entry = document.catalog_entry()
return replace(envelope, schema_id=entry.schema_id, catalog_entry=entry)
@staticmethod
def _check_envelope(document: DocumentDefinition[Any], envelope: CultCacheEnvelope) -> None:
if envelope.type != document.type:
@ -581,6 +610,27 @@ class CultCache:
raise CultCacheError(f"Document type is not global: {document.type}")
def _resolve_document_for_envelope(self, envelope: CultCacheEnvelope) -> DocumentDefinition[Any] | None:
"""A record resolves by its schema id: the document that owns the id, else the one document that lists it as compatible.
Several listers and no owner name no single document, and the record is refused. The name its catalog carries is
metadata; it names a document only when the id names none (a store from a runtime whose schema ids this one cannot
know)."""
if envelope.schema_id:
entries = [(document, document.catalog_entry()) for document in self._state.documents.values()]
for document, entry in entries:
if entry.schema_id == envelope.schema_id:
return document
listers = [document for document, entry in entries if envelope.schema_id in entry.compatible_schema_ids]
if len(listers) > 1:
raise SchemaConflictError(
envelope.schema_id,
sorted(document.catalog_entry().schema_name for document in listers),
envelope.key,
message=f"Schema id {envelope.schema_id!r} of record {envelope.key!r} is owned by no registered document and "
f"declared compatible by several ({', '.join(sorted(document.type for document in listers))}), so it resolves "
"to none. Register the document that owns the id, or declare it on one document.",
)
if listers:
return listers[0]
document = self._state.documents.get(envelope.type)
if document is not None:
return document

217
packages/cultcache-py/src/cultcache_py/stores.py
View file

@ -14,11 +14,42 @@ from .backing_store import (
)
STORE_FORMAT_VERSION = "cultcache.store.v1"
# A store that can hold element ids. A reader older than element ids refuses this header: it would skip the id slots of a
# nested element and rewrite the element without them. This runtime writes it back only for a store it read under it; it
# does not decide when a document carries ids.
STORE_FORMAT_ELEMENT_IDS = "cultcache.store.v3"
_READABLE_STORE_FORMATS = (STORE_FORMAT_VERSION, STORE_FORMAT_ELEMENT_IDS)
_STORE_FORMAT_PREFIX = "cultcache.store."
_PERSISTED_RECORD_SLOTS = 4
_Contents = TypeVar("_Contents")
class StoreUnreadableError(ValueError):
"""A store file this runtime cannot read: not exactly one complete store (truncated, bytes after it, a missing or extra
slot), a header or record it does not know, or a body it cannot decode. Open, push, delete and push_all refuse a file
with this error, and a refused file is left as it was. The cause is `__cause__`."""
def __init__(self, path: Path, cause: BaseException) -> None:
super().__init__(f"CultCache store {path} is not readable: {cause}")
self.path = path
class SchemaConflictError(ValueError):
"""A write the catalog cannot describe: records of different types under one schema id, two arrived schemas that share an id
and disagree on the schema name, or a record no chosen entry publishes. Nothing is written. It names the id, the names
involved (`schema_names`) and a record key (`record_key`). A cache raises it too, with an empty record key, when a
document registers a schema id another registered document already carries."""
def __init__(self, schema_id: str, schema_names: list[str], record_key: str, *, message: str | None = None) -> None:
super().__init__(
message
or f"Schema id {schema_id!r} cannot describe record {record_key!r}: {', '.join(repr(name) for name in schema_names)}"
)
self.schema_id = schema_id
self.schema_names = schema_names
self.record_key = record_key
class JsonLinesBackingStore:
def __init__(self, path: str | os.PathLike[str]) -> None:
self.path = Path(path)
@ -86,43 +117,45 @@ class SingleFileMessagePackBackingStore:
def __init__(self, path: str | os.PathLike[str]) -> None:
self.path = Path(path)
self._lock = threading.RLock()
# The header this store read; a rewrite keeps it, so a store that holds element ids never sheds its marker.
self._format = STORE_FORMAT_VERSION
def pull_all(self) -> list[CultCacheEnvelope]:
msgpack = self._msgpack()
with self._lock:
# An empty file is not a store (no writer leaves one), so it is decoded and refused.
# The disk decides the header: a file that is gone is not marked.
self._format = STORE_FORMAT_VERSION
data = _read_store(self.path, Path.read_bytes)
if data is None:
return []
decoded = msgpack.unpackb(data, raw=False)
snapshot = _decode_v1_snapshot(decoded)
if snapshot is not None:
return snapshot
return _decode_legacy_envelopes(decoded, msgpack)
try:
decoded = msgpack.unpackb(data, raw=False)
snapshot = _decode_snapshot(decoded)
if snapshot is not None:
self._format, envelopes = snapshot
return envelopes
return _decode_legacy_envelopes(decoded, msgpack)
except Exception as exc:
raise StoreUnreadableError(self.path, exc) from exc
def push(self, envelope: CultCacheEnvelope) -> None:
with self._lock:
existing = {(item.type, item.key): item for item in self.pull_all()}
existing[(envelope.type, envelope.key)] = envelope
self._replace_all(list(existing.values()))
self._replace_all([envelope], self.pull_all())
def push_all(self, envelopes: list[CultCacheEnvelope]) -> None:
with self._lock:
existing = {(item.type, item.key): item for item in self.pull_all()}
for envelope in envelopes:
existing[(envelope.type, envelope.key)] = envelope
self._replace_all(list(existing.values()))
self._replace_all(list(envelopes), self.pull_all())
def delete(self, type: str, key: str) -> None:
with self._lock:
existing = [item for item in self.pull_all() if not (item.type == type and item.key == key)]
self._replace_all(existing)
self._replace_all([], [item for item in self.pull_all() if not (item.type == type and item.key == key)])
def _replace_all(self, envelopes: list[CultCacheEnvelope]) -> None:
def _replace_all(self, supplied: list[CultCacheEnvelope], arrived: list[CultCacheEnvelope]) -> None:
msgpack = self._msgpack()
self.path.parent.mkdir(parents=True, exist_ok=True)
temp = self.path.with_suffix(self.path.suffix + ".tmp")
temp.write_bytes(msgpack.packb(_encode_v1_snapshot(envelopes), use_bin_type=True))
temp.write_bytes(msgpack.packb(_encode_snapshot(supplied, arrived, self._format), use_bin_type=True))
temp.replace(self.path)
@staticmethod
@ -137,22 +170,96 @@ class SingleFileMessagePackBackingStore:
return msgpack
def _encode_v1_snapshot(envelopes: list[CultCacheEnvelope]) -> list[Any]:
catalog_by_schema_id: dict[str, CultCacheSchemaCatalogEntry] = {}
for envelope in envelopes:
schema_id = _schema_id_for(envelope)
if schema_id not in catalog_by_schema_id:
catalog_by_schema_id[schema_id] = envelope.catalog_entry or _default_catalog_entry(envelope)
def canonical(candidates: Any) -> list[CultCacheSchemaCatalogEntry]:
"""Entries that tie, in one fixed order, so the catalog does not depend on the order they arrive in."""
return sorted(candidates, key=lambda entry: (entry.schema_name, entry.content_hash, ",".join(entry.compatible_schema_ids)))
def _derive_catalog(records: list[tuple[CultCacheEnvelope, bool]]) -> list[CultCacheSchemaCatalogEntry]:
"""The catalog a write leaves, derived from the records being written and nothing else. For each schema id a record carries,
one entry that publishes it: the entry that owns the id (a registered one, else one that arrived with the records), else one
that lists it as compatible (registered first). Entries are written as chosen, never merged; a registered entry wins over an
arrived one with the same own id. Records of different types under one id, two arrived entries with the same own id and
different schema names, or a record no chosen entry publishes, refuse the write. `records` are (envelope, registered): an
envelope the caller supplied is registered, one read back from the file is arrived."""
entries: list[tuple[CultCacheSchemaCatalogEntry, bool]] = []
by_id: dict[str, list[CultCacheEnvelope]] = {}
tiers: dict[str, list[tuple[CultCacheEnvelope, bool, CultCacheSchemaCatalogEntry]]] = {}
for envelope, registered in records:
schema_id = _schema_id_for(envelope)
supplied = envelope.catalog_entry
publishes = supplied is not None and (supplied.schema_id == schema_id or schema_id in supplied.compatible_schema_ids)
entry = supplied if publishes else _default_catalog_entry(envelope)
entries.append((entry, registered))
by_id.setdefault(schema_id, []).append(envelope)
tiers.setdefault(schema_id, []).append((envelope, registered, entry))
chosen: dict[str, tuple[CultCacheSchemaCatalogEntry, bool]] = {}
for schema_id in sorted(by_id):
carrying = by_id[schema_id]
record_key = min(envelope.key for envelope in carrying)
# Records of one tier under one id are one type. Across tiers the id keeps the type its records resolve to, so a write may
# not retype records that are already there: unless the entries of both tiers own the id, which is a rename of the schema
# under a stable id, and the registered descriptor wins. A read-back record's type is its schema's name, so tiers compare
# the names of the entries that publish the id.
for tier in (True, False):
types = sorted({envelope.type for envelope, registered, _ in tiers[schema_id] if registered == tier})
if len(types) > 1:
raise SchemaConflictError(schema_id, types, record_key)
registered_names = sorted({entry.schema_name for _, registered, entry in tiers[schema_id] if registered})
arrived_names = sorted({entry.schema_name for _, registered, entry in tiers[schema_id] if not registered})
owned_by = lambda tier: any(entry.schema_id == schema_id for _, registered, entry in tiers[schema_id] if registered == tier)
if registered_names and arrived_names and registered_names[0] != arrived_names[0] and not (owned_by(True) and owned_by(False)):
raise SchemaConflictError(schema_id, sorted([registered_names[0], arrived_names[0]]), record_key)
own_registered = next(iter(canonical(entry for entry, registered in entries if registered and entry.schema_id == schema_id)), None)
own_arrived = canonical(entry for entry, registered in entries if not registered and entry.schema_id == schema_id)
own_registered_all = canonical(entry for entry, registered in entries if registered and entry.schema_id == schema_id)
# Entries of one tier that share an own id must be one schema; a registered entry settles a disagreement among arrived ones.
names = sorted({entry.schema_name for entry in (own_registered_all or own_arrived)})
if len(names) > 1:
raise SchemaConflictError(schema_id, names, record_key)
if own_registered is not None:
pick, registered_pick = own_registered, True
elif own_arrived:
pick, registered_pick = own_arrived[0], False
else:
listed_registered = next(iter(canonical(entry for entry, registered in entries if registered and schema_id in entry.compatible_schema_ids)), None)
listed_arrived = next(iter(canonical(entry for entry, registered in entries if not registered and schema_id in entry.compatible_schema_ids)), None)
pick = listed_registered if listed_registered is not None else listed_arrived
registered_pick = listed_registered is not None
if pick is None:
continue
prior = chosen.get(pick.schema_id)
if prior is not None and prior[1] and not registered_pick:
continue
chosen[pick.schema_id] = (pick, registered_pick)
for schema_id in sorted(by_id):
if not any(entry.schema_id == schema_id or schema_id in entry.compatible_schema_ids for entry, _ in chosen.values()):
raise SchemaConflictError(
schema_id, sorted(entry.schema_name for entry, _ in chosen.values()), min(envelope.key for envelope in by_id[schema_id])
)
return [entry for entry, _ in chosen.values()]
def _encode_snapshot(
supplied: list[CultCacheEnvelope], arrived: list[CultCacheEnvelope], format_version: str
) -> list[Any]:
replaced = {(envelope.type, envelope.key) for envelope in supplied}
kept = [envelope for envelope in arrived if (envelope.type, envelope.key) not in replaced]
envelopes = [*kept, *supplied]
catalog = [
_encode_catalog_entry(entry)
for entry in sorted(catalog_by_schema_id.values(), key=lambda item: item.schema_name)
for entry in sorted(
_derive_catalog([*((envelope, False) for envelope in kept), *((envelope, True) for envelope in supplied)]),
key=lambda item: (item.schema_name, item.schema_id),
)
]
records = [
[envelope.key, _schema_id_for(envelope), envelope.stored_at, envelope.payload]
for envelope in sorted(envelopes, key=lambda item: item.key)
]
return [STORE_FORMAT_VERSION, catalog, records]
return [format_version, catalog, records]
def _read_store(path: Path, read: Callable[[Path], _Contents]) -> _Contents | None:
@ -186,25 +293,29 @@ def _describe_header(header: str) -> str:
return f"an unrecognised {_STORE_FORMAT_PREFIX}* header of {len(header.encode('utf-8'))} bytes"
def _decode_v1_snapshot(decoded: Any) -> list[CultCacheEnvelope] | None:
def _decode_snapshot(decoded: Any) -> tuple[str, list[CultCacheEnvelope]] | None:
if not isinstance(decoded, list) or not decoded:
return None
if isinstance(decoded[0], str) and decoded[0].startswith(_STORE_FORMAT_PREFIX) and decoded[0] != STORE_FORMAT_VERSION:
if isinstance(decoded[0], str) and decoded[0].startswith(_STORE_FORMAT_PREFIX) and decoded[0] not in _READABLE_STORE_FORMATS:
raise ValueError(
f"CultCache store format {_describe_header(decoded[0])} is not one this runtime reads; "
f"it reads {STORE_FORMAT_VERSION!r} only"
f"it reads {STORE_FORMAT_VERSION!r} and {STORE_FORMAT_ELEMENT_IDS!r} only"
)
if decoded[0] != STORE_FORMAT_VERSION:
if decoded[0] not in _READABLE_STORE_FORMATS:
return None
if len(decoded) < 3 or not isinstance(decoded[1], list) or not isinstance(decoded[2], list):
if len(decoded) != 3:
raise ValueError(f"CultCache store has {len(decoded)} top-level slots; this runtime reads 3 (header, schema catalog, records)")
if not isinstance(decoded[1], list) or not isinstance(decoded[2], list):
raise ValueError("CultCache v1 snapshot must contain a schema catalog and record array")
# An id names the entry that owns it; only an id no entry owns names the entry that lists it as compatible.
entries = [_decode_catalog_entry(raw_entry) for raw_entry in decoded[1]]
catalog_by_schema_id: dict[str, CultCacheSchemaCatalogEntry] = {}
for raw_entry in decoded[1]:
entry = _decode_catalog_entry(raw_entry)
catalog_by_schema_id[entry.schema_id] = entry
for entry in entries:
catalog_by_schema_id.setdefault(entry.schema_id, entry)
for entry in entries:
for compatible_schema_id in entry.compatible_schema_ids:
catalog_by_schema_id[compatible_schema_id] = entry
catalog_by_schema_id.setdefault(compatible_schema_id, entry)
envelopes: list[CultCacheEnvelope] = []
for raw_record in decoded[2]:
@ -225,19 +336,7 @@ def _decode_v1_snapshot(decoded: Any) -> list[CultCacheEnvelope] | None:
raise ValueError("CultCache persisted records must declare storedAt")
catalog_entry = catalog_by_schema_id.get(schema_id)
if catalog_entry is None:
schema_version = _infer_schema_version_from_payload(_normalize_payload(payload))
schema_name = _infer_schema_name(schema_version) if schema_version is not None else None
if schema_name is None:
raise ValueError(f'CultCache persisted record "{key}" references missing schema id "{schema_id}"')
catalog_entry = CultCacheSchemaCatalogEntry(
schema_id=schema_id,
schema_name=schema_name,
schema_version=schema_version,
content_hash=schema_id,
canonical_schema_json="",
compatible_schema_ids=(),
members=(),
)
raise ValueError(f"CultCache persisted record '{key}' references schema id '{schema_id}', which the store's catalog does not publish")
normalized_payload = _normalize_payload(payload)
envelopes.append(
CultCacheEnvelope(
@ -249,7 +348,7 @@ def _decode_v1_snapshot(decoded: Any) -> list[CultCacheEnvelope] | None:
catalog_entry=catalog_entry,
)
)
return envelopes
return decoded[0], envelopes
def _decode_legacy_envelopes(decoded: Any, msgpack: Any) -> list[CultCacheEnvelope]:
@ -376,27 +475,3 @@ def _normalize_payload(payload: Any) -> bytes:
if isinstance(payload, list) and all(isinstance(item, int) and 0 <= item <= 255 for item in payload):
return bytes(payload)
raise ValueError("CultCache record payload must be binary MessagePack bytes")
def _infer_schema_version_from_payload(payload: bytes) -> str | None:
try:
msgpack = SingleFileMessagePackBackingStore._msgpack()
decoded = msgpack.unpackb(payload, raw=False)
except Exception:
return None
if isinstance(decoded, list) and decoded and isinstance(decoded[0], str):
return decoded[0]
if isinstance(decoded, dict):
value = decoded.get("schemaVersion", decoded.get("schema_version"))
return value if isinstance(value, str) else None
return None
def _infer_schema_name(schema_version: str) -> str | None:
marker = schema_version.rfind(".v")
if marker <= 0:
return None
version = schema_version[marker + 2:]
if not version or not version.isdigit():
return None
return schema_version[:marker]

633
packages/cultcache-py/tests/test_cultcache.py
View file

@ -12,6 +12,7 @@ from cultcache_py import (
CultCache,
JsonLinesBackingStore,
SingleFileMessagePackBackingStore,
StoreUnreadableError,
define_database_entry_type,
define_document_type,
)
@ -186,7 +187,7 @@ class CultCacheTests(unittest.TestCase):
self.assertEqual(raw[2][0][0], "app")
self.assertEqual(raw[2][0][1], "settings")
def test_messagepack_store_recovers_schema_stamped_record_missing_catalog_entry(self) -> None:
def test_messagepack_store_refuses_a_record_whose_schema_the_catalog_does_not_publish(self) -> None:
try:
import msgpack # type: ignore
except ModuleNotFoundError:
@ -227,11 +228,490 @@ class CultCacheTests(unittest.TestCase):
.build()
)
cache.pull_all_backing_stores()
self.assertEqual(cache.get_required(document, "record-1")["value"], "still readable")
envelope = cache.get_required_envelope(document, "record-1")
self.assertEqual(envelope.type, "runtime-policy")
self.assertEqual(envelope.schema_id, "sha256:stale-schema-id-from-cold-record")
# The payload opens with a schema version, and the catalog is the store's own description: no recovery.
with self.assertRaises(StoreUnreadableError):
cache.pull_all_backing_stores()
def _foreign_id_document(self):
return define_database_entry_type(
"tests.foreign-id",
[("name", 0)],
schema_id="tests.foreign-id.current",
schema_name="tests.foreign-id",
schema_version="tests.foreign_id.v1",
compatible_schema_ids=["tests.foreign-id.current", "tests.foreign-id.older"],
)
def _open_foreign_id_store(self, document, path: Path) -> CultCache:
cache = CultCache.builder().register_document_type(document).add_generic_store(
SingleFileMessagePackBackingStore(path)
).build()
cache.pull_all_backing_stores()
return cache
# A record put under an id its schema only lists as compatible is stamped with the registered id, and the store the write
# leaves opens and takes the next write.
def test_put_envelope_under_a_compatible_schema_id_writes_a_store_that_reopens(self) -> None:
import msgpack # type: ignore
from cultcache_py import CultCacheSchemaCatalogEntry
document = self._foreign_id_document()
with tempfile.TemporaryDirectory() as tmp:
path = Path(tmp) / "store.msgpack"
cache = self._open_foreign_id_store(document, path)
payload = document.encode_payload({"name": "foreign"})
foreign_entry = CultCacheSchemaCatalogEntry(
schema_id="tests.foreign-id.older",
schema_name="tests.foreign-id",
schema_version="tests.foreign_id.v1",
content_hash="tests.foreign-id.older",
canonical_schema_json="",
compatible_schema_ids=("tests.foreign-id.older",),
members=(),
)
cache.put_envelope(
document,
CultCacheEnvelope(
key="foreign",
type="tests.foreign-id",
payload=payload,
stored_at="2026-09-30T00:00:00Z",
schema_id="tests.foreign-id.older",
catalog_entry=foreign_entry,
),
)
stored = msgpack.unpackb(path.read_bytes(), raw=False)
self.assertEqual([entry[0] for entry in stored[1]], ["tests.foreign-id.current"])
self.assertEqual(stored[2][0][1], "tests.foreign-id.current")
reopened = self._open_foreign_id_store(document, path)
self.assertEqual(reopened.get_required(document, "foreign")["name"], "foreign")
reopened.put(document, "next", {"name": "next"})
self.assertEqual(self._open_foreign_id_store(document, path).get_required(document, "next")["name"], "next")
def test_the_store_publishes_a_records_schema_id_even_when_the_envelopes_catalog_entry_does_not(self) -> None:
import msgpack # type: ignore
from cultcache_py import CultCacheSchemaCatalogEntry
with tempfile.TemporaryDirectory() as tmp:
path = Path(tmp) / "store.msgpack"
store = SingleFileMessagePackBackingStore(path)
store.push(CultCacheEnvelope(
key="k",
type="tests.store-publishes",
payload=msgpack.packb({"name": "k"}, use_bin_type=True),
stored_at="2026-09-30T00:00:00Z",
schema_id="tests.store-publishes.record",
catalog_entry=CultCacheSchemaCatalogEntry(
schema_id="tests.store-publishes.other",
schema_name="tests.store-publishes",
schema_version="tests.store_publishes.v1",
content_hash="tests.store-publishes.other",
canonical_schema_json="",
compatible_schema_ids=("tests.store-publishes.other",),
members=(),
),
))
stored = msgpack.unpackb(path.read_bytes(), raw=False)
self.assertEqual(stored[2][0][1], "tests.store-publishes.record")
self.assertEqual([entry[0] for entry in stored[1]], ["tests.store-publishes.record"])
self.assertEqual(len(SingleFileMessagePackBackingStore(path).pull_all()), 1)
# The catalog a write leaves is derived from its records: one entry per carried id, chosen and never merged, the same whatever
# order the records arrive in. A registered (supplied) entry wins over an arrived (read back) entry with the same own id.
@staticmethod
def _writer_entry(schema_id: str, name: str, content_hash: str, compatible: tuple[str, ...]):
from cultcache_py import CultCacheSchemaCatalogEntry
return CultCacheSchemaCatalogEntry(
schema_id=schema_id, schema_name=name, schema_version=name + ".v1", content_hash=content_hash,
canonical_schema_json="", compatible_schema_ids=compatible, members=(),
)
def _writer_record(self, key: str, type: str, schema_id: str, catalog_entry=None) -> CultCacheEnvelope:
import msgpack # type: ignore
return CultCacheEnvelope(
key=key, type=type, payload=msgpack.packb({"key": key}, use_bin_type=True),
stored_at="2026-09-30T00:00:00Z", schema_id=schema_id, catalog_entry=catalog_entry,
)
@staticmethod
def _raw_entry(entry) -> list:
return [entry.schema_id, entry.schema_name, entry.schema_version, entry.content_hash, "", list(entry.compatible_schema_ids), []]
def test_a_registered_entry_is_written_as_it_is_over_the_arrived_entries_with_the_same_own_id_in_either_order(self) -> None:
import msgpack # type: ignore
stale1 = self._writer_entry("tests.x", "tests.n", "a-stale-1", ("tests.x", "old"))
stale2 = self._writer_entry("tests.x", "tests.n", "a-stale-2", ("tests.x",))
registered = self._writer_record("a", "tests.n", "tests.x", self._writer_entry("tests.x", "tests.n", "fresh", ("tests.x",)))
for entries in ([stale1, stale2], [stale2, stale1]):
with tempfile.TemporaryDirectory() as tmp:
path = Path(tmp) / "store.msgpack"
path.write_bytes(msgpack.packb([
"cultcache.store.v1", [self._raw_entry(entry) for entry in entries],
[["b", "tests.x", "2026-09-30T00:00:00Z", msgpack.packb({"b": 1}, use_bin_type=True)]],
], use_bin_type=True))
SingleFileMessagePackBackingStore(path).push(registered)
catalog = msgpack.unpackb(path.read_bytes(), raw=False)[1]
self.assertEqual(len(catalog), 1)
self.assertEqual(catalog[0][3], "fresh")
self.assertEqual(catalog[0][5], ["tests.x"], "written as chosen, no union with the arrived entries' ids")
def test_a_rename_with_a_stable_schema_id_a_whole_view_write_names_the_registered_schema(self) -> None:
import msgpack # type: ignore
with tempfile.TemporaryDirectory() as tmp:
path = Path(tmp) / "store.msgpack"
path.write_bytes(msgpack.packb([
"cultcache.store.v1", [self._raw_entry(self._writer_entry("tests.x", "tests.old", "stale", ("tests.x",)))],
[["a", "tests.x", "2026-09-30T00:00:00Z", msgpack.packb({"a": 1}, use_bin_type=True)]],
], use_bin_type=True))
SingleFileMessagePackBackingStore(path).push_all(
[self._writer_record("a", "tests.old", "tests.x", self._writer_entry("tests.x", "tests.new", "fresh", ("tests.x",)))]
)
catalog = msgpack.unpackb(path.read_bytes(), raw=False)[1]
self.assertEqual([(entry[1], entry[3]) for entry in catalog], [("tests.new", "fresh")])
def test_two_entries_of_one_tier_that_share_an_own_id_and_disagree_on_the_schema_name_refuse_the_write_typed(self) -> None:
from cultcache_py import SchemaConflictError
first = self._writer_record("b", "tests.first", "tests.x", self._writer_entry("tests.x", "tests.first", "h1", ("tests.x",)))
second = self._writer_record("a", "tests.first", "tests.x", self._writer_entry("tests.x", "tests.second", "h2", ("tests.x",)))
for order in ([first, second], [second, first]):
with tempfile.TemporaryDirectory() as tmp:
path = Path(tmp) / "store.msgpack"
with self.assertRaises(SchemaConflictError) as refused:
SingleFileMessagePackBackingStore(path).push_all(order)
self.assertEqual(refused.exception.schema_id, "tests.x")
self.assertEqual(refused.exception.record_key, "a")
self.assertEqual(refused.exception.schema_names, ["tests.first", "tests.second"])
self.assertFalse(path.exists())
def test_records_of_different_types_under_one_schema_id_refuse_the_write_typed_and_no_record_changes_type(self) -> None:
from cultcache_py import SchemaConflictError
a = self._writer_record("a", "tests.n", "tests.y", self._writer_entry("tests.x", "tests.n", "h1", ("tests.x", "tests.y")))
b = self._writer_record("b", "tests.m", "tests.y", self._writer_entry("tests.y", "tests.m", "h2", ("tests.y",)))
for order in ([a, b], [b, a]):
with tempfile.TemporaryDirectory() as tmp:
with self.assertRaises(SchemaConflictError) as refused:
SingleFileMessagePackBackingStore(Path(tmp) / "store.msgpack").push_all(order)
self.assertEqual(refused.exception.schema_id, "tests.y")
self.assertEqual(refused.exception.schema_names, ["tests.m", "tests.n"])
def test_an_entry_that_owns_an_id_is_chosen_over_one_that_lists_it(self) -> None:
import msgpack # type: ignore
owner = self._writer_record("a", "tests.n", "tests.y", self._writer_entry("tests.y", "tests.n", "h1", ("tests.y",)))
lister = self._writer_record("b", "tests.n", "tests.x", self._writer_entry("tests.x", "tests.n", "h2", ("tests.x", "tests.y")))
for order in ([lister, owner], [owner, lister]):
with tempfile.TemporaryDirectory() as tmp:
path = Path(tmp) / "store.msgpack"
SingleFileMessagePackBackingStore(path).push_all(order)
catalog = msgpack.unpackb(path.read_bytes(), raw=False)[1]
self.assertEqual(sorted((entry[0], entry[5]) for entry in catalog), [("tests.x", ["tests.x", "tests.y"]), ("tests.y", ["tests.y"])])
with tempfile.TemporaryDirectory() as tmp:
path = Path(tmp) / "store.msgpack"
SingleFileMessagePackBackingStore(path).push_all([owner])
self.assertEqual([entry[0] for entry in msgpack.unpackb(path.read_bytes(), raw=False)[1]], ["tests.y"])
def test_entries_of_one_tier_that_tie_are_taken_in_one_fixed_order_and_a_registered_lister_beats_an_arrived_one(self) -> None:
import msgpack # type: ignore
# The content hash orders them against the order their compatible ids would give: h1 sorts first, its list sorts last.
one = self._writer_record("a", "tests.n", "tests.x", self._writer_entry("tests.x", "tests.n", "h2", ("tests.x",)))
two = self._writer_record("b", "tests.n", "tests.x", self._writer_entry("tests.x", "tests.n", "h1", ("tests.x", "tests.y")))
for order in ([one, two], [two, one]):
with tempfile.TemporaryDirectory() as tmp:
path = Path(tmp) / "store.msgpack"
SingleFileMessagePackBackingStore(path).push_all(order)
self.assertEqual([entry[3] for entry in msgpack.unpackb(path.read_bytes(), raw=False)[1]], ["h1"])
# z sits under old, which the arrived entry (id x) lists; the supplied entry (id r) lists it too: the registered one publishes it.
with tempfile.TemporaryDirectory() as tmp:
path = Path(tmp) / "store.msgpack"
path.write_bytes(msgpack.packb([
"cultcache.store.v1", [self._raw_entry(self._writer_entry("tests.x", "tests.n", "stale", ("tests.x", "old")))],
[["z", "old", "2026-09-30T00:00:00Z", msgpack.packb({"z": 1}, use_bin_type=True)]],
], use_bin_type=True))
SingleFileMessagePackBackingStore(path).push(
self._writer_record("a", "tests.n", "tests.r", self._writer_entry("tests.r", "tests.n", "fresh", ("tests.r", "old")))
)
self.assertEqual([entry[0] for entry in msgpack.unpackb(path.read_bytes(), raw=False)[1]], ["tests.r"])
# A registered entry that owns an id keeps it against an arrived entry that owns it too and lists the id another record sits
# under, whichever of the two ids sorts first: the arrived entry's list is not merged in, so the record under the listed id is
# published by no chosen entry.
def test_an_arrived_entry_that_owns_a_registered_id_does_not_replace_it_nor_publish_what_it_lists(self) -> None:
import msgpack # type: ignore
from cultcache_py import SchemaConflictError
for own_id, listed_id in (("tests.a", "tests.b"), ("tests.b", "tests.a")):
with tempfile.TemporaryDirectory() as tmp:
path = Path(tmp) / "store.msgpack"
path.write_bytes(msgpack.packb([
"cultcache.store.v1", [self._raw_entry(self._writer_entry(own_id, "tests.arrived", "stale", (own_id, listed_id)))],
[["k2", listed_id, "2026-09-30T00:00:00Z", msgpack.packb({"k": 2}, use_bin_type=True)]],
], use_bin_type=True))
before = path.read_bytes()
with self.assertRaises(SchemaConflictError) as refused:
SingleFileMessagePackBackingStore(path).push(
self._writer_record("k1", "tests.registered", own_id, self._writer_entry(own_id, "tests.registered", "fresh", (own_id,)))
)
self.assertEqual(refused.exception.schema_id, listed_id)
self.assertEqual(refused.exception.record_key, "k2")
self.assertEqual(path.read_bytes(), before, "the file is left as it was")
# Ids are taken in sorted order, so which entry survives, and so which id the refusal names, does not depend on the order the
# records arrive in.
def test_the_refusal_names_the_same_id_whatever_order_the_records_arrive_in(self) -> None:
from cultcache_py import SchemaConflictError
one = self._writer_record("r1", "tests.n", "tests.y", self._writer_entry("tests.x", "tests.n", "h1", ("tests.x", "tests.y")))
two = self._writer_record("r2", "tests.n", "tests.z", self._writer_entry("tests.x", "tests.n", "h2", ("tests.x", "tests.z")))
for order in ([one, two], [two, one]):
with tempfile.TemporaryDirectory() as tmp:
with self.assertRaises(SchemaConflictError) as refused:
SingleFileMessagePackBackingStore(Path(tmp) / "store.msgpack").push_all(order)
self.assertEqual(refused.exception.schema_id, "tests.y")
self.assertEqual(refused.exception.record_key, "r1")
# A write may not retype records already in the file: across tiers the id keeps the type its records resolve to, unless the
# entries of both tiers own it, which is a rename under a stable id and the registered descriptor wins.
def test_a_write_that_would_change_the_type_an_existing_record_resolves_to_refuses_typed_and_leaves_the_file(self) -> None:
import msgpack # type: ignore
from cultcache_py import SchemaConflictError
cases = [
("registered entry lists the id, arrived entry owns it",
self._writer_entry("tests.x", "tests.arrived", "h1", ("tests.x",)),
self._writer_record("a", "tests.registered", "tests.x", self._writer_entry("tests.r", "tests.registered", "h2", ("tests.r", "tests.x")))),
("registered entry owns the id, arrived entry lists it",
self._writer_entry("tests.r", "tests.arrived", "h1", ("tests.r", "tests.x")),
self._writer_record("a", "tests.registered", "tests.x", self._writer_entry("tests.x", "tests.registered", "h2", ("tests.x",)))),
]
payload = msgpack.packb({"z": 1}, use_bin_type=True)
with tempfile.TemporaryDirectory() as tmp:
for name, catalog, registered in cases:
path = Path(tmp) / (name.split(",")[0].replace(" ", "-") + ".msgpack")
path.write_bytes(msgpack.packb(
["cultcache.store.v1", [self._raw_entry(catalog)], [["z", "tests.x", "2026-09-30T00:00:00Z", payload]]], use_bin_type=True))
before = path.read_bytes()
with self.assertRaises(SchemaConflictError, msg=name) as refused:
SingleFileMessagePackBackingStore(path).push(registered)
self.assertEqual(refused.exception.schema_id, "tests.x", name)
self.assertEqual(sorted(refused.exception.schema_names), ["tests.arrived", "tests.registered"], name)
self.assertEqual(path.read_bytes(), before, name)
# The same registered record renames the schema when both tiers own the id: it writes, and the reader finds every record.
path = Path(tmp) / "rename.msgpack"
path.write_bytes(msgpack.packb([
"cultcache.store.v1", [self._raw_entry(self._writer_entry("tests.x", "tests.arrived", "h1", ("tests.x",)))],
[["z", "tests.x", "2026-09-30T00:00:00Z", payload]],
], use_bin_type=True))
SingleFileMessagePackBackingStore(path).push(
self._writer_record("a", "tests.registered", "tests.x", self._writer_entry("tests.x", "tests.registered", "h2", ("tests.x",)))
)
self.assertEqual([(entry[0], entry[1]) for entry in msgpack.unpackb(path.read_bytes(), raw=False)[1]], [("tests.x", "tests.registered")])
self.assertEqual(len(SingleFileMessagePackBackingStore(path).pull_all()), 2)
def test_a_record_no_chosen_entry_publishes_refuses_the_write_typed_and_leaves_the_file(self) -> None:
import msgpack # type: ignore
from cultcache_py import SchemaConflictError
with tempfile.TemporaryDirectory() as tmp:
path = Path(tmp) / "store.msgpack"
path.write_bytes(msgpack.packb([
"cultcache.store.v1", [self._raw_entry(self._writer_entry("tests.x", "tests.legacy", "stale", ("tests.x", "old")))],
[["z", "old", "2026-09-30T00:00:00Z", msgpack.packb({"z": 1}, use_bin_type=True)]],
], use_bin_type=True))
before = path.read_bytes()
with self.assertRaises(SchemaConflictError) as refused:
SingleFileMessagePackBackingStore(path).push(
self._writer_record("a", "tests.legacy", "tests.x", self._writer_entry("tests.x", "tests.legacy", "fresh", ("tests.x",)))
)
self.assertEqual(refused.exception.schema_id, "old")
self.assertEqual(refused.exception.record_key, "z")
self.assertEqual(path.read_bytes(), before)
def test_an_id_an_entry_owns_names_that_entry_not_one_that_lists_it_as_compatible(self) -> None:
for vector in ("own-id-over-compatible-v3.bin", "compatible-before-owner-v3.bin"):
store = SingleFileMessagePackBackingStore(self._C2A_VECTORS / "readability" / vector)
self.assertEqual([envelope.type for envelope in store.pull_all()], ["vectors.item", "vectors.item"], vector)
def test_a_record_loaded_under_a_compatible_schema_id_is_written_back_under_the_id_its_catalog_entry_carries(self) -> None:
import msgpack # type: ignore
document = self._foreign_id_document()
with tempfile.TemporaryDirectory() as tmp:
path = Path(tmp) / "store.msgpack"
path.write_bytes(msgpack.packb([
"cultcache.store.v1",
[["tests.foreign-id.current", "tests.foreign-id", "tests.foreign_id.v1", "tests.foreign-id.current", "",
["tests.foreign-id.current", "tests.foreign-id.older"], []]],
[["old", "tests.foreign-id.older", "2026-09-30T00:00:00Z", document.encode_payload({"name": "old"})]],
], use_bin_type=True))
cache = self._open_foreign_id_store(document, path)
self.assertEqual(cache.get_required_envelope(document, "old").schema_id, "tests.foreign-id.current")
cache.put(document, "next", {"name": "next"})
reopened = self._open_foreign_id_store(document, path)
self.assertEqual(reopened.get_required(document, "old")["name"], "old")
self.assertEqual(reopened.get_required(document, "next")["name"], "next")
# The cache restamps a record on load, so a store's writer never has to: a record loaded under an id the registered document
# only lists is held, and written back by a whole write, under the id its catalog entry carries. A document that does not list
# the id a record sits under cannot have that record published by a write that leaves it in place: the put is refused, typed.
def test_a_cache_that_does_not_list_the_id_a_loaded_record_sits_under_is_refused_a_put_and_the_file_is_left(self) -> None:
import msgpack # type: ignore
from cultcache_py import SchemaConflictError
document = define_database_entry_type(
"tests.foreign-id", [("name", 0)], schema_id="tests.foreign-id.current",
schema_name="tests.foreign-id", schema_version="tests.foreign_id.v1",
)
with tempfile.TemporaryDirectory() as tmp:
path = Path(tmp) / "store.msgpack"
path.write_bytes(msgpack.packb([
"cultcache.store.v1",
[["tests.foreign-id.current", "tests.foreign-id", "tests.foreign_id.v1", "tests.foreign-id.current", "",
["tests.foreign-id.current", "tests.foreign-id.older"], []]],
[["old", "tests.foreign-id.older", "2026-09-30T00:00:00Z", document.encode_payload({"name": "old"})]],
], use_bin_type=True))
before = path.read_bytes()
cache = self._open_foreign_id_store(document, path)
self.assertEqual(cache.get_required_envelope(document, "old").schema_id, "tests.foreign-id.current")
with self.assertRaises(SchemaConflictError) as refused:
cache.put(document, "next", {"name": "next"})
self.assertEqual(refused.exception.schema_id, "tests.foreign-id.older")
self.assertEqual(refused.exception.record_key, "old")
self.assertEqual(path.read_bytes(), before)
# A record resolves to a local document by its schema id, never by the name its catalog entry carries: a schema renamed under
# a stable id opens, and an id that names one document is not overruled by a name that names another.
def _store_of(self, path: Path, catalog: list, records: list) -> None:
import msgpack # type: ignore
path.write_bytes(msgpack.packb(["cultcache.store.v1", catalog, records], use_bin_type=True))
def test_a_schema_renamed_under_a_stable_id_opens_by_its_id(self) -> None:
document = define_database_entry_type(
"tests.old", [("name", 0)], schema_id="tests.stable", schema_name="tests.old", schema_version="tests.old.v1",
)
with tempfile.TemporaryDirectory() as tmp:
path = Path(tmp) / "store.msgpack"
self._store_of(
path,
[["tests.stable", "tests.renamed", "tests.renamed.v1", "renamed", "", ["tests.stable"], []]],
[["a", "tests.stable", "2026-09-30T00:00:00Z", document.encode_payload({"name": "a"})]],
)
cache = self._open_foreign_id_store(document, path)
self.assertEqual(cache.get_required(document, "a")["name"], "a")
def test_a_schema_id_that_names_one_document_is_not_overruled_by_a_name_that_names_another(self) -> None:
first = define_database_entry_type("tests.a", [("name", 0)], schema_id="id.a", schema_name="tests.a", schema_version="tests.a.v1")
second = define_database_entry_type(
"tests.b", [("name", 0)], schema_id="id.b", schema_name="tests.b", schema_version="tests.b.v1", compatible_schema_ids=["id.b", "id.b.older"],
)
with tempfile.TemporaryDirectory() as tmp:
for schema_id in ("id.b", "id.b.older"):
path = Path(tmp) / (schema_id + ".msgpack")
# The entry that publishes the id carries the other document's name.
self._store_of(
path,
[["id.b", "tests.a", "tests.b.v1", "h", "", ["id.b", "id.b.older"], []]],
[["k", schema_id, "2026-09-30T00:00:00Z", second.encode_payload({"name": "k"})]],
)
cache = CultCache.builder().register_document_type(first).register_document_type(second).add_generic_store(
SingleFileMessagePackBackingStore(path)
).build()
cache.pull_all_backing_stores()
self.assertEqual(cache.get_required(second, "k")["name"], "k", schema_id)
self.assertIsNone(cache.get(first, "k"), schema_id)
def _assert_second_claim_refused(self, first, second, schema_id: str) -> None:
"""A registration refusal is typed, names the id and both schema names and document types, and the cache keeps what it
had."""
from cultcache_py import SchemaConflictError
cache = CultCache()
cache.register_document_type(first)
with self.assertRaises(SchemaConflictError) as refused:
cache.register_document_type(second)
self.assertEqual(refused.exception.schema_id, schema_id)
self.assertEqual(refused.exception.schema_names, [first.catalog_entry().schema_name, second.catalog_entry().schema_name])
self.assertIn(first.type, str(refused.exception))
self.assertIn(second.type, str(refused.exception))
if second is not first:
with self.assertRaisesRegex(CultCacheError, "not registered"):
cache.put(second, "k", {"name": "k"})
def _pull(self, documents: list, catalog: list, records: list) -> CultCache:
tmp = tempfile.mkdtemp()
path = Path(tmp) / "store.msgpack"
self._store_of(path, catalog, records)
builder = CultCache.builder()
for document in documents:
builder = builder.register_document_type(document)
cache = builder.add_generic_store(SingleFileMessagePackBackingStore(path)).build()
cache.pull_all_backing_stores()
return cache
def test_a_document_listing_an_id_another_owns_registers_beside_it_in_either_order_and_the_owner_wins(self) -> None:
owner = define_database_entry_type("tests.owner", [("name", 0)], schema_id="id.owned", schema_name="tests.owner", schema_version="tests.owner.v1")
lister = define_database_entry_type(
"tests.lister", [("name", 0)], schema_id="id.lister", schema_name="tests.lister", schema_version="tests.lister.v1",
compatible_schema_ids=["id.lister", "id.owned"],
)
catalog = [
["id.owned", "tests.owner", "tests.owner.v1", "h1", "", ["id.owned"], []],
["id.lister", "tests.lister", "tests.lister.v1", "h2", "", ["id.lister", "id.owned"], []],
]
records = [
["o", "id.owned", "2026-09-30T00:00:00Z", owner.encode_payload({"name": "o"})],
["l", "id.lister", "2026-09-30T00:00:00Z", lister.encode_payload({"name": "l"})],
]
for order in ([owner, lister], [lister, owner]):
cache = self._pull(order, catalog, records)
self.assertEqual(cache.get_required(owner, "o")["name"], "o")
self.assertIsNone(cache.get(lister, "o"))
self.assertEqual(cache.get_required(lister, "l")["name"], "l")
self.assertEqual(self._pull([lister], catalog, records[:1]).get_required(lister, "o")["name"], "o")
def test_two_documents_listing_one_id_register_and_a_record_under_it_needs_an_owner(self) -> None:
from cultcache_py import SchemaConflictError
a = define_database_entry_type(
"tests.list-a", [("name", 0)], schema_id="id.a", schema_name="tests.list-a", compatible_schema_ids=["id.a", "id.shared"],
)
b = define_database_entry_type(
"tests.list-b", [("name", 0)], schema_id="id.b", schema_name="tests.list-b", compatible_schema_ids=["id.b", "id.shared"],
)
owner = define_database_entry_type("tests.shared", [("name", 0)], schema_id="id.shared", schema_name="tests.shared")
catalog = [["id.shared", "tests.shared", "tests.shared.v1", "h", "", ["id.shared"], []]]
records = [["k", "id.shared", "2026-09-30T00:00:00Z", owner.encode_payload({"name": "k"})]]
for listers in ([a, b], [b, a]):
with self.assertRaises(SchemaConflictError) as refused:
self._pull(listers, catalog, records)
self.assertEqual(refused.exception.schema_id, "id.shared")
self.assertEqual(refused.exception.record_key, "k")
self.assertEqual(refused.exception.schema_names, ["tests.list-a", "tests.list-b"])
for documents in ([*listers, owner], [owner, *listers]):
cache = self._pull(documents, catalog, records)
self.assertEqual(cache.get_required(owner, "k")["name"], "k")
def test_two_documents_owning_one_schema_id_are_refused_in_either_order(self) -> None:
# Each declares an older id and not its own, so the id they share is only ever their own.
a = define_database_entry_type("tests.own-a", [("name", 0)], schema_id="id.same", schema_name="tests.own-a", compatible_schema_ids=["id.a-older"])
b = define_database_entry_type("tests.own-b", [("name", 0)], schema_id="id.same", schema_name="tests.own-b", compatible_schema_ids=["id.b-older"])
self._assert_second_claim_refused(a, b, "id.same")
self._assert_second_claim_refused(b, a, "id.same")
def test_every_registration_refusal_is_a_schema_conflict(self) -> None:
first = define_database_entry_type("tests.named", [("name", 0)], schema_id="id.first", schema_name="tests.named")
same_name = define_database_entry_type("tests.renamed", [("name", 0)], schema_id="id.second", schema_name="tests.named")
self._assert_second_claim_refused(first, same_name, "id.second")
self._assert_second_claim_refused(first, first, "id.first")
def test_interop_cli_helpers_round_trip_v1_store(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
@ -893,6 +1373,32 @@ class CultCacheTests(unittest.TestCase):
self.assertNotIn(repr(value[0]), message)
self.assertIn(f"of {len(value[0])} bytes", message)
def test_single_file_refusals_say_exactly_what_was_found_and_what_is_read(self) -> None:
import msgpack # type: ignore
with tempfile.TemporaryDirectory() as tmp:
path = Path(tmp) / "store.cc"
reads = "it reads 'cultcache.store.v1' and 'cultcache.store.v3' only"
for header, found in (
("cultcache.store.v12", "'cultcache.store.v12'"),
("cultcache.store.SECRET", "an unrecognised cultcache.store.* header of 22 bytes"),
("cultcache.store.v", "an unrecognised cultcache.store.* header of 17 bytes"),
# Digits are ASCII only: two Arabic-Indic digits are four bytes and are not the known shape.
("cultcache.store.v١٢", "an unrecognised cultcache.store.* header of 21 bytes"),
):
path.write_bytes(msgpack.packb([header, [], []], use_bin_type=True))
with self.assertRaises(ValueError) as caught:
SingleFileMessagePackBackingStore(path).pull_all()
self.assertIn(f"CultCache store format {found} is not one this runtime reads; {reads}", str(caught.exception))
path.write_bytes(msgpack.packb(["cultcache.store.v1", [], [["k", "s", "t", b"\x90", "extra"]]], use_bin_type=True))
with self.assertRaises(ValueError) as caught:
SingleFileMessagePackBackingStore(path).pull_all()
self.assertIn(
"CultCache record 'k' (schema 's') has 5 slots, more than the 4 of a cultcache.store.v1 record, so this is not a valid store",
str(caught.exception),
)
def test_single_file_record_refusal_names_only_a_string_key(self) -> None:
import msgpack # type: ignore
@ -966,5 +1472,120 @@ class CultCacheTests(unittest.TestCase):
self.assertNotIsInstance(caught.exception, FileNotFoundError)
# The element-id marker: tests/vectors/document-variants-c2a/v3-base.msgpack is v1-base with its header replaced.
_C2A_VECTORS = Path(__file__).resolve().parents[3] / "tests" / "vectors" / "document-variants-c2a"
def _header_after_rewrite(self, vector: Path) -> str:
import msgpack
with tempfile.TemporaryDirectory() as tmp:
store_path = Path(tmp) / "store.msgpack"
store_path.write_bytes(vector.read_bytes())
store = SingleFileMessagePackBackingStore(store_path)
envelopes = store.pull_all()
self.assertEqual([e.key for e in envelopes], ["alpha", "beta"])
store.push(envelopes[0])
return msgpack.unpackb(store_path.read_bytes(), raw=False)[0]
def test_single_file_reads_a_v3_element_id_store_and_a_rewrite_keeps_the_marker(self) -> None:
self.assertEqual(self._header_after_rewrite(self._C2A_VECTORS / "v3-base.msgpack"), "cultcache.store.v3")
def test_single_file_rewrite_of_a_v1_store_stays_v1(self) -> None:
self.assertEqual(self._header_after_rewrite(self._VECTORS / "v1-base.msgpack"), "cultcache.store.v1")
def test_single_file_flush_after_the_marked_file_is_gone_writes_unmarked(self) -> None:
import msgpack
with tempfile.TemporaryDirectory() as tmp:
store_path = Path(tmp) / "store.msgpack"
store_path.write_bytes((self._C2A_VECTORS / "v3-base.msgpack").read_bytes())
store = SingleFileMessagePackBackingStore(store_path)
envelopes = store.pull_all()
store_path.unlink()
store.push_all(envelopes)
self.assertEqual(msgpack.unpackb(store_path.read_bytes(), raw=False)[0], "cultcache.store.v1")
def test_single_file_push_after_the_marked_file_is_gone_writes_unmarked(self) -> None:
import msgpack
with tempfile.TemporaryDirectory() as tmp:
store_path = Path(tmp) / "store.msgpack"
store_path.write_bytes((self._C2A_VECTORS / "v3-base.msgpack").read_bytes())
store = SingleFileMessagePackBackingStore(store_path)
envelopes = store.pull_all()
store_path.unlink()
store.push(envelopes[0])
self.assertEqual(msgpack.unpackb(store_path.read_bytes(), raw=False)[0], "cultcache.store.v1")
def test_single_file_push_all_refuses_a_store_whose_header_it_cannot_read_and_leaves_it_untouched(self) -> None:
envelopes = SingleFileMessagePackBackingStore(self._C2A_VECTORS / "v3-base.msgpack").pull_all()
vectors = self._VECTORS.parent
for vector in (
self._VECTORS / "unknown-header.msgpack",
self._VECTORS / "variant-v2.msgpack",
vectors / "document-variants-c1" / "variant-store.msgpack",
):
with tempfile.TemporaryDirectory() as tmp:
store_path = Path(tmp) / "store.msgpack"
store_path.write_bytes(vector.read_bytes())
before = store_path.read_bytes()
with self.assertRaisesRegex(StoreUnreadableError, "is not readable"):
SingleFileMessagePackBackingStore(store_path).push_all(envelopes)
self.assertEqual(store_path.read_bytes(), before, f"{vector.name} was rewritten")
# A file is replaced by a rewrite exactly when this runtime's own reader opens it: one verdict per file, asked by
# pull_all, push_all, push and delete alike. The bytes and every runtime's verdict:
# tests/vectors/document-variants-c2a/readability.
def test_single_file_replaces_a_file_exactly_when_it_reads(self) -> None:
import msgpack # type: ignore
root = self._C2A_VECTORS / "readability"
rows = [
line.split()
for line in (root / "manifest.txt").read_text(encoding="utf-8").splitlines()
if line and not line.startswith("#")
]
# Every vector in the folder has a manifest row.
self.assertEqual(
sorted(path.name for path in root.glob("*.bin")),
sorted(vector for vector, *_ in rows if not vector.startswith("..")),
)
envelopes = SingleFileMessagePackBackingStore(self._C2A_VECTORS / "v3-base.msgpack").pull_all()
for vector, _, _, _, python in rows:
content = (root / vector).read_bytes()
for operation in ("pull_all", "push_all", "push", "delete"):
with tempfile.TemporaryDirectory() as tmp:
store_path = Path(tmp) / "store.msgpack"
store_path.write_bytes(content)
store = SingleFileMessagePackBackingStore(store_path)
run = {
"pull_all": store.pull_all,
"push_all": lambda: store.push_all(envelopes),
"push": lambda: store.push(envelopes[0]),
"delete": lambda: store.delete(envelopes[0].type, envelopes[0].key),
}[operation]
if python == "reads":
run()
if operation == "pull_all":
continue
# The replaced file is a store: it carries the header its old content decides (a v3 store keeps its
# marker, all else is v1), and it holds what the operation wrote.
expected = "cultcache.store.v3" if "v3" in vector else "cultcache.store.v1"
self.assertEqual(msgpack.unpackb(store_path.read_bytes(), raw=False)[0], expected, f"{vector} {operation}")
keys = [envelope.key for envelope in SingleFileMessagePackBackingStore(store_path).pull_all()]
if operation == "push_all":
self.assertEqual(sorted(set(keys) & {e.key for e in envelopes}), sorted(e.key for e in envelopes), f"{vector} {operation}")
elif operation == "push":
self.assertIn(envelopes[0].key, keys, f"{vector} {operation}")
else:
self.assertNotIn(envelopes[0].key, keys, f"{vector} {operation}")
else:
with self.assertRaises(StoreUnreadableError, msg=f"{vector} {operation}") as refused:
run()
self.assertIsNotNone(refused.exception.__cause__, f"{vector} {operation}")
self.assertEqual(store_path.read_bytes(), content, f"{vector} {operation} rewrote a file it cannot read")
if __name__ == "__main__":
unittest.main()

3
packages/cultcache-py/tests/test_typed_leaves.py
View file

@ -133,6 +133,7 @@ def _inner_record() -> t.SlotCodec:
[
database_entry_field("n", 0, default="", codec=t.string),
database_entry_field("w", 1, default=0.0, codec=t.float32),
database_entry_field("id", 2, default="", codec=t.string),
],
type_name=REFERENCE + "LeafInner",
)
@ -662,7 +663,7 @@ def _python_authored_leaf_record() -> dict:
return {
"id": "py", "flag": False, "blob": bytearray([1, 255]), "i": -(2**31), "l": 2**63 - 1, "f": 0.1, "d": 1e-300,
"maybe_i": None, "tags": [], "names": ("\u00e9", "z"), "any": {"k": [1, True, None, 2.5, "s"], "n": -3},
"nested": None, "items": [{"n": "one", "w": 2**53 + 2**29 + 1}, {"n": "two", "w": 1e-40}],
"nested": None, "items": [{"n": "one", "w": 2**53 + 2**29 + 1, "id": "000000000021"}, {"n": "two", "w": 1e-40, "id": "000000000022"}],
"shape": UnionArm("LeafCircle", {"r": 0.1}), "floats": [0.1, 3, -0.0, 1e39], "maybe_f": None,
}

85
packages/cultcache-py/tests/test_union_parity.py
View file

@ -68,22 +68,51 @@ image = define_record_codec(
],
type_name=_GOLDEN + "Image",
)
wrap = define_record_codec("Wrap", [database_entry_field("inner", 0, codec=_node)], type_name=_GOLDEN + "Wrap")
wrap = define_record_codec(
"Wrap",
[database_entry_field("inner", 0, codec=_node), database_entry_field("id", 1, codec=t.string)],
type_name=_GOLDEN + "Wrap",
)
node = define_union(
"Node",
{0: ("Box", box), 1: ("Text", text), 2: ("Use", use), 5: ("Image", image), 6: ("Wrap", wrap)},
type_name=_GOLDEN + "Node",
)
# The golden's Outcome: the arms of a list carry an element id, so Denied, Floor and Ceiling are one-slot records.
_id_only = lambda name: define_record_codec(name, [database_entry_field("id", 0, codec=t.string)]) # noqa: E731
outcome = define_union(
"Outcome",
{
0: (
"Pending",
define_record_codec(
"Pending",
[database_entry_field("message", 0, codec=t.string), database_entry_field("id", 1, codec=t.string)],
),
),
1: (
"Completed",
define_record_codec(
"Completed",
[database_entry_field("source_version", 0, codec=t.int32), database_entry_field("id", 1, codec=t.string)],
),
),
2: ("Denied", _id_only("Denied")),
-2147483648: ("Floor", _id_only("Floor")),
2147483647: ("Ceiling", _id_only("Ceiling")),
},
type_name=_GOLDEN + "Outcome",
)
# The codec's unit arms (no value, written [key, []]), kept apart from the golden: a document list's arms carry ids.
unit_outcome = define_union(
"Outcome",
{
0: ("Pending", define_record_codec("Pending", [database_entry_field("message", 0, codec=t.string)])),
1: ("Completed", define_record_codec("Completed", [database_entry_field("source_version", 0, codec=t.int32)])),
2: ("Denied", None),
-2147483648: ("Floor", None),
2147483647: ("Ceiling", None),
},
type_name=_GOLDEN + "Outcome",
type_name=_GOLDEN + "UnitOutcome",
)
tool = define_record_codec(
"Tool",
@ -121,41 +150,41 @@ def _surface_value() -> dict:
"root": UnionArm(
"Box",
{
"id": "root",
"id": "0000000000c0",
"children": [
UnionArm("Text", {"id": "t1", "content": "hello"}),
UnionArm("Text", {"id": "0000000000c1", "content": "hello"}),
UnionArm(
"Use",
{
"id": "u1",
"id": "0000000000c2",
"composition": {
"name": "comp",
"body": UnionArm(
"Box",
{
"id": "inner",
"id": "0000000000d0",
"children": [
UnionArm("Text", {"id": "deep", "content": "x"}),
UnionArm("Image", {"id": "img", "uri": "uri://a", "alt": "alt"}),
UnionArm("Text", {"id": "0000000000d1", "content": "x"}),
UnionArm("Image", {"id": "0000000000d2", "uri": "uri://a", "alt": "alt"}),
],
},
),
},
"slots": [
UnionArm("Text", {"id": "s1", "content": "slot"}),
UnionArm("Text", {"id": "0000000000e1", "content": "slot"}),
None,
UnionArm("Box", {"id": "b2", "children": []}),
UnionArm("Box", {"id": "0000000000e2", "children": []}),
],
},
),
UnionArm("Image", {"id": "i2", "uri": "uri://b", "alt": ""}),
UnionArm("Image", {"id": "0000000000c3", "uri": "uri://b", "alt": ""}),
],
},
),
"outcomes": [
UnionArm("Pending", {"message": "wait"}),
UnionArm("Completed", {"source_version": 7}),
UnionArm("Denied"),
UnionArm("Pending", {"message": "wait", "id": "0000000000a1"}),
UnionArm("Completed", {"source_version": 7, "id": "0000000000a2"}),
UnionArm("Denied", {"id": "0000000000a3"}),
],
"part": UnionArm("Tool", {"name": "hammer", "uses": 12}),
"crafted": UnionArm("Part", {"name": "gear", "count": 4}),
@ -256,12 +285,14 @@ class UnionCodecRuleTests(unittest.TestCase):
)
def test_unit_arm_is_written_with_empty_slots_and_reads_back(self) -> None:
self.assertEqual(outcome.encode(UnionArm("Denied")), [2, []])
self.assertEqual(outcome.decode([2, []]), UnionArm("Denied", None))
# The bytes are the C# reference's (tests/vectors/union-parity/unit-arm-denied.msgpack), not spelled here.
reference = _vector("unit-arm-denied.msgpack")
self.assertEqual(msgpack.packb(unit_outcome.encode(UnionArm("Denied"))), reference)
self.assertEqual(unit_outcome.decode(msgpack.unpackb(reference, raw=False)), UnionArm("Denied", None))
def test_unit_arm_refuses_a_value_to_carry(self) -> None:
with self.assertRaises(ValueError) as caught:
outcome.encode(UnionArm("Denied", {"message": self.CANARY}))
unit_outcome.encode(UnionArm("Denied", {"message": self.CANARY}))
self.assertEqual(str(caught.exception), "Union 'Outcome' arm 2 is a unit arm and carries no value")
def test_arm_keys_are_written_as_declared(self) -> None:
@ -333,7 +364,7 @@ class UnionCodecRuleTests(unittest.TestCase):
self.assertEqual(row.decode(encoded), {"id": "r", "cells": cells})
def test_a_unit_arm_with_extra_slots_is_read_past(self) -> None:
self.assertEqual(outcome.decode([2, ["extra"]]), UnionArm("Denied"))
self.assertEqual(unit_outcome.decode([2, ["extra"]]), UnionArm("Denied"))
def test_union_arms_are_immutable_hashable_values(self) -> None:
arm = UnionArm("Denied")
@ -478,7 +509,7 @@ CANARY = "CANARY-union-golden-4c2a"
def _wrap_chain(wraps: int, leaf: UnionArm) -> UnionArm:
value = leaf
for _ in range(wraps):
value = UnionArm("Wrap", {"inner": value})
value = UnionArm("Wrap", {"inner": value, "id": "0000000000b1"})
return value
@ -521,9 +552,9 @@ class BoundsVectorTests(unittest.TestCase):
self.assertEqual(counts, {"accepted": 4, "refused": 7, "read-only": 1})
def test_the_value_at_the_limit_writes_the_vector_and_the_value_past_it_is_refused_on_write(self) -> None:
at_limit = {**_surface_value(), "root": _wrap_chain(126, UnionArm("Box", {"id": "leaf", "children": []}))}
at_limit = {**_surface_value(), "root": _wrap_chain(126, UnionArm("Box", {"id": "0000000000f0", "children": []}))}
self.assertEqual(surface_document.encode_payload(at_limit), _vector("depth-at-limit.msgpack"))
past = {**_surface_value(), "root": _wrap_chain(127, UnionArm("Text", {"id": "leaf", "content": CANARY}))}
past = {**_surface_value(), "root": _wrap_chain(127, UnionArm("Text", {"id": "0000000000f0", "content": CANARY}))}
with self.assertRaises(CultCachePayloadDepthError) as caught:
surface_document.encode_payload(past)
message = str(caught.exception)
@ -533,7 +564,7 @@ class BoundsVectorTests(unittest.TestCase):
self.assertNotIn(CANARY, message)
def test_a_cache_put_of_the_value_past_the_limit_refuses_and_stores_nothing(self) -> None:
past = {**_surface_value(), "root": _wrap_chain(127, UnionArm("Box", {"id": "leaf", "children": []}))}
past = {**_surface_value(), "root": _wrap_chain(127, UnionArm("Box", {"id": "0000000000f0", "children": []}))}
with tempfile.TemporaryDirectory() as tmp:
path = Path(tmp) / "store.cc"
cache = (
@ -709,8 +740,12 @@ class UnionDecodeBoundsTests(unittest.TestCase):
self.assertEqual(str(caught.exception), expected)
def test_the_ends_of_the_int32_key_domain_are_unit_arms_that_read_and_write(self) -> None:
self.assertEqual(outcome.decode([-(2**31), []]), UnionArm("Floor"))
self.assertEqual(outcome.encode(UnionArm("Ceiling")), [2**31 - 1, []])
floor = _vector("unit-arm-floor.msgpack")
ceiling = _vector("unit-arm-ceiling.msgpack")
self.assertEqual(unit_outcome.decode(msgpack.unpackb(floor, raw=False)), UnionArm("Floor"))
self.assertEqual(msgpack.packb(unit_outcome.encode(UnionArm("Floor"))), floor)
self.assertEqual(unit_outcome.decode(msgpack.unpackb(ceiling, raw=False)), UnionArm("Ceiling"))
self.assertEqual(msgpack.packb(unit_outcome.encode(UnionArm("Ceiling"))), ceiling)
if __name__ == "__main__":

11
packages/cultcache-rs/CHANGELOG.md
View file

@ -4,6 +4,17 @@ All notable changes to `cultcache-rs` are recorded here, following
[Keep a Changelog](https://keepachangelog.com/en/1.1.0/). Earlier releases are recorded
only in git history.
## Unreleased
### Breaking
- A single-file store written as `cultcache.store.v3` is read and written: a record that holds an element id (an owned list element with an `[CultElementId]` member, carried through every write) marks the store v3, and a v3 store is carried through a rewrite with its ids. A runtime from before this change refuses a v3 store by name.
- A write over a file the reader refuses (a zero-byte file, a truncated store, a stamp this runtime does not read) refuses with `CultCacheStoreUnreadable` and leaves the file as it is. A whole-store write used to replace such a file. This reverses 0.3.1, where `push_all` replaced a file with no readable stamp (healing it); the stamp rule now has one owner, the reader every write asks under the lock, and it accepts v1 and v3 (0.3.1 refused v3).
### Added
- `SchemaConflictError`, the typed refusal of a write the catalog cannot describe: records of different types under one schema id, or an arrived schema and a registered one that share an id and disagree on the schema name. A whole-store write (`push_all`) refuses with it, names the schema id, the names involved and a record key, writes nothing and leaves the file as it is. A caller tells it apart with `error.downcast_ref::<SchemaConflictError>()`.
## [0.3.1]
### Fixed

33
packages/cultcache-rs/examples/cultcache_interop.rs
View file

@ -26,6 +26,38 @@ struct CultCacheInteropNote {
tags: Vec<String>,
}
// An object list: each mark is a positional array [label, id], the id an ordinary member at slot 1.
#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
struct CultCacheInteropMark(String, String);
#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)]
#[cultcache(type = "cultcache.interop-deck", schema = "CultCacheInteropDeck")]
struct CultCacheInteropDeck {
#[cultcache(key = 0)]
document_id: String,
#[cultcache(key = 1, default)]
marks: Vec<CultCacheInteropMark>,
}
fn read_deck(file: &str) -> Result<()> {
let mut cache = CultCache::new();
cache.register_entry_type::<CultCacheInteropDeck>()?;
cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(file))?;
cache.pull_all_backing_stores()?;
let decks = cache.get_all::<CultCacheInteropDeck>()?;
let deck = decks
.first()
.ok_or_else(|| anyhow!("no cultcache.interop-deck records found"))?;
println!(
"{}",
serde_json::to_string(&json!({
"documentId": deck.document_id,
"ids": deck.marks.iter().map(|mark| mark.1.clone()).collect::<Vec<_>>(),
}))?
);
Ok(())
}
fn main() -> Result<()> {
let mut args = std::env::args().skip(1);
let mode = args
@ -37,6 +69,7 @@ fn main() -> Result<()> {
match mode.as_str() {
"write" => write_note(file, require_arg(&options, "runtime-id")?)?,
"read" => read_note(file)?,
"read-deck" => read_deck(file)?,
"hold-lock" => hold_lock(file, require_arg(&options, "ms")?.parse()?)?,
_ => return Err(anyhow!("unknown mode {mode}")),
}

1011
packages/cultcache-rs/src/lib.rs
View file

File diff suppressed because it is too large Load diff

95
packages/cultcache-rs/tests/union_parity.rs
View file

@ -32,6 +32,8 @@ enum Node {
struct WrapArm {
#[cultcache(key = 0)]
inner: Option<std::boxed::Box<Node>>,
#[cultcache(key = 1)]
id: String,
}
#[derive(Clone, Debug, PartialEq, CultSlots)]
@ -85,6 +87,17 @@ enum Outcome {
Pending(PendingArm),
#[cultcache(key = 1)]
Completed(CompletedArm),
#[cultcache(key = 2)]
Denied(IdArm),
#[cultcache(key = -2147483648)]
Floor(IdArm),
#[cultcache(key = 2147483647)]
Ceiling(IdArm),
}
// The codec's unit arm (no members, written [key, []]), kept apart from the golden: a document list's arms carry ids.
#[derive(Clone, Debug, PartialEq, CultUnion)]
enum UnitOutcome {
#[cultcache(key = 2)]
Denied,
#[cultcache(key = -2147483648)]
@ -97,12 +110,23 @@ enum Outcome {
struct PendingArm {
#[cultcache(key = 0)]
message: String,
#[cultcache(key = 1)]
id: String,
}
// An arm with no members of its own: just its element id.
#[derive(Clone, Debug, PartialEq, CultSlots)]
struct IdArm {
#[cultcache(key = 0)]
id: String,
}
#[derive(Clone, Debug, PartialEq, CultSlots)]
struct CompletedArm {
#[cultcache(key = 0)]
source_version: i32,
#[cultcache(key = 1)]
id: String,
}
#[derive(Clone, Debug, PartialEq, CultSlots)]
@ -178,26 +202,26 @@ fn boxed(id: &str, children: Vec<Node>) -> Node {
}
fn constructed() -> UnionSurface {
let inner = boxed("inner", vec![text("deep", "x"), image("img", "uri://a", "alt")]);
let inner = boxed("0000000000d0", vec![text("0000000000d1", "x"), image("0000000000d2", "uri://a", "alt")]);
UnionSurface {
id: "surface".into(),
version: 3,
root: Some(boxed(
"root",
"0000000000c0",
vec![
text("t1", "hello"),
text("0000000000c1", "hello"),
Node::Use(UseArm {
id: "u1".into(),
id: "0000000000c2".into(),
composition: Composition { name: "comp".into(), body: Some(std::boxed::Box::new(inner)) },
slots: vec![Some(text("s1", "slot")), None, Some(boxed("b2", vec![]))],
slots: vec![Some(text("0000000000e1", "slot")), None, Some(boxed("0000000000e2", vec![]))],
}),
image("i2", "uri://b", ""),
image("0000000000c3", "uri://b", ""),
],
)),
outcomes: vec![
Outcome::Pending(PendingArm { message: "wait".into() }),
Outcome::Completed(CompletedArm { source_version: 7 }),
Outcome::Denied,
Outcome::Pending(PendingArm { message: "wait".into(), id: "0000000000a1".into() }),
Outcome::Completed(CompletedArm { source_version: 7, id: "0000000000a2".into() }),
Outcome::Denied(id_arm("0000000000a3")),
],
part: Some(Item::Tool(UsesArm { name: "hammer".into(), uses: 12 })),
crafted: Some(CraftedItem::Part(CountArm { name: "gear".into(), count: 4 })),
@ -205,6 +229,10 @@ fn constructed() -> UnionSurface {
}
}
fn id_arm(id: &str) -> IdArm {
IdArm { id: id.into() }
}
fn vector(name: &str) -> Vec<u8> {
let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
.join("../../tests/vectors/union-parity")
@ -226,8 +254,25 @@ fn union_surface_decodes_and_reencodes() {
#[test]
fn a_unit_arm_is_a_key_and_an_empty_slots_array() {
assert_eq!(rmp_serde::to_vec(&Outcome::Denied).unwrap(), [0x92, 0x02, 0x90]);
assert_eq!(rmp_serde::from_slice::<Outcome>(&[0x92, 0x02, 0x90]).unwrap(), Outcome::Denied);
// The bytes are the C# reference's, not spelled here.
for (file, arm) in [
("unit-arm-denied.msgpack", UnitOutcome::Denied),
("unit-arm-floor.msgpack", UnitOutcome::Floor),
("unit-arm-ceiling.msgpack", UnitOutcome::Ceiling),
] {
let reference = vector(file);
assert_eq!(rmp_serde::to_vec(&arm).unwrap(), reference, "{file}");
assert_eq!(rmp_serde::from_slice::<UnitOutcome>(&reference).unwrap(), arm, "{file}");
}
}
#[test]
fn an_id_only_arm_is_a_key_and_a_one_slot_array() {
let mut bytes = vec![0x92, 0x02, 0x91, 0xac];
bytes.extend_from_slice(b"0000000000a3");
let denied = Outcome::Denied(id_arm("0000000000a3"));
assert_eq!(rmp_serde::to_vec(&denied).unwrap(), bytes);
assert_eq!(rmp_serde::from_slice::<Outcome>(&bytes).unwrap(), denied);
}
#[test]
@ -246,7 +291,7 @@ fn union_refusal_vectors() {
let key = key.expect(&file);
assert!(message.contains(&format!("unknown arm key {key} ")), "{file}: {message}");
}
for canary in ["root", "hello", "uri://", "surface"] {
for canary in ["0000000000c0", "hello", "uri://", "surface"] {
assert!(!message.contains(canary), "{file} echoed a value: {message}");
}
}
@ -287,8 +332,8 @@ fn a_union_that_is_not_a_key_and_slots_pair_is_refused_naming_the_enum() {
("a unit arm with slots", rmp_serde::to_vec(&(2u64, ("CANARY-slot",))).unwrap()),
];
for (label, bytes) in cases {
let message = rmp_serde::from_slice::<Outcome>(&bytes).unwrap_err().to_string();
assert!(message.contains("Outcome"), "{label}: {message}");
let message = rmp_serde::from_slice::<UnitOutcome>(&bytes).unwrap_err().to_string();
assert!(message.contains("UnitOutcome"), "{label}: {message}");
assert!(!message.contains("CANARY"), "{label}: {message}");
}
}
@ -679,9 +724,9 @@ fn bounds_vectors() {
let value = result.unwrap_or_else(|message| panic!("{file}: {message}"));
assert_eq!(rmp_serde::to_vec(&value).unwrap(), vector(detail), "{file}");
match file {
"arm-key-int32-min.msgpack" => assert_eq!(value.outcomes.last(), Some(&Outcome::Floor)),
"arm-key-int32-min.msgpack" => assert_eq!(value.outcomes.last(), Some(&Outcome::Floor(id_arm("0000000000a4")))),
"arm-key-int32-max.msgpack" | "arm-key-wide-coding.msgpack" => {
assert_eq!(value.outcomes.last(), Some(&Outcome::Ceiling))
assert_eq!(value.outcomes.last(), Some(&Outcome::Ceiling(id_arm("0000000000a5"))))
}
_ => {}
}
@ -715,7 +760,7 @@ fn bounds_vectors() {
fn wrap_chain(wraps: usize, leaf: Node) -> UnionSurface {
let mut node = leaf;
for _ in 0..wraps {
node = Node::Wrap(WrapArm { inner: Some(std::boxed::Box::new(node)) });
node = Node::Wrap(WrapArm { inner: Some(std::boxed::Box::new(node)), id: "0000000000b1".into() });
}
UnionSurface { root: Some(node), ..constructed() }
}
@ -724,9 +769,9 @@ fn wrap_chain(wraps: usize, leaf: Node) -> UnionSurface {
// children) ends at 2 * (wraps + 1) + 2 and a Text leaf at 2 * (wraps + 1) + 1.
#[test]
fn the_constructed_chains_write_the_vector_bytes() {
let at_limit = wrap_chain(126, boxed("leaf", vec![]));
let at_limit = wrap_chain(126, boxed("0000000000f0", vec![]));
assert_eq!(rmp_serde::to_vec(&at_limit).unwrap(), vector("depth-at-limit.msgpack"));
let past_limit = wrap_chain(127, text("leaf", CANARY));
let past_limit = wrap_chain(127, text("0000000000f0", CANARY));
assert_eq!(rmp_serde::to_vec(&past_limit).unwrap(), vector("depth-past-limit.msgpack"));
}
@ -744,14 +789,14 @@ fn write_refused_past_limit() {
cache.register_entry_type::<UnionSurface>().unwrap();
cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&path)).unwrap();
let at_limit = wrap_chain(126, boxed("leaf", vec![]));
let at_limit = wrap_chain(126, boxed("0000000000f0", vec![]));
assert_eq!(cache.put("at", &at_limit).unwrap(), at_limit);
let stored = cache.snapshot_envelopes();
assert_eq!(stored.len(), 1);
assert_eq!(stored[0].payload, vector("depth-at-limit.msgpack"));
let file_before = std::fs::read(&path).unwrap();
let error = cache.put("past", &wrap_chain(127, text("leaf", CANARY))).unwrap_err();
let error = cache.put("past", &wrap_chain(127, text("0000000000f0", CANARY))).unwrap_err();
assert_eq!(error.downcast_ref::<PayloadTooDeep>(), Some(&PayloadTooDeep { depth: 257, limit: 256 }));
assert!(!format!("{error:#}").contains(CANARY), "{error:#}");
assert_eq!(cache.get::<UnionSurface>("past").unwrap(), None);
@ -808,8 +853,8 @@ fn raw(type_name: &str, key: &str, payload: Vec<u8>) -> CultCacheEnvelope {
#[test]
fn the_raw_doors_hold_the_bound_for_arms_maps_and_arrays() {
let arms_at = rmp_serde::to_vec(&wrap_chain(126, boxed("leaf", vec![]))).unwrap();
let arms_past = rmp_serde::to_vec(&wrap_chain(127, text("leaf", CANARY))).unwrap();
let arms_at = rmp_serde::to_vec(&wrap_chain(126, boxed("0000000000f0", vec![]))).unwrap();
let arms_past = rmp_serde::to_vec(&wrap_chain(127, text("0000000000f0", CANARY))).unwrap();
let maps_at = rmp_serde::to_vec_named(&map_doc(MAX_PAYLOAD_DEPTH - 1)).unwrap();
let maps_past = rmp_serde::to_vec_named(&map_doc(MAX_PAYLOAD_DEPTH)).unwrap();
let arrays_at = vector("depth-at-limit.msgpack");
@ -857,13 +902,13 @@ fn the_union_chain_at_the_limit_decodes_on_a_1mib_thread() {
.spawn(|| {
assert_eq!(
through_cache(&vector("depth-at-limit.msgpack")).unwrap(),
wrap_chain(126, boxed("leaf", vec![]))
wrap_chain(126, boxed("0000000000f0", vec![]))
);
let refused = through_cache(&vector("depth-past-limit.msgpack")).unwrap_err();
assert!(refused.contains("the limit is 256"), "{refused}");
let mut cache = CultCache::new();
cache.register_entry_type::<UnionSurface>().unwrap();
let value = wrap_chain(126, boxed("leaf", vec![]));
let value = wrap_chain(126, boxed("0000000000f0", vec![]));
assert_eq!(cache.put("at", &value).unwrap(), value);
assert_eq!(cache.get::<UnionSurface>("at").unwrap(), Some(value));
let mut deep = vec![0x91u8; 100_000];

18
packages/cultcache-ts/CHANGELOG.md
View file

@ -3,6 +3,24 @@
All notable changes to `@gamecult/cultcache-ts` are recorded here, following
[Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
## Unreleased
### Breaking
- A single-file store written as `cultcache.store.v3` is read and written: a record that holds an element id (an owned list element with an `[CultElementId]` member, carried through every write) marks the store v3, and a v3 store is carried through a rewrite with its ids. A runtime from before this change refuses a v3 store by name.
- A single-file store refuses a zero-byte file as `StoreUnreadableError`; only a path with nothing at it is an empty store. A header this runtime does not read is refused, and the refusal echoes the header only as `cultcache.store.v<digits>`.
- A write over a file the reader refuses (`StoreUnreadableError`, from `push`, `pushAll` and `delete`) refuses and leaves the file as it is. A flush used to replace such a file. A write set cannot merge with a disk it cannot read, so the file is repaired or removed by its owner before the store writes again.
- This reverses the 0.14.2 `pushAll` fix in two ways. A full `pushAll` over a file with no readable stamp no longer replaces it (healing it): the reader refuses it, and the file is left as it is. And a `cultcache.store.v3` stamp is read, not refused. The stamp rule has one owner, `refuseUnreadFormat`, asked only by the one reader that every write consults under the lock. The reader judges the stamp from the header before it decodes the body, so a newer store is refused by name whether or not its body is whole.
- A dangling symbolic link at the store path is an I/O error with code `ENOENT`, not an empty store.
### Added
- `readSingleFileStore(path)`, the one synchronous reader of a single-file store, for consumers that must read a store file rather than decode it themselves. `pullAll`, `push`, `delete` and `pushAll` ask the same reader.
- `StoreUnreadableError` for a store file this runtime cannot read, `SchemaConflictError` for a write the catalog cannot describe, and `schemaIdentityOf` with its result type `CultCacheSchemaIdentity` (`schemaId`, `schemaName`, `schemaVersion`, `compatibleSchemaIds`), the one derivation of a definition's identity.
- `SingleFileStoreRead`, exported: the result type of `readSingleFileStore` (`absent`, `format`, `envelopes`, `repairedLegacyPayload`).
- `CultCacheInspection.format` is `StoreFormat | "legacy.envelope-array"`: it reports `cultcache.store.v3` for a v3 store.
- A soft `pushAll` writes only where nothing is at the path, leaves a store it can read byte for byte, and refuses what the reader refuses.
## [0.14.2]
### Added

94
packages/cultcache-ts/src/cult-cache-inspector.ts
View file

@ -1,12 +1,12 @@
import { decode } from "@msgpack/msgpack";
import { checkPayloadDepth } from "./payload-depth";
import { STORE_FORMAT_VERSION, isV1Snapshot, requireV1RecordSlots } from "./store-format";
import { type StoreFormat, isStoreSnapshot, refuseUnreadStoreBytes, requireStoreSlots, requireV1RecordSlots } from "./store-format";
export interface CultCacheInspection {
filePath: string;
fileSizeBytes: number;
format: "cultcache.store.v1" | "legacy.envelope-array";
format: StoreFormat | "legacy.envelope-array";
catalog: InspectedCatalogEntry[];
records: InspectedRecord[];
}
@ -47,9 +47,10 @@ export function inspectCultCacheBytes(
bytes: Uint8Array,
fileSizeBytes = bytes.length,
): CultCacheInspection {
refuseUnreadStoreBytes(bytes);
const decoded = decode(bytes);
if (isV1Snapshot(decoded)) {
if (isStoreSnapshot(decoded)) {
return inspectV1Snapshot(filePath, fileSizeBytes, decoded);
}
@ -60,7 +61,8 @@ export function inspectCultCacheBytes(
throw new Error("CultCache file is not a recognized v1 snapshot or legacy envelope array.");
}
function inspectV1Snapshot(filePath: string, fileSizeBytes: number, decoded: unknown[]): CultCacheInspection {
function inspectV1Snapshot(filePath: string, fileSizeBytes: number, decoded: [StoreFormat, ...unknown[]]): CultCacheInspection {
requireStoreSlots(decoded);
const catalogRaw = decoded[1];
const recordsRaw = decoded[2];
if (!Array.isArray(catalogRaw) || !Array.isArray(recordsRaw)) {
@ -68,36 +70,25 @@ function inspectV1Snapshot(filePath: string, fileSizeBytes: number, decoded: unk
}
const catalog = catalogRaw.map(decodeCatalogEntry);
// An id names the entry that owns it; only an id no entry owns names the entry that lists it as compatible.
const catalogBySchemaId = new Map<string, InspectedCatalogEntry>();
for (const entry of catalog) {
catalogBySchemaId.set(entry.schemaId, entry);
for (const compatibleSchemaId of entry.compatibleSchemaIds) {
catalogBySchemaId.set(compatibleSchemaId, entry);
if (!catalogBySchemaId.has(entry.schemaId)) {
catalogBySchemaId.set(entry.schemaId, entry);
}
}
const records = recordsRaw.map((record) => decodeV1Record(record, catalogBySchemaId));
for (const record of records) {
if (!catalog.some((entry) => entry.schemaId === record.schemaId)) {
const schemaVersion = inferSchemaVersionFromPayloadPreview(record.payloadPreview);
const schemaName = schemaVersion ? inferSchemaName(schemaVersion) : undefined;
if (schemaName) {
catalog.push({
schemaId: record.schemaId,
schemaName,
schemaVersion: schemaVersion!,
contentHash: record.schemaId,
canonicalSchemaJson: "",
compatibleSchemaIds: [],
members: [],
});
for (const entry of catalog) {
for (const compatibleSchemaId of entry.compatibleSchemaIds) {
if (!catalogBySchemaId.has(compatibleSchemaId)) {
catalogBySchemaId.set(compatibleSchemaId, entry);
}
}
}
const records = recordsRaw.map((record) => decodeV1Record(record, catalogBySchemaId));
return {
filePath,
fileSizeBytes,
format: STORE_FORMAT_VERSION,
format: decoded[0],
catalog,
records,
};
@ -205,64 +196,21 @@ function decodeV1Record(value: unknown, catalogBySchemaId: Map<string, Inspected
}
const payloadBytes = normalizePayloadBytes(payload);
const catalogEntry = catalogBySchemaId.get(schemaId) ?? recoverCatalogEntry(schemaId, payloadBytes);
const catalogEntry = catalogBySchemaId.get(schemaId);
if (!catalogEntry) {
throw new Error(`CultCache persisted record "${key}" references missing schema id "${schemaId}".`);
}
return {
key,
schemaId,
schemaName: catalogEntry?.schemaName ?? "<missing catalog entry>",
schemaName: catalogEntry.schemaName,
storedAt,
payloadBytes: payloadBytes.length,
...previewPayload(payloadBytes),
};
}
function recoverCatalogEntry(schemaId: string, payload: Uint8Array): InspectedCatalogEntry | undefined {
let payloadPreview: unknown;
try {
payloadPreview = decodeBoundedPayload(payload);
} catch {
return undefined;
}
const schemaVersion = inferSchemaVersionFromPayloadPreview(payloadPreview);
if (!schemaVersion) {
return undefined;
}
const schemaName = inferSchemaName(schemaVersion);
if (!schemaName) {
return undefined;
}
return {
schemaId,
schemaName,
schemaVersion,
contentHash: schemaId,
canonicalSchemaJson: "",
compatibleSchemaIds: [],
members: [],
};
}
function inferSchemaVersionFromPayloadPreview(payloadPreview: unknown): string | undefined {
if (Array.isArray(payloadPreview) && typeof payloadPreview[0] === "string") {
return payloadPreview[0];
}
if (isRecord(payloadPreview)) {
const schemaVersion = payloadPreview.schemaVersion ?? payloadPreview.schema_version;
return typeof schemaVersion === "string" ? schemaVersion : undefined;
}
return undefined;
}
function inferSchemaName(schemaVersion: string): string | undefined {
const match = /^(.*)\.v\d+$/.exec(schemaVersion);
return match?.[1];
}
function decodeLegacyEnvelope(value: unknown): InspectedRecord {
if (!isRecord(value)) {
throw new Error("CultCache legacy envelopes must be MessagePack maps.");

67
packages/cultcache-ts/src/cult-cache.ts
View file

@ -4,9 +4,11 @@ import {
documentPayloadFromBytes,
encodeDocumentPayload,
resolveDocumentMembers,
schemaIdentityOf,
} from "./document";
import { resolveDocumentFormatter } from "./document-formatter";
import type { DocumentPayload } from "./document";
import { SchemaConflictError } from "./store-format";
import type {
AnyCultCacheDocumentDefinition,
CacheBackingStore,
@ -161,7 +163,9 @@ export class CultCache {
}
readonly #definitions = new Map<string, RegisteredDefinition>();
// The definition that owns each schema id, and every definition listing an id as compatible, in registration order.
readonly #schemaIdDefinitions = new Map<string, RegisteredDefinition>();
readonly #listingDefinitions = new Map<string, RegisteredDefinition[]>();
readonly #schemaNameDefinitions = new Map<string, RegisteredDefinition>();
#hydrated = emptyHydratedState();
readonly #stores: CultCacheStoreRegistration[] = [];
@ -474,8 +478,10 @@ export class CultCache {
type: envelope.type,
payload,
storedAt: envelope.storedAt,
schemaId: envelope.schemaId ?? registered.catalogEntry.schemaId,
catalogEntry: envelope.catalogEntry ?? registered.catalogEntry,
// The record is stamped with its registered schema's id, whatever id the envelope arrived under, so the
// catalog entry the store writes is the one its schema id names.
schemaId: registered.catalogEntry.schemaId,
catalogEntry: registered.catalogEntry,
};
await this.#writeNow(registered, entry, parsed);
return parsed;
@ -535,9 +541,13 @@ export class CultCache {
}
#registerNow(definition: AnyCultCacheDocumentDefinition): void {
// Every registration refusal is a SchemaConflictError naming the new definition's schema id and both schema names.
const refuse = (holder: RegisteredDefinition, claimed: CultCacheSchemaCatalogEntry, what: string) =>
new SchemaConflictError(claimed.schemaId, [holder.catalogEntry.schemaName, claimed.schemaName], "",
`CultCache ${what} is already registered for type "${holder.definition.type}" and cannot also be claimed by type "${definition.type}".`);
const existing = this.#definitions.get(definition.type);
if (existing && existing.definition !== definition) {
throw new Error(`CultCache already has a different definition registered for type "${definition.type}".`);
throw refuse(existing, this.#createCatalogEntry(definition), `type "${definition.type}"`);
}
const registered: RegisteredDefinition = existing ?? {
@ -557,25 +567,28 @@ export class CultCache {
indexAccessors.set(indexName, this.#compileAccessor(definition.type, `index "${indexName}"`, accessor));
}
const schemaIds = registered.catalogEntry.compatibleSchemaIds ?? [registered.catalogEntry.schemaId];
for (const schemaId of schemaIds) {
const existingBySchema = this.#schemaIdDefinitions.get(schemaId);
if (existingBySchema && existingBySchema !== registered) {
throw new Error(`CultCache schema id "${schemaId}" is already registered for type "${existingBySchema.definition.type}".`);
}
// One definition owns each schema id. Others may list it as compatible beside its owner: a record under the id resolves to
// its owner, and to a lister only when nothing owns it. Neither depends on the order definitions were registered in.
const ownId = registered.catalogEntry.schemaId;
const listed = (registered.catalogEntry.compatibleSchemaIds ?? []).filter((schemaId) => schemaId !== ownId);
const owner = this.#schemaIdDefinitions.get(ownId);
if (owner && owner !== registered) {
throw refuse(owner, registered.catalogEntry, `schema id "${ownId}"`);
}
const existingBySchemaName = this.#schemaNameDefinitions.get(registered.catalogEntry.schemaName);
if (existingBySchemaName && existingBySchemaName !== registered) {
throw new Error(
`CultCache schema name "${registered.catalogEntry.schemaName}" is already registered for type "${existingBySchemaName.definition.type}".`,
);
throw refuse(existingBySchemaName, registered.catalogEntry, `schema name "${registered.catalogEntry.schemaName}"`);
}
const derived = this.#deriveTypeLookups(definition.type, nameAccessor, indexAccessors);
// Every check and accessor has run; install.
this.#definitions.set(definition.type, registered);
for (const schemaId of schemaIds) {
this.#schemaIdDefinitions.set(schemaId, registered);
this.#schemaIdDefinitions.set(ownId, registered);
for (const schemaId of listed) {
const listers = this.#listingDefinitions.get(schemaId) ?? [];
if (!listers.includes(registered)) {
this.#listingDefinitions.set(schemaId, [...listers, registered]);
}
}
this.#schemaNameDefinitions.set(registered.catalogEntry.schemaName, registered);
this.#installAccessors(registered, nameAccessor, indexAccessors, derived);
@ -698,8 +711,7 @@ export class CultCache {
}
#createCatalogEntry(definition: AnyCultCacheDocumentDefinition): CultCacheSchemaCatalogEntry {
const schemaName = definition.schemaName ?? definition.type;
const schemaVersion = definition.schemaVersion ?? `${schemaName}.v1`;
const { schemaId, schemaName, schemaVersion, compatibleSchemaIds } = schemaIdentityOf(definition);
const canonicalSchemaJson = definition.canonicalSchemaJson ?? JSON.stringify({
schemaName,
schemaVersion,
@ -716,10 +728,6 @@ export class CultCache {
isName: member.isName === true,
})),
});
const schemaId = definition.schemaId ?? schemaName;
const compatibleSchemaIds = [
...new Set([schemaId, ...(definition.compatibleSchemaIds ?? [])]),
];
return {
schemaId,
@ -727,7 +735,7 @@ export class CultCache {
schemaVersion,
contentHash: definition.contentHash ?? schemaId,
canonicalSchemaJson,
compatibleSchemaIds,
compatibleSchemaIds: [...compatibleSchemaIds],
members: [...resolveDocumentMembers(definition)]
.sort((left, right) => left.slot - right.slot)
.map((member) => ({
@ -810,7 +818,16 @@ export class CultCache {
#resolveDefinitionForEnvelope(entry: CultCacheEnvelope): RegisteredDefinition | undefined {
if (entry.schemaId) {
return this.#schemaIdDefinitions.get(entry.schemaId)
const listers = this.#listingDefinitions.get(entry.schemaId) ?? [];
const owner = this.#schemaIdDefinitions.get(entry.schemaId);
if (!owner && listers.length > 1) {
const types = listers.map((lister) => lister.definition.type).sort();
throw new SchemaConflictError(entry.schemaId, listers.map((lister) => lister.catalogEntry.schemaName).sort(), entry.key,
`CultCache schema id "${entry.schemaId}" of record "${entry.key}" is owned by no registered type and declared compatible by ` +
`several (${types.join(", ")}), so it resolves to none. Register the type that owns the id, or declare it on one type.`);
}
return owner
?? listers[0]
?? this.#schemaNameDefinitions.get(entry.type)
?? this.#definitions.get(entry.type);
}
@ -880,8 +897,10 @@ export class CultCache {
return {
...entry,
payload: this.#cloneBytes(entry.payload),
schemaId: entry.schemaId ?? registered.catalogEntry.schemaId,
catalogEntry: entry.catalogEntry ?? registered.catalogEntry,
// Admitted under the registered schema's id, as a put stamps it: a record loaded under a compatible id is
// written back under the id its catalog entry carries.
schemaId: registered.catalogEntry.schemaId,
catalogEntry: registered.catalogEntry,
value,
lookups: this.#lookupsFor(registered.nameAccessor, registered.indexAccessors, value),
};

21
packages/cultcache-ts/src/document.ts
View file

@ -149,6 +149,27 @@ export function resolveDocumentMembers(definition: AnyCultCacheDocumentDefinitio
}));
}
// The identity a definition's records carry: the schema id they are stamped with, the schema name and version, and every id
// the definition answers to (its own first). Every CultCache runtime surface and every CultNet or CultMesh layer over it
// derives a definition's identity here and nowhere else, so a record's id means the same type wherever it is read.
export interface CultCacheSchemaIdentity {
readonly schemaId: string;
readonly schemaName: string;
readonly schemaVersion: string;
readonly compatibleSchemaIds: readonly string[];
}
export function schemaIdentityOf(definition: AnyCultCacheDocumentDefinition): CultCacheSchemaIdentity {
const schemaName = definition.schemaName ?? definition.type;
const schemaId = definition.schemaId ?? schemaName;
return {
schemaId,
schemaName,
schemaVersion: definition.schemaVersion ?? `${schemaName}.v1`,
compatibleSchemaIds: [...new Set([schemaId, ...(definition.compatibleSchemaIds ?? [])])],
};
}
export function defineDocumentRegistry<
TDefinitions extends readonly AnyCultCacheDocumentDefinition[],
>(...definitions: TDefinitions): CultCacheDocumentRegistry<TDefinitions> {

5
packages/cultcache-ts/src/index.ts
View file

@ -9,10 +9,13 @@ export {
documentPayloadBytes,
documentPayloadFromBytes,
encodeDocumentPayload,
schemaIdentityOf,
type CultCacheSchemaIdentity,
} from "./document";
export { CultCachePayloadDepthError, MAX_PAYLOAD_DEPTH } from "./payload-depth";
export type { DocumentPayload } from "./document";
export { SingleFileMessagePackBackingStore } from "./single-file-messagepack-backing-store";
export { SingleFileMessagePackBackingStore, readSingleFileStore, type SingleFileStoreRead } from "./single-file-messagepack-backing-store";
export { SchemaConflictError, StoreUnreadableError } from "./store-format";
export type {
AnyCultCacheDocumentDefinition,
CacheBackingStore,

444
packages/cultcache-ts/src/single-file-messagepack-backing-store.ts
View file

@ -1,13 +1,13 @@
import { lstat, mkdir, readFile } from "node:fs/promises";
import { lstatSync, readFileSync } from "node:fs";
import { mkdir } from "node:fs/promises";
import { dirname, resolve } from "node:path";
import { decode, encode } from "@msgpack/msgpack";
import { z } from "zod";
import { documentPayloadBytes, documentPayloadFromBytes } from "./document";
import { checkPayloadDepth } from "./payload-depth";
import { commitStoreFile, withStoreWriteLock } from "./store-file";
import { STORE_FORMAT_VERSION, isV1Snapshot, refuseUnreadStoreBytes, requireV1RecordSlots } from "./store-format";
import { STORE_FORMAT_VERSION, SchemaConflictError, type StoreFormat, StoreUnreadableError, isStoreSnapshot, refuseUnreadStoreBytes, requireStoreSlots, requireV1RecordSlots } from "./store-format";
import type {
CacheBackingStore,
CultCacheEnvelope,
@ -39,116 +39,89 @@ export class SingleFileMessagePackBackingStore implements CacheBackingStore {
#writeQueue: Promise<void> = Promise.resolve();
// The header this store read; a rewrite keeps it, so a store that holds element ids never sheds its marker.
#format: StoreFormat = STORE_FORMAT_VERSION;
constructor(filePath: string) {
this.filePath = resolve(filePath);
}
async pullAll(): Promise<CultCacheEnvelope[]> {
const { envelopes, healLegacy } = await this.#readDisk();
if (healLegacy) {
// The disk decides the header: a file that is gone or legacy is not marked.
this.#format = STORE_FORMAT_VERSION;
const disk = this.#readDisk();
this.#format = disk.format;
if (disk.repairedLegacyPayload) {
// The heal decides from its own read, taken under the store lock.
await this.#enqueue(async () => {
const again = await this.#readDisk();
if (again.healLegacy) {
await this.#writeAll(again.envelopes);
const again = this.#readDisk();
this.#format = again.format;
if (again.repairedLegacyPayload) {
await this.#writeAll([], again.envelopes);
}
});
}
return envelopes;
}
async #readDisk(): Promise<{ envelopes: CultCacheEnvelope[]; healLegacy: boolean }> {
// An empty file is not a store, since no CultCache writer leaves one, so it is
// decoded and refused. Only a read that finds nothing at the path is an empty store.
const data = await readStore(this.filePath);
if (data === undefined) {
return { envelopes: [], healLegacy: false };
}
const decoded = decode(data);
const snapshot = decodeSnapshot(decoded);
if (snapshot) {
const envelopes = snapshot.records.map((record) => {
const catalogEntry = resolveCatalogEntryForRecord(record, snapshot.catalogBySchemaId);
if (!catalogEntry) {
throw new Error(`CultCache persisted record "${record.key}" references missing schema id "${record.schemaId}".`);
}
return {
key: record.key,
type: catalogEntry.schemaName,
schemaId: record.schemaId,
catalogEntry,
payload: documentPayloadFromBytes(record.payload),
storedAt: record.storedAt,
};
});
return { envelopes, healLegacy: false };
}
const legacy = decodeLegacyEnvelopeArray(decoded);
if (legacy) {
let repairedLegacyPayload = false;
const normalized = legacy.map((entry) => {
const payload = normalizePayload(entry.payload);
if (payload !== entry.payload) {
repairedLegacyPayload = true;
}
return {
...entry,
payload,
};
});
const parsed: CultCacheEnvelope[] = envelopeArraySchema.parse(normalized).map((entry) => ({
...entry,
payload: documentPayloadFromBytes(entry.payload),
}));
return { envelopes: parsed, healLegacy: repairedLegacyPayload };
}
throw new Error(`CultCache file ${this.filePath} is not a recognized CultCache MessagePack store.`);
return disk.envelopes;
}
async push(entry: CultCacheEnvelope): Promise<void> {
await this.#enqueue(async () => {
const existing = (await this.#readDisk()).envelopes;
const disk = this.#readDisk();
this.#format = disk.format;
const existing = disk.envelopes;
const filtered = existing.filter(
(candidate) => !(candidate.type === entry.type && candidate.key === entry.key),
);
filtered.push(entry);
await this.#writeAll(filtered);
await this.#writeAll([entry], filtered);
});
}
async delete(entry: CultCacheEnvelope): Promise<void> {
await this.#enqueue(async () => {
const existing = (await this.#readDisk()).envelopes;
const disk = this.#readDisk();
this.#format = disk.format;
const existing = disk.envelopes;
const filtered = existing.filter(
(candidate) => !(candidate.type === entry.type && candidate.key === entry.key),
);
await this.#writeAll(filtered);
await this.#writeAll([], filtered);
});
}
async pushAll(entries: CultCacheEnvelope[], options: PushAllOptions = {}): Promise<void> {
await this.#enqueue(async () => {
// A whole write of a file it cannot read proceeds as before (a dangling link is
// replaced, R3); only a stamp it can read is judged.
const data = options.soft ? await readStore(this.filePath) : await readStore(this.filePath).catch(() => undefined);
if (data !== undefined) {
if (options.soft) {
// A flush of the whole store writes the header the file on disk carries, read now: a file marked for element ids
// stays marked, and one that is not (or is gone or legacy) is written unmarked. The file is read by the same
// reader as `pullAll`, so one it would refuse (not exactly one store, a variant, a body it cannot decode) is refused
// and left as it is. A soft flush asks that reader too: nothing at the path is written, a store is left alone.
try {
const disk = this.#readDisk();
if (options.soft && !disk.absent) {
return;
}
refuseUnreadStoreBytes(data);
this.#format = disk.format;
} catch (error) {
// Only a dangling link reaches here with ENOENT: it holds no store, so a whole-store flush has no header to keep and
// replaces the link (R3 pins this). A soft flush does not replace what it cannot read, and every merging writer
// has already refused the link when it read the current entries.
if (options.soft || (error as NodeJS.ErrnoException).code !== "ENOENT") {
throw error;
}
this.#format = STORE_FORMAT_VERSION;
}
await this.#writeAll(entries);
await this.#writeAll(entries, []);
});
}
// The one reader of the store file, asked by open, push, delete and flush alike; see readSingleFileStore.
#readDisk(): SingleFileStoreRead {
return readSingleFileStore(this.filePath);
}
async #enqueue<T>(operation: () => Promise<T>): Promise<T> {
let result!: T;
@ -167,29 +140,54 @@ export class SingleFileMessagePackBackingStore implements CacheBackingStore {
return result;
}
async #writeAll(entries: CultCacheEnvelope[]): Promise<void> {
async #writeAll(supplied: CultCacheEnvelope[], arrived: CultCacheEnvelope[]): Promise<void> {
await mkdir(dirname(this.filePath), { recursive: true });
await commitStoreFile(this.filePath, encode(encodeSnapshot(entries)));
await commitStoreFile(this.filePath, encode(encodeSnapshot(supplied, arrived, this.#format)));
}
}
/**
* Reads a store file, or returns undefined when nothing is at its path. A dangling symbolic
* link is something: reading it as empty would let the writer rename a file over the link
* and move the store off its volume. Any other failure to reach the file is thrown.
* The one reader of a CultCache single-file store, for this package's store and for any consumer that must read a
* store file (an authority bundle, say) rather than decode it itself. A path with nothing at it is an absent, empty,
* unmarked store. A dangling symbolic link is something: reading it as empty would let a writer rename a file over
* the link and move the store off its volume, so it throws with code ENOENT, as does any other failure to reach the
* file. A zero-byte file is not a store and, like anything that does not decode completely, throws StoreUnreadableError.
* The read is synchronous so a consumer that loads authority at start-up needs no second reader to stay synchronous.
*/
async function readStore(path: string): Promise<Uint8Array | undefined> {
export function readSingleFileStore(path: string): SingleFileStoreRead {
const data = readStoreBytes(path);
if (data === undefined) {
return { absent: true, format: STORE_FORMAT_VERSION, envelopes: [], repairedLegacyPayload: false };
}
try {
return await readFile(path);
return { absent: false, ...decodeStoreFile(data) };
} catch (error) {
throw new StoreUnreadableError(path, error);
}
}
/**
* The bytes at `path`, or undefined when nothing is there. Nothing is there only when both looks agree: the read found no
* file and the follow-up lstat found no entry. Every other outcome is an error, because the path may have changed
* between the two calls (a link in a parent swapped for a file, a store created or removed): an lstat failure other than
* ENOENT is thrown as it is, and an entry the lstat does find, a dangling link or one that appeared since the read, is
* thrown with code ENOENT rather than read as an empty store. `lstat` is a seam so a test can stage the second look;
* this module exports the function for that test, and the package index does not.
*/
export function readStoreBytes(path: string, lstat: (path: string) => { isSymbolicLink(): boolean } = lstatSync): Uint8Array | undefined {
try {
return readFileSync(path);
} catch (error) {
if ((error as NodeJS.ErrnoException).code !== "ENOENT") {
throw error;
}
}
let entry: { isSymbolicLink(): boolean };
try {
await lstat(path);
entry = lstat(path);
} catch (error) {
if ((error as NodeJS.ErrnoException).code === "ENOENT") {
return undefined;
@ -198,7 +196,8 @@ async function readStore(path: string): Promise<Uint8Array | undefined> {
throw error;
}
throw Object.assign(new Error(`CultCache store ${path} is a symbolic link whose target does not exist.`), { code: "ENOENT" });
const what = entry.isSymbolicLink() ? "is a symbolic link whose target does not exist" : "appeared while it was being read";
throw Object.assign(new Error(`CultCache store ${path} ${what}.`), { code: "ENOENT" });
}
type PersistedRecord = {
@ -209,13 +208,19 @@ type PersistedRecord = {
};
type DecodedSnapshot = {
format: StoreFormat;
catalogBySchemaId: Map<string, CultCacheSchemaCatalogEntry>;
records: PersistedRecord[];
};
function encodeSnapshot(entries: CultCacheEnvelope[]): unknown[] {
const catalog = [...catalogEntriesFor(entries).values()]
.sort((left, right) => compareOrdinal(left.schemaName, right.schemaName));
function encodeSnapshot(supplied: CultCacheEnvelope[], arrived: CultCacheEnvelope[], format: StoreFormat): unknown[] {
const replaced = new Set(supplied.map((entry) => `${entry.type}::${entry.key}`));
const kept = arrived.filter((entry) => !replaced.has(`${entry.type}::${entry.key}`));
const entries = [...kept, ...supplied];
const catalog = catalogEntriesFor([
...kept.map((envelope) => ({ envelope, registered: false })),
...supplied.map((envelope) => ({ envelope, registered: true })),
]).sort((left, right) => compareOrdinal(left.schemaName, right.schemaName) || compareOrdinal(left.schemaId, right.schemaId));
const records = [...entries]
.sort((left, right) => compareOrdinal(left.key, right.key))
.map((entry) => [
@ -226,34 +231,134 @@ function encodeSnapshot(entries: CultCacheEnvelope[]): unknown[] {
]);
return [
STORE_FORMAT_VERSION,
format,
catalog.map(encodeCatalogEntry),
records,
];
}
function catalogEntriesFor(entries: CultCacheEnvelope[]): Map<string, CultCacheSchemaCatalogEntry> {
const catalog = new Map<string, CultCacheSchemaCatalogEntry>();
for (const entry of entries) {
const schemaId = schemaIdForEnvelope(entry);
if (!catalog.has(schemaId)) {
catalog.set(schemaId, entry.catalogEntry ?? {
// The catalog a write leaves, derived from the records being written and nothing else. For each schema id a record carries, one
// entry that publishes it: the entry that owns the id (a registered one, else one that arrived with the records), else one that
// lists it as compatible (registered first). Entries are written as chosen, never merged; a registered entry wins over an arrived
// one with the same own id. Records of different types under one id, two arrived entries with the same own id and different schema
// names, or a record no chosen entry publishes, refuse the write. An envelope the caller supplied is registered; one read back
// from the file is arrived.
function catalogEntriesFor(
records: Array<{ envelope: CultCacheEnvelope; registered: boolean }>,
): CultCacheSchemaCatalogEntry[] {
const entries: Array<{ entry: CultCacheSchemaCatalogEntry; registered: boolean }> = [];
const byId = new Map<string, CultCacheEnvelope[]>();
const tiers = new Map<string, Array<{ envelope: CultCacheEnvelope; registered: boolean; entry: CultCacheSchemaCatalogEntry }>>();
for (const { envelope, registered } of records) {
const schemaId = schemaIdForEnvelope(envelope);
const supplied = envelope.catalogEntry;
const publishes =
supplied !== undefined && (supplied.schemaId === schemaId || (supplied.compatibleSchemaIds ?? []).includes(schemaId));
const entry = publishes && supplied ? supplied : defaultEntryFor(envelope);
entries.push({ entry, registered });
byId.set(schemaId, [...(byId.get(schemaId) ?? []), envelope]);
tiers.set(schemaId, [...(tiers.get(schemaId) ?? []), { envelope, registered, entry }]);
}
const lists = (entry: CultCacheSchemaCatalogEntry, schemaId: string): boolean =>
(entry.compatibleSchemaIds ?? [entry.schemaId]).includes(schemaId);
const chosen = new Map<string, { entry: CultCacheSchemaCatalogEntry; registered: boolean }>();
for (const schemaId of [...byId.keys()].sort(compareOrdinal)) {
const carrying = byId.get(schemaId)!;
const recordKey = carrying.map((envelope) => envelope.key).sort(compareOrdinal)[0]!;
// Records of one tier under one id are one type. Across tiers the id keeps the type its records resolve to, so a write may
// not retype records that are already there: unless the entries of both tiers own the id, which is a rename of the schema
// under a stable id, and the registered descriptor wins. A read-back record's type is its schema's name, so tiers compare
// the names of the entries that publish the id.
const carried = tiers.get(schemaId)!;
for (const tier of [true, false]) {
const types = [...new Set(carried.filter((candidate) => candidate.registered === tier).map((candidate) => candidate.envelope.type))].sort(
compareOrdinal,
);
if (types.length > 1) {
throw new SchemaConflictError(schemaId, types, recordKey);
}
}
const tierNames = (tier: boolean): string[] => [
...new Set(carried.filter((candidate) => candidate.registered === tier).map((candidate) => candidate.entry.schemaName)),
];
const ownedBy = (tier: boolean): boolean =>
carried.some((candidate) => candidate.registered === tier && candidate.entry.schemaId === schemaId);
const [registeredName, arrivedName] = [tierNames(true)[0], tierNames(false)[0]];
if (registeredName !== undefined && arrivedName !== undefined && registeredName !== arrivedName && !(ownedBy(true) && ownedBy(false))) {
throw new SchemaConflictError(schemaId, [registeredName, arrivedName].sort(compareOrdinal), recordKey);
}
const ownRegistered = canonical(entries.filter((candidate) => candidate.registered && candidate.entry.schemaId === schemaId))[0];
const ownArrived = canonical(entries.filter((candidate) => !candidate.registered && candidate.entry.schemaId === schemaId));
// Entries of one tier that share an own id must be one schema; a registered entry settles a disagreement among arrived ones.
const ownRegisteredAll = canonical(entries.filter((candidate) => candidate.registered && candidate.entry.schemaId === schemaId));
const names = [
...new Set((ownRegisteredAll.length > 0 ? ownRegisteredAll : ownArrived).map((candidate) => candidate.entry.schemaName)),
].sort(compareOrdinal);
if (names.length > 1) {
throw new SchemaConflictError(schemaId, names, recordKey);
}
let pick: { entry: CultCacheSchemaCatalogEntry; registered: boolean } | undefined;
if (ownRegistered !== undefined) {
pick = ownRegistered;
} else if (ownArrived.length > 0) {
pick = ownArrived[0];
} else {
pick =
canonical(entries.filter((candidate) => candidate.registered && lists(candidate.entry, schemaId)))[0] ??
canonical(entries.filter((candidate) => !candidate.registered && lists(candidate.entry, schemaId)))[0];
}
if (pick === undefined) {
continue;
}
const prior = chosen.get(pick.entry.schemaId);
if (prior !== undefined && prior.registered && !pick.registered) {
continue;
}
chosen.set(pick.entry.schemaId, pick);
}
for (const [schemaId, carrying] of byId) {
if (![...chosen.values()].some(({ entry }) => entry.schemaId === schemaId || lists(entry, schemaId))) {
throw new SchemaConflictError(
schemaId,
schemaName: entry.type,
schemaVersion: `${entry.type}.v1`,
contentHash: schemaId,
canonicalSchemaJson: JSON.stringify({
schemaName: entry.type,
schemaVersion: `${entry.type}.v1`,
members: [],
}),
compatibleSchemaIds: [schemaId],
members: [],
});
[...chosen.values()].map(({ entry }) => entry.schemaName).sort(compareOrdinal),
carrying.map((envelope) => envelope.key).sort(compareOrdinal)[0]!,
);
}
}
return catalog;
return [...chosen.values()].map(({ entry }) => entry);
}
// Entries that tie, in one fixed order, so the catalog does not depend on the order they arrive in.
function canonical<T extends { entry: CultCacheSchemaCatalogEntry }>(candidates: T[]): T[] {
const key = (candidate: T): string =>
[candidate.entry.schemaName, candidate.entry.contentHash ?? "", (candidate.entry.compatibleSchemaIds ?? []).join(",")].join("\u0000");
return [...candidates].sort((left, right) => compareOrdinal(key(left), key(right)));
}
function defaultEntryFor(entry: CultCacheEnvelope): CultCacheSchemaCatalogEntry {
const schemaId = schemaIdForEnvelope(entry);
return {
schemaId,
schemaName: entry.type,
schemaVersion: `${entry.type}.v1`,
contentHash: schemaId,
canonicalSchemaJson: JSON.stringify({
schemaName: entry.type,
schemaVersion: `${entry.type}.v1`,
members: [],
}),
compatibleSchemaIds: [schemaId],
members: [],
};
}
function schemaIdForEnvelope(entry: CultCacheEnvelope): string {
@ -285,82 +390,89 @@ function encodeCatalogMember(member: CultCacheSchemaCatalogMember): unknown[] {
];
}
export type SingleFileStoreRead = {
// True when nothing is at the path; the other fields then describe an empty unmarked store.
absent: boolean;
format: StoreFormat;
envelopes: CultCacheEnvelope[];
// A legacy file whose payloads were not bytes: reading it repairs them, and the store rewrites the file.
repairedLegacyPayload: boolean;
};
function decodeStoreFile(data: Uint8Array): Omit<SingleFileStoreRead, "absent"> {
// The stamp is judged before the body is decoded, as Rust's store_header does: a newer store is refused by name.
refuseUnreadStoreBytes(data);
const decoded = decode(data);
const snapshot = decodeSnapshot(decoded);
if (snapshot) {
const envelopes = snapshot.records.map((record): CultCacheEnvelope => {
const catalogEntry = snapshot.catalogBySchemaId.get(record.schemaId);
if (!catalogEntry) {
throw new Error(`CultCache persisted record "${record.key}" references missing schema id "${record.schemaId}".`);
}
return {
key: record.key,
type: catalogEntry.schemaName,
schemaId: record.schemaId,
catalogEntry,
payload: documentPayloadFromBytes(record.payload),
storedAt: record.storedAt,
};
});
return { format: snapshot.format, envelopes, repairedLegacyPayload: false };
}
const legacy = decodeLegacyEnvelopeArray(decoded);
if (!legacy) {
throw new Error("CultCache file is not a recognized CultCache MessagePack store.");
}
let repairedLegacyPayload = false;
const normalized = legacy.map((entry) => {
const payload = normalizePayload(entry.payload);
if (payload !== entry.payload) {
repairedLegacyPayload = true;
}
return {
...entry,
payload,
};
});
return { format: STORE_FORMAT_VERSION, envelopes: envelopeArraySchema.parse(normalized).map((entry) => ({ ...entry, payload: documentPayloadFromBytes(entry.payload) })) as CultCacheEnvelope[], repairedLegacyPayload };
}
function decodeSnapshot(decoded: unknown): DecodedSnapshot | undefined {
if (!isV1Snapshot(decoded)) {
if (!isStoreSnapshot(decoded)) {
return undefined;
}
requireStoreSlots(decoded);
const catalogRaw = decoded[1];
const recordsRaw = decoded[2];
if (!Array.isArray(catalogRaw) || !Array.isArray(recordsRaw)) {
throw new Error("CultCache v1 snapshot must contain a schema catalog and record array.");
}
// A schema id names the entry that has it as its own id; only an id no entry owns names the entry that lists it as compatible.
const catalogEntries = catalogRaw.map(decodeCatalogEntry);
const catalogBySchemaId = new Map<string, CultCacheSchemaCatalogEntry>();
for (const entry of catalogRaw) {
const catalogEntry = decodeCatalogEntry(entry);
catalogBySchemaId.set(catalogEntry.schemaId, catalogEntry);
for (const catalogEntry of catalogEntries) {
if (!catalogBySchemaId.has(catalogEntry.schemaId)) {
catalogBySchemaId.set(catalogEntry.schemaId, catalogEntry);
}
}
for (const catalogEntry of catalogEntries) {
for (const compatibleSchemaId of catalogEntry.compatibleSchemaIds ?? []) {
catalogBySchemaId.set(compatibleSchemaId, catalogEntry);
if (!catalogBySchemaId.has(compatibleSchemaId)) {
catalogBySchemaId.set(compatibleSchemaId, catalogEntry);
}
}
}
const records = recordsRaw.map(decodeRecord);
return { catalogBySchemaId, records };
}
function resolveCatalogEntryForRecord(
record: PersistedRecord,
catalogBySchemaId: Map<string, CultCacheSchemaCatalogEntry>,
): CultCacheSchemaCatalogEntry | undefined {
const existing = catalogBySchemaId.get(record.schemaId);
if (existing) {
return existing;
}
const schemaVersion = inferSchemaVersionFromPayload(record.payload);
if (!schemaVersion) {
return undefined;
}
const schemaName = inferSchemaName(schemaVersion);
if (!schemaName) {
return undefined;
}
return {
schemaId: record.schemaId,
schemaName,
schemaVersion,
contentHash: record.schemaId,
canonicalSchemaJson: "",
compatibleSchemaIds: [],
members: [],
};
}
function inferSchemaVersionFromPayload(payload: Uint8Array): string | undefined {
try {
checkPayloadDepth(payload);
const decoded = decode(payload);
if (Array.isArray(decoded) && typeof decoded[0] === "string") {
return decoded[0];
}
if (isObject(decoded)) {
const schemaVersion = decoded.schemaVersion ?? decoded.schema_version;
return typeof schemaVersion === "string" ? schemaVersion : undefined;
}
return undefined;
} catch {
return undefined;
}
}
function inferSchemaName(schemaVersion: string): string | undefined {
const match = /^(.*)\.v\d+$/.exec(schemaVersion);
return match?.[1];
return { format: decoded[0], catalogBySchemaId, records };
}
function decodeCatalogEntry(value: unknown): CultCacheSchemaCatalogEntry {

95
packages/cultcache-ts/src/store-format.ts
View file

@ -2,26 +2,33 @@ import { decode } from "@msgpack/msgpack";
export const STORE_FORMAT_VERSION = "cultcache.store.v1";
/**
* A store that can hold element ids. A reader older than element ids refuses this header: it would skip the id slots
* of a nested element and rewrite the element without them. This runtime writes it back only for a store it read
* under it; it does not decide when a document carries ids.
*/
export const STORE_FORMAT_ELEMENT_IDS = "cultcache.store.v3";
export type StoreFormat = typeof STORE_FORMAT_VERSION | typeof STORE_FORMAT_ELEMENT_IDS;
const STORE_FORMAT_PREFIX = "cultcache.store.";
const PERSISTED_RECORD_SLOTS = 4;
const STORE_SLOTS = 3;
/**
* True for a v1 snapshot, false for anything that is not a CultCache store header (the legacy
* envelope array). Any other `cultcache.store.*` header is refused by name.
* True for a decoded array that leads with a `cultcache.store.*` stamp, false for the legacy envelope array. It judges the
* shape only: the verdict on the stamp is {@link refuseUnreadStoreBytes}, which a reader calls on the bytes before it decodes,
* so the stamp of a decoded array it is given is one this runtime reads.
*/
export function isV1Snapshot(decoded: unknown): decoded is unknown[] {
if (!Array.isArray(decoded) || typeof decoded[0] !== "string" || !decoded[0].startsWith(STORE_FORMAT_PREFIX)) {
return false;
}
refuseUnreadFormat(decoded[0]);
return true;
export function isStoreSnapshot(decoded: unknown): decoded is [StoreFormat, ...unknown[]] {
return Array.isArray(decoded) && typeof decoded[0] === "string" && decoded[0].startsWith(STORE_FORMAT_PREFIX);
}
/**
* Refuses a store whose header names a format this runtime does not read, judging the header
* alone: the leading array's first element, when it is a complete string. Bytes with no such
* header (a legacy array, an empty, truncated or garbled file) are not refused here.
* Refuses a store whose header names a format this runtime does not read, judging the header alone: the leading array's first
* element, when it is a complete string. The reader calls it before it decodes the body, so a newer store is refused by name
* whether or not its body is whole. Bytes with no such header (a legacy array, an empty, truncated or garbled file) are not
* refused here.
*/
export function refuseUnreadStoreBytes(bytes: Uint8Array): void {
const header = peekHeader(bytes);
@ -30,17 +37,8 @@ export function refuseUnreadStoreBytes(bytes: Uint8Array): void {
}
}
function refuseUnreadFormat(header: string): void {
if (header !== STORE_FORMAT_VERSION) {
throw new Error(
`CultCache store format ${describeHeader(header)} is not one this runtime reads; it reads "${STORE_FORMAT_VERSION}" only.`,
);
}
}
// The first element of a leading non-empty MessagePack array when it is a complete string;
// anything else is undefined. The string is decoded by the decoder a read uses, so a stamp
// gets one verdict from a read and from a whole write.
// The first element of a leading non-empty MessagePack array when it is a complete string; anything else is undefined. The
// string is decoded by the decoder a read uses, so a stamp gets one verdict from the peek and from the whole decode.
function peekHeader(bytes: Uint8Array): string | undefined {
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
const lead = bytes[0];
@ -64,6 +62,50 @@ function peekHeader(bytes: Uint8Array): string | undefined {
return typeof header === "string" ? header : undefined;
}
/** The one verdict on a stamp: this runtime reads v1 and v3, and any other `cultcache.store.*` header is refused by name. */
function refuseUnreadFormat(header: string): void {
if (header !== STORE_FORMAT_VERSION && header !== STORE_FORMAT_ELEMENT_IDS) {
throw new Error(
`CultCache store format ${describeHeader(header)} is not one this runtime reads; it reads "${STORE_FORMAT_VERSION}" and "${STORE_FORMAT_ELEMENT_IDS}" only.`,
);
}
}
/**
* A write the catalog cannot describe: records of different types under one schema id, two arrived schemas that share an id and
* disagree on the schema name, or a record no chosen entry publishes. Nothing is written. It names the id, the names involved and
* a record key. A cache throws it too, with an empty record key, when a definition cannot register beside one it already holds,
* and when a record's schema id names no single registered definition.
*/
export class SchemaConflictError extends Error {
readonly schemaId: string;
readonly schemaNames: string[];
readonly recordKey: string;
constructor(schemaId: string, schemaNames: string[], recordKey: string, message?: string) {
super(message ?? `Schema id "${schemaId}" cannot describe record "${recordKey}": ${schemaNames.map((name) => `"${name}"`).join(", ")}.`);
this.name = "SchemaConflictError";
this.schemaId = schemaId;
this.schemaNames = schemaNames;
this.recordKey = recordKey;
}
}
/**
* A store file this runtime cannot read: not exactly one complete store (truncated, bytes after it, a missing or extra slot),
* a header or record it does not know, or a body it cannot decode. Open, flush and every rewrite refuse a file with this
* error, and a refused file is left as it was. `cause` is what the reader choked on.
*/
export class StoreUnreadableError extends Error {
readonly filePath: string;
constructor(filePath: string, cause: unknown) {
super(`CultCache store ${filePath} is not readable: ${cause instanceof Error ? cause.message : String(cause)}`, { cause });
this.name = "StoreUnreadableError";
this.filePath = filePath;
}
}
/**
* A store header as a refusal may show it: echoed in the shape `cultcache.store.v<digits>`,
* and otherwise described only by its length, since the bytes are the store's.
@ -102,3 +144,10 @@ export function requireV1RecordSlots(record: unknown[]): void {
);
}
}
/** Refuses a store array with more or fewer slots than the header, the schema catalog and the records; a header alone is not a store. */
export function requireStoreSlots(decoded: unknown[]): void {
if (decoded.length !== STORE_SLOTS) {
throw new Error(`CultCache store has ${decoded.length} top-level slots; this runtime reads ${STORE_SLOTS} (header, schema catalog, records).`);
}
}

1111
packages/cultcache-ts/test/cult-cache.test.ts
View file

File diff suppressed because it is too large Load diff

20
packages/cultcache-ts/test/payload-depth.test.ts
View file

@ -20,7 +20,6 @@ import {
} from "../src/document";
import * as codecs from "../src/codec";
import { checkPayloadDepth } from "../src/payload-depth";
import { SingleFileMessagePackBackingStore } from "../src/single-file-messagepack-backing-store";
const L = MAX_PAYLOAD_DEPTH;
const CANARY = "CANARY-payload-depth-91d3";
@ -175,11 +174,11 @@ test("the limit and its error are exported from the root, and the walk itself is
assert.equal("checkPayloadDepth" in root, false);
});
// A v1 store whose one record carries `payload` as bin and no catalog entry, so the record's schema is inferred from the
// payload's first element.
// A v1 store whose one record carries `payload` as bin, under a catalog entry that names its schema.
async function storeWith(payload: Uint8Array): Promise<string> {
const file = join(await mkdtemp(join(tmpdir(), "cultcache-payload-depth-")), "deep.cc");
await writeFile(file, encode(["cultcache.store.v1", [], [["record-1", "sha256:stale", "2026-06-25T12:00:00Z", payload]]]));
const entry = ["sha256:stale", "tests.schema_stamped_entry", "tests.schema_stamped_entry.v1", "stale", "", [], []];
await writeFile(file, encode(["cultcache.store.v1", [entry], [["record-1", "sha256:stale", "2026-06-25T12:00:00Z", payload]]]));
return file;
}
const stamped = (containersInsideTheStamp: number) =>
@ -189,25 +188,16 @@ test("the inspector previews a payload at the limit and reports one past it, or
const atLimit = await storeWith(stamped(L - 1));
const previewed = inspectCultCacheBytes(atLimit, await readFile(atLimit)).records[0]!;
assert.equal(previewed.payloadDecodeError, undefined);
assert.equal(previewed.schemaName, "tests.schema_stamped_entry", "a payload the walk accepts yields its stamp");
assert.equal(previewed.schemaName, "tests.schema_stamped_entry");
for (const inside of [L, 100_000]) {
const file = await storeWith(stamped(inside));
const record = inspectCultCacheBytes(file, await readFile(file)).records[0]!;
assert.equal(record.payloadDecodeError, "document payload nests 257 MessagePack containers deep; the limit is 256.");
assert.equal(record.schemaName, "<missing catalog entry>", "the stamp of a payload the walk refuses is not recovered");
assert.equal(record.schemaName, "tests.schema_stamped_entry");
}
});
test("the single-file store infers a legacy record's schema from a payload at the limit and from none past it", async () => {
const atLimit = await storeWith(stamped(L - 1));
const [read] = await new SingleFileMessagePackBackingStore(atLimit).pullAll();
assert.equal(read.catalogEntry?.schemaVersion, "tests.schema_stamped_entry.v1");
const past = await storeWith(stamped(L));
await assert.rejects(() => new SingleFileMessagePackBackingStore(past).pullAll(), /missing schema id/u);
});
// A header whose length or count needs all four bytes: the low sixteen bits of the length are small, the whole is not.
const be32 = (n: number) => bytes((n >>> 24) & 0xff, (n >>> 16) & 0xff, (n >>> 8) & 0xff, n & 0xff);

72
packages/cultcache-ts/test/union-parity.test.ts
View file

@ -48,16 +48,23 @@ type Node =
| { kind: "Text"; id: string; content: string }
| { kind: "Use"; id: string; composition: Composition; slots: (Node | null)[] }
| { kind: "Image"; id: string; uri: string; alt: string }
| { kind: "Wrap"; inner: Node | null };
| { kind: "Wrap"; inner: Node | null; id: string };
interface Composition { name: string; body: Node | null }
type Outcome = { kind: "Pending"; message: string } | { kind: "Completed"; sourceVersion: number } | { kind: "Denied" } | { kind: "Floor" } | { kind: "Ceiling" };
// The golden's Outcome: the arms of a list carry an element id, so Denied, Floor and Ceiling are one-slot records.
type GoldenOutcome =
| { kind: "Pending"; message: string; id: string }
| { kind: "Completed"; sourceVersion: number; id: string }
| { kind: "Denied"; id: string }
| { kind: "Floor"; id: string }
| { kind: "Ceiling"; id: string };
type Item = { kind: "Gem"; name: string } | { kind: "Tool"; name: string; uses: number } | { kind: "Part"; name: string; count: number };
type CraftedItem = { kind: "Tool"; name: string; uses: number } | { kind: "Part"; name: string; count: number };
interface Surface {
id: string;
version: number;
root: Node | null;
outcomes: (Outcome | null)[];
outcomes: (GoldenOutcome | null)[];
part: Item | null;
crafted: CraftedItem | null;
missing: Node | null;
@ -71,7 +78,7 @@ const node: Codec<Node | null> = union<Node>("Node", {
0: ["id", string], 1: ["composition", composition], 2: ["slots", list(lazy(() => node))],
})],
5: ["Image", slots<{ id: string; uri: string; alt: string }>({ 0: ["id", string], 1: ["uri", string], 3: ["alt", string] })],
6: ["Wrap", slots<{ inner: Node | null }>({ 0: ["inner", lazy(() => node)] })],
6: ["Wrap", slots<{ inner: Node | null; id: string }>({ 0: ["inner", lazy(() => node)], 1: ["id", string] })],
}, `${CS}Node`);
const outcome = union<Outcome>("Outcome", {
0: ["Pending", slots<{ message: string }>({ 0: ["message", string] })],
@ -80,6 +87,14 @@ const outcome = union<Outcome>("Outcome", {
[-2147483648]: ["Floor", unit],
2147483647: ["Ceiling", unit],
}, `${CS}Outcome`);
const idOnly = slots<{ id: string }>({ 0: ["id", string] });
const goldenOutcome = union<GoldenOutcome>("Outcome", {
0: ["Pending", slots<{ message: string; id: string }>({ 0: ["message", string], 1: ["id", string] })],
1: ["Completed", slots<{ sourceVersion: number; id: string }>({ 0: ["sourceVersion", int32], 1: ["id", string] })],
2: ["Denied", idOnly],
[-2147483648]: ["Floor", idOnly],
2147483647: ["Ceiling", idOnly],
}, `${CS}Outcome`);
const toolArm = slots<{ name: string; uses: number }>({ 0: ["name", string], 1: ["uses", int32] });
const partArm = slots<{ name: string; count: number }>({ 0: ["name", string], 1: ["count", int32] });
const item = union<Item>("Item", {
@ -92,7 +107,7 @@ const surfaceCodec = slots<Surface>({
0: ["id", string],
1: ["version", int32],
2: ["root", node],
3: ["outcomes", list(outcome)],
3: ["outcomes", list(goldenOutcome)],
4: ["part", item],
5: ["crafted", craftedItem],
6: ["missing", node, null],
@ -111,7 +126,7 @@ const nodeSchema: z.ZodType<Node | null> = z.lazy(() =>
slots: z.array(nodeSchema),
}),
z.object({ kind: z.literal("Image"), id: z.string(), uri: z.string(), alt: z.string() }),
z.object({ kind: z.literal("Wrap"), inner: nodeSchema }),
z.object({ kind: z.literal("Wrap"), inner: nodeSchema, id: z.string() }),
]),
]));
const surfaceSchema = z.object({
@ -121,11 +136,11 @@ const surfaceSchema = z.object({
outcomes: z.array(z.union([
z.null(),
z.discriminatedUnion("kind", [
z.object({ kind: z.literal("Pending"), message: z.string() }),
z.object({ kind: z.literal("Completed"), sourceVersion: z.number() }),
z.object({ kind: z.literal("Denied") }),
z.object({ kind: z.literal("Floor") }),
z.object({ kind: z.literal("Ceiling") }),
z.object({ kind: z.literal("Pending"), message: z.string(), id: z.string() }),
z.object({ kind: z.literal("Completed"), sourceVersion: z.number(), id: z.string() }),
z.object({ kind: z.literal("Denied"), id: z.string() }),
z.object({ kind: z.literal("Floor"), id: z.string() }),
z.object({ kind: z.literal("Ceiling"), id: z.string() }),
]),
])),
part: z.union([z.null(), z.discriminatedUnion("kind", [
@ -157,29 +172,33 @@ const golden: Surface = {
version: 3,
root: {
kind: "Box",
id: "root",
id: "0000000000c0",
children: [
{ kind: "Text", id: "t1", content: "hello" },
{ kind: "Text", id: "0000000000c1", content: "hello" },
{
kind: "Use",
id: "u1",
id: "0000000000c2",
composition: {
name: "comp",
body: {
kind: "Box",
id: "inner",
id: "0000000000d0",
children: [
{ kind: "Text", id: "deep", content: "x" },
{ kind: "Image", id: "img", uri: "uri://a", alt: "alt" },
{ kind: "Text", id: "0000000000d1", content: "x" },
{ kind: "Image", id: "0000000000d2", uri: "uri://a", alt: "alt" },
],
},
},
slots: [{ kind: "Text", id: "s1", content: "slot" }, null, { kind: "Box", id: "b2", children: [] }],
slots: [{ kind: "Text", id: "0000000000e1", content: "slot" }, null, { kind: "Box", id: "0000000000e2", children: [] }],
},
{ kind: "Image", id: "i2", uri: "uri://b", alt: "" },
{ kind: "Image", id: "0000000000c3", uri: "uri://b", alt: "" },
],
},
outcomes: [{ kind: "Pending", message: "wait" }, { kind: "Completed", sourceVersion: 7 }, { kind: "Denied" }],
outcomes: [
{ kind: "Pending", message: "wait", id: "0000000000a1" },
{ kind: "Completed", sourceVersion: 7, id: "0000000000a2" },
{ kind: "Denied", id: "0000000000a3" },
],
part: { kind: "Tool", name: "hammer", uses: 12 },
crafted: { kind: "Part", name: "gear", count: 4 },
missing: null,
@ -200,8 +219,13 @@ test("the union surface vector decodes to the constructed value and re-encodes b
assert.equal(hex(writeWire(surfaceCodec.encode(golden))), hex(committed));
});
test("a unit arm is written [key, []] and its payload must be an array", () => {
assert.deepEqual(outcome.encode({ kind: "Denied" }), [2, []]);
test("a unit arm is written as the C# reference writes it, and its payload must be an array", async () => {
// The bytes are tests/vectors/union-parity/unit-arm-*.msgpack, written by the reference, not spelled here.
for (const [file, kind] of [["denied", "Denied"], ["floor", "Floor"], ["ceiling", "Ceiling"]] as const) {
const reference = await read(`unit-arm-${file}.msgpack`);
assert.deepEqual(outcome.decode(readWire(reference)), { kind }, file);
assert.equal(hex(writeWire(outcome.encode({ kind }))), hex(reference), file);
}
assert.throws(() => outcome.decode([2, null]), /Union Outcome/u);
});
@ -814,14 +838,14 @@ test("a bytes slot reads a MessagePack str as its bytes and writes bin, and refu
const wrapChain = (wraps: number, leaf: Node): Node => {
let node: Node = leaf;
for (let index = 0; index < wraps; index += 1) node = { kind: "Wrap", inner: node };
for (let index = 0; index < wraps; index += 1) node = { kind: "Wrap", inner: node, id: "0000000000b1" };
return node;
};
// A Wrap adds 2 containers, a Box with no children 3, a Text leaf 2, and the outermost array is 1 (the C# test's arithmetic).
const AT_LIMIT_WRAPS = (MAX_PAYLOAD_DEPTH - 1 - 3) / 2;
const PAST_LIMIT_WRAPS = (MAX_PAYLOAD_DEPTH + 1 - 1 - 2) / 2;
const atLimitValue: Surface = { ...golden, root: wrapChain(AT_LIMIT_WRAPS, { kind: "Box", id: "leaf", children: [] }) };
const pastLimitValue: Surface = { ...golden, root: wrapChain(PAST_LIMIT_WRAPS, { kind: "Text", id: "leaf", content: CANARY }) };
const atLimitValue: Surface = { ...golden, root: wrapChain(AT_LIMIT_WRAPS, { kind: "Box", id: "0000000000f0", children: [] }) };
const pastLimitValue: Surface = { ...golden, root: wrapChain(PAST_LIMIT_WRAPS, { kind: "Text", id: "0000000000f0", content: CANARY }) };
async function boundsEntries() {
const raw = decode(await read("bounds-outcomes.msgpack")) as unknown[][];

63
packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/CultMesh.kt
View file

@ -56,6 +56,17 @@ data class CultCacheEnvelope(
val payload: ByteArray,
)
/**
* A schema this cache cannot give to one more type: another type already holds the schema id, or the type already holds
* another schema id. Nothing is registered. It names the refused codec's schema id and both document types.
*/
class CultSchemaConflictException(
val schemaId: String,
val schemaNames: List<String>,
val recordKey: String,
message: String,
) : IOException(message)
class CultCache {
companion object {
const val GLOBAL_KEY = "__global__"
@ -65,7 +76,19 @@ class CultCache {
private val codecsBySchema = linkedMapOf<String, CultDocumentCodec<*>>()
private val values = linkedMapOf<String, LinkedHashMap<String, ByteArray>>()
// One type owns a schema id. Every path that names a codec (register, put, get, getAll, delete) comes through here.
fun <T : Any> register(codec: CultDocumentCodec<T>) {
val holder = codecs[codec.documentType]?.takeIf { it.schemaVersion != codec.schemaVersion }
?: codecsBySchema[codec.schemaVersion]?.takeIf { it.documentType != codec.documentType }
if (holder != null) {
throw CultSchemaConflictException(
codec.schemaVersion,
listOf(holder.documentType, codec.documentType),
"",
"Kotlin CultCache type ${holder.documentType} holds schema ${holder.schemaVersion}; " +
"type ${codec.documentType} cannot also be registered for schema id ${codec.schemaVersion}",
)
}
codecs[codec.documentType] = codec
codecsBySchema[codec.schemaVersion] = codec
}
@ -3555,6 +3578,42 @@ private fun cultCacheRawSnapshotsRoundTripThroughCultNetMessages() {
check(syncedAlias == KotlinUiNote("kotlin.alias_note.v1", "canonical-to-ui"))
check(target.require(uiNote, "note:alias") == syncedAlias)
// One type per schema in one cache: a second type for a held schema id, or a held type for another schema id, is
// refused typed, by register and by the implicit registration in put, get, getAll and delete alike.
fun refusal(action: () -> Unit): CultSchemaConflictException? =
try { action(); null } catch (conflict: CultSchemaConflictException) { conflict }
val canonicalOtherSchema = cultDocument(KotlinAliasNoteCodec(
documentType = "kotlin.alias_note",
schemaVersion = "kotlin.alias_note.v2",
create = ::KotlinCanonicalNote,
readSchemaVersion = KotlinCanonicalNote::schemaVersion,
readBody = KotlinCanonicalNote::body,
))
for ((first, second) in listOf(canonicalNote to uiNote, uiNote to canonicalNote, canonicalNote to canonicalOtherSchema)) {
@Suppress("UNCHECKED_CAST")
val claimant = second as CultDocumentDefinition<Any>
val claimantValue = claimant.codec.decode(canonicalNote.codec.encode(KotlinCanonicalNote("kotlin.alias_note.v1", "x")))
val attempts: List<(CultCache) -> Unit> = listOf(
{ cache -> cache.register(claimant) },
{ cache -> cache.put(claimant, "note:second", claimantValue) },
{ cache -> cache.get(claimant, "note:second") },
{ cache -> cache.getAll(claimant) },
{ cache -> cache.delete(claimant.codec, "note:second") },
)
for (attempt in attempts) {
val cache = CultCache()
@Suppress("UNCHECKED_CAST")
cache.register(first as CultDocumentDefinition<Any>)
val conflict = refusal { attempt(cache) }
check(conflict != null) { "${second.documentType}/${second.schemaVersion} beside ${first.documentType}/${first.schemaVersion} was not refused" }
check(conflict.schemaId == second.schemaVersion && conflict.recordKey == "")
check(conflict.schemaNames == listOf(first.documentType, second.documentType))
check(conflict.message!!.contains(second.schemaVersion))
check(cache.codecForSchema(first.schemaVersion).documentType == first.documentType)
}
}
CultCache().apply { register(canonicalNote); register(canonicalNote); put(canonicalNote, "note:again", KotlinCanonicalNote("kotlin.alias_note.v1", "again")) }
val reactiveUiNote = cultDocument(KotlinReactiveNoteCodec(
documentType = "kotlin.reactive_note.ui",
schemaVersion = "kotlin.reactive_note.v1",
@ -3580,8 +3639,8 @@ private fun cultCacheRawSnapshotsRoundTripThroughCultNetMessages() {
}
val reactiveCanonicalNote = cultDocument(KotlinReactiveNoteCodec(
documentType = "kotlin.reactive_note",
schemaVersion = "kotlin.reactive_note.v1",
documentType = "kotlin.reconcile_note",
schemaVersion = "kotlin.reconcile_note.v1",
))
target.cache.register(reactiveCanonicalNote)
target.remember(reactiveCanonicalNote, "note:reconcile", KotlinReactiveNote("initial", 1))

27
packages/cultmesh-py/src/cultmesh_py/node.py
View file

@ -26,7 +26,7 @@ from cultnet_py import (
apply_shard_log_response as apply_cultnet_shard_log_response,
document_delete,
document_put_raw,
resolve_document_and_schema_id_for_raw_record,
resolve_document_for_raw_record,
schema_document_map,
)
@ -689,17 +689,14 @@ class CultMeshDatabase:
schema_id = str(document_record.get("schemaId") or "")
documents_by_schema_id = schema_document_map(self.documents)
try:
document, resolved_schema_id = resolve_document_and_schema_id_for_raw_record(
documents_by_schema_id,
schema_id,
document_record,
)
document = resolve_document_for_raw_record(documents_by_schema_id, schema_id, document_record)
except KeyError:
return None
# One id: the local schema id, under which the cache stores the record.
return document, CultCacheEnvelope(
key=str(document_record["recordKey"]),
type=document.type,
schema_id=resolved_schema_id,
schema_id=document.catalog_entry().schema_id,
payload=bytes(document_record["payload"]),
stored_at=str(document_record.get("storedAt") or ""),
catalog_entry=document.catalog_entry(),
@ -1168,12 +1165,8 @@ class CultMeshDatabase:
documents_by_schema_id = schema_document_map(self.documents)
if raw_record is not None:
try:
_, resolved_schema_id = resolve_document_and_schema_id_for_raw_record(
documents_by_schema_id,
schema_id,
raw_record,
)
return resolved_schema_id, record_key
document = resolve_document_for_raw_record(documents_by_schema_id, schema_id, raw_record)
return document.catalog_entry().schema_id, record_key
except KeyError:
pass
document = documents_by_schema_id.get(schema_id)
@ -1333,15 +1326,11 @@ class CultMeshDatabase:
continue
schema_id = str(record.get("schemaId"))
try:
document, resolved_schema_id = resolve_document_and_schema_id_for_raw_record(
documents_by_schema_id,
schema_id,
record,
)
document = resolve_document_for_raw_record(documents_by_schema_id, schema_id, record)
except KeyError:
continue
record_key = str(record.get("recordKey"))
previous[(resolved_schema_id, record_key)] = self.cache.get(document, record_key)
previous[(document.catalog_entry().schema_id, record_key)] = self.cache.get(document, record_key)
return previous
@staticmethod

40
packages/cultmesh-py/tests/test_cultmesh.py
View file

@ -380,6 +380,46 @@ class CultMeshTests(unittest.TestCase):
self.assertEqual(observed[0].value["value"], "new")
self.assertEqual(node.cache.get_required_envelope(document, "policy:1").schema_id, local_schema_id)
# A record arriving under a compatible id or the schema name is the same record: one id, the local one, so an existing
# record is updated with its previous value.
def test_cultmesh_node_reports_an_update_when_a_record_arrives_under_a_compatible_id_or_the_schema_name(self) -> None:
document = define_database_entry_type(
"mesh.compatible-policy",
[("schema_version", 0), ("name", 1), ("value", 2)],
schema_id="mesh.compatible-policy.current",
schema_name="mesh.compatible_policy",
schema_version="mesh.compatible_policy.v1",
compatible_schema_ids=["mesh.compatible-policy.current", "mesh.compatible-policy.older"],
)
for wire_schema_id in ("mesh.compatible-policy.older", "mesh.compatible_policy"):
node = create_node(runtime_id="python-node")
node.register_document(document)
node.put(document, "policy:1", {"schema_version": "mesh.compatible_policy.v1", "name": "policy", "value": "old"})
observed: list[CultMeshDatabaseChange] = []
unsubscribe = node.database.watch_record(document, "policy:1", observed.append)
try:
applied = node.database.apply_snapshot_response({
"schemaVersion": "cultnet.snapshot_response_raw.v0",
"messageId": "compatible-snapshot",
"documents": [{
"schemaId": wire_schema_id,
"recordKey": "policy:1",
"storedAt": "2026-06-25T00:00:00Z",
"payloadEncoding": "messagepack",
"payload": document.encode_payload({
"schema_version": "mesh.compatible_policy.v1", "name": "policy", "value": "new",
}),
}],
})
finally:
unsubscribe()
local_schema_id = document.catalog_entry().schema_id
self.assertEqual(applied[0].schema_id, local_schema_id, wire_schema_id)
self.assertEqual([change.change_kind for change in observed], ["updated"], wire_schema_id)
self.assertEqual(observed[0].previous_value["value"], "old", wire_schema_id)
self.assertEqual(observed[0].value["value"], "new", wire_schema_id)
def test_cultmesh_node_syncs_snapshot_and_shard_log_through_cultnet_client(self) -> None:
import msgpack # type: ignore
from cultnet_py import read_frame, write_frame

2
packages/cultmesh-rs/src/lib.rs
View file

@ -62,7 +62,7 @@ macro_rules! cultmesh_documents {
$schema_version.to_string(),
$schema_version.to_string(),
),
);
)?;
)*
Ok(())
}

17
packages/cultmesh-ts/CHANGELOG.md
View file

@ -3,6 +3,23 @@
All notable changes to `cultmesh-ts` are recorded here, following
[Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
## Unreleased
### Breaking
- `asSchemaAlias` is removed from `CultMeshDocumentHandle`, `CultMeshCollectionHandle` and their bound forms. Read the handle by the schema you mean.
- `CultMeshDocumentCatalog` (`document`, `tryDocument`, `latest`, `replace`, `submitPrediction`, `authoritativeWriter`, `predictionWriter`, `watch`) and `CultMeshCollectionCatalog` (`collection`, `tryCollection`, `latest`, `watchChanges`) take `<TDocument = never, TSchema>` and yield `CultMeshCatalogValue<TDocument, TSchema>`: an explicit type argument wins, a CultCache document definition names its own value type, and a bare schema descriptor yields `unknown`. A value type is no longer inferred from a value or an expected result. The `parse` option is removed from these calls, and a lookup by type answers with that type's handle or none, not an alias of a compatible schema.
- Adding a document or collection to a catalog under a schema id or schema name another type already holds is refused with `SchemaConflictError`.
### Added
- `CultMeshCatalogValue<TDocument, TSchema>`, exported.
- `documentById` and `tryDocumentById` on `CultMeshDocumentCatalog`, which address one document by its document id when a catalog holds several documents of one type.
### Dependencies
- Needs the cultcache-ts release that ships with this change (its manifest range moves with that release): it imports `schemaIdentityOf` and `SchemaConflictError`, which 0.14.x does not export.
## [0.2.0]
### Added

42
packages/cultmesh-ts/README.md
View file

@ -87,16 +87,16 @@ await input.submitPrediction("pilot-a", predictedInput);
const stop = input.watch("pilot-a", latest => reconcileInput(latest));
```
Durable nodes resolve same-schema aliases at the boundary. Register the
canonical document that owns storage and replication, then ask the node for the
typed view the current runtime wants:
Durable nodes hand out typed handles for the documents they register. One type
owns each schema: a node refuses a definition it did not register, and refuses
to register a second type for a schema it already holds.
```ts
const station = await CultMesh.startNode(statePath, {
documents: [stationStockDocument],
documents: [stationStockDocument, daemonHealthDocument],
});
const stock = station.document(stationStockUiDocument, "station:starbridge:stock");
const stock = station.document(stationStockDocument, "station:starbridge:stock");
const current = await stock.latest();
await stock.authoritativeWriter().write(updatedStockFromUi);
@ -108,6 +108,20 @@ reactive.update(draft => {
});
```
`node.get`, `node.getRequired` and the value `node.put` resolves to return the
object the node's cache holds, not a copy: two reads of one record return the
same object. That object belongs to the cache and must not be mutated. Mutating
it changes what later reads see without writing anything to the store. To
change a document, build a new value (or use a writer) and put it:
```ts
const held = station.getRequired(stationStockDocument, "station:starbridge:stock");
await station.put(stationStockDocument, "station:starbridge:stock", {
...held,
availableMissiles: held.availableMissiles - 4,
});
```
State pointers are the same kind of managed surface for UI and tools. They can
advertise source documents, inherit a Verse route when bound, and resolve
through that Verse without caller-side context plumbing:
@ -386,14 +400,14 @@ const latestHealth = await health.latest();
const syncedHealth = await station.syncDocumentFromPeerSnapshot(
() => client,
daemonHealthUiDocument,
daemonHealthDocument,
"daemon:aetheria.health.v1",
{
timeoutMs: 5_000,
},
);
const localHealth = await station
.document(daemonHealthUiDocument, "daemon:aetheria.health.v1")
.document(daemonHealthDocument, "daemon:aetheria.health.v1")
.latest();
```
@ -409,11 +423,11 @@ before using the peer-card endpoint as a dial target.
snapshots. It hides the snapshot request message id, response listener, raw
payload binary normalization, and MessagePack decode behind the same
`CultMeshDocumentHandle` shape as local documents. The helper prefers an exact
schema id match and falls back to the requested record key, so runtimes that
alias the same logical document through different generated schema names can
still read the publication with one CultMesh call. The handle takes a CultCache
document definition, and its `latest()` result is read and parsed through that
definition: a payload is a `DocumentPayload`, which only
schema id match, else a record at the requested key under an id the
definition declares compatible; a record under any other schema id is another
type's and is never read in its place, whatever its payload parses as. The
handle takes a CultCache document definition, and its `latest()` result is read
and parsed through that definition: a payload is a `DocumentPayload`, which only
`decodeDocumentPayload(definition, payload)` reads, so no document is read
without the definition that says how its bytes are laid out.
@ -421,8 +435,8 @@ Use `node.syncDocumentFromPeerSnapshot(...)` or
`CultMesh.syncDocumentFromPeerSnapshot(...)` when a runtime should hydrate its
local node from the remote snapshot. The helper requests the raw snapshot,
applies it through the node's document registry, and returns the requested
typed definition. Same-schema aliases stay local after the call: the caller can
keep using `node.document(aliasDefinition, key)` and explicitly select
typed definition. The record then lives in the node: the caller can keep using
`node.document(definition, key)` and explicitly select
`authoritativeWriter()`, `predictionWriter()`, or `observe()`
without re-threading the RUDP peer.

541
packages/cultmesh-ts/src/index.ts
View file

@ -3,7 +3,9 @@ import { decodeWireValue, encodeWireValue } from "cultnet-ts/wire";
import {
CultCache,
SchemaConflictError,
SingleFileMessagePackBackingStore,
schemaIdentityOf,
type AnyCultCacheDocumentDefinition,
type CacheBackingStore,
type CultCacheEnvelope,
@ -207,6 +209,15 @@ export type CultMeshCollectionWatcher<TDocument> = (
callback: (change: CultMeshCollectionChange<TDocument>) => void,
) => CultMeshUnsubscribe;
/**
* The value type a catalog lookup yields. An explicit type argument wins; otherwise a CultCache document definition
* names its own value type, and a bare schema descriptor yields `unknown`. The value type is never inferred from a
* value or an expected result, so a wrong value is an error rather than a new type.
*/
export type CultMeshCatalogValue<TDocument, TSchema> = [TDocument] extends [never]
? TSchema extends AnyCultCacheDocumentDefinition ? CultCacheDocumentValue<TSchema> : unknown
: NoInfer<TDocument>;
export interface CultMeshDocumentSchemaDescriptor {
readonly type?: string;
readonly schemaId?: string;
@ -1019,38 +1030,6 @@ export class CultMeshDocumentHandle<TDocument> {
);
}
public asSchemaAlias<TAlias>(
schema: CultMeshDocumentSchemaDescriptor,
options: { parse?: (value: unknown) => TAlias } = {},
): CultMeshDocumentHandle<TAlias> {
const aliasSchema = normalizeCultMeshDocumentSchema(schema);
if (!cultMeshSchemasAreCompatible(this.schema, aliasSchema)) {
throw new Error(
`Document '${this.documentId}' schema ${cultMeshSchemaLabel(this.schema)} is not compatible with alias ${cultMeshSchemaLabel(aliasSchema)}.`,
);
}
const parse = options.parse ?? ((value: unknown) => value as TAlias);
return new CultMeshDocumentHandle<TAlias>(
this.documentId,
aliasSchema,
async (context) => parse(await this.latest(context)),
{
sources: this.sources,
routeHint: this.routeHint,
watchDocument: this.watchDocument
? (context, callback) => this.watch(context, value => callback(parse(value)))
: undefined,
replaceDocument: this.replaceDocument
? (context, value) => this.replace(context, value as unknown as TDocument)
: undefined,
submitPrediction: this.submitPredictionDocument
? (context, value) => this.submitPrediction(context, value as unknown as TDocument)
: undefined,
},
);
}
public bind(verse: CultMeshVerseContext | CultMeshVerse): CultMeshBoundDocumentHandle<TDocument> {
return cultMeshBindDocument(verse, this);
}
@ -1484,12 +1463,6 @@ export class CultMeshBoundDocumentHandle<TDocument> {
});
}
public asSchemaAlias<TAlias>(
schema: CultMeshDocumentSchemaDescriptor,
options: { parse?: (value: unknown) => TAlias } = {},
): CultMeshBoundDocumentHandle<TAlias> {
return cultMeshBindDocument(this.verse, this.document.asSchemaAlias(schema, options));
}
}
export class CultMeshDocumentCatalog {
@ -1509,6 +1482,7 @@ export class CultMeshDocumentCatalog {
}
public add<TDocument>(document: CultMeshDocumentHandle<TDocument>): this {
cultMeshRefuseSecondCatalogType(document.schema, this.#byType, this.#bySchemaId, this.#bySchemaNameVersion);
this.#byDocumentId.set(document.documentId, document as CultMeshDocumentHandle<any>);
if (document.schema.type) {
this.#byType.set(document.schema.type, document as CultMeshDocumentHandle<any>);
@ -1523,129 +1497,120 @@ export class CultMeshDocumentCatalog {
return this;
}
public tryDocument<TDocument>(
schema: CultMeshDocumentSchemaDescriptor,
options: { parse?: (value: unknown) => TDocument } = {},
): CultMeshDocumentHandle<TDocument> | undefined {
const descriptor = normalizeCultMeshDocumentSchema(schema);
const exact =
(descriptor.type ? this.#byType.get(descriptor.type) : undefined) ??
(descriptor.schemaId ? this.#bySchemaId.get(descriptor.schemaId) : undefined) ??
(() => {
const key = cultMeshSchemaNameVersionKey(descriptor);
return key ? this.#bySchemaNameVersion.get(key) : undefined;
})();
if (!exact) {
return undefined;
}
return cultMeshSchemasAreCompatible(exact.schema, descriptor)
? exact.asSchemaAlias(descriptor, options)
: undefined;
// A catalog may hold several documents of one type, one per document id. A lookup by schema answers with one of them;
// a lookup by document id addresses each.
public tryDocumentById<TDocument = unknown>(documentId: string): CultMeshDocumentHandle<TDocument> | undefined {
return this.#byDocumentId.get(documentId);
}
public document<TDocument>(
schema: CultMeshDocumentSchemaDescriptor,
options: { parse?: (value: unknown) => TDocument } = {},
): CultMeshDocumentHandle<TDocument> {
const document = this.tryDocument(schema, options);
public documentById<TDocument = unknown>(documentId: string): CultMeshDocumentHandle<TDocument> {
const document = this.tryDocumentById<TDocument>(documentId);
if (!document) {
throw new Error(`Document catalog has no document for ${cultMeshSchemaLabel(schema)}.`);
throw new Error(`Document catalog has no document with id '${documentId}'.`);
}
return document;
}
public latest<TDocument>(
schema: CultMeshDocumentSchemaDescriptor,
public tryDocument<TDocument = never, TSchema extends CultMeshDocumentSchemaDescriptor = CultMeshDocumentSchemaDescriptor>(
schema: TSchema,
): CultMeshDocumentHandle<CultMeshCatalogValue<TDocument, TSchema>> | undefined {
const descriptor = normalizeCultMeshDocumentSchema(schema);
// One type owns a schema, so a typed lookup is answered by that type's handle or not at all.
if (descriptor.type) {
return this.#byType.get(descriptor.type);
}
const key = cultMeshSchemaNameVersionKey(descriptor);
return (descriptor.schemaId ? this.#bySchemaId.get(descriptor.schemaId) : undefined) ??
(key ? this.#bySchemaNameVersion.get(key) : undefined);
}
public document<TDocument = never, TSchema extends CultMeshDocumentSchemaDescriptor = CultMeshDocumentSchemaDescriptor>(
schema: TSchema,
): CultMeshDocumentHandle<CultMeshCatalogValue<TDocument, TSchema>> {
return this.#require(schema);
}
public latest<TDocument = never, TSchema extends CultMeshDocumentSchemaDescriptor = CultMeshDocumentSchemaDescriptor>(
schema: TSchema,
context: CultMeshQueryContext | string = "local",
options: { parse?: (value: unknown) => TDocument } = {},
): Promise<TDocument> {
return this.document(schema, options).latest(context);
): Promise<CultMeshCatalogValue<TDocument, TSchema>> {
return this.#require(schema).latest(context);
}
public canReplace(schema: CultMeshDocumentSchemaDescriptor): boolean {
return this.tryDocument(schema)?.canReplace ?? false;
}
public replace<TDocument>(
schema: CultMeshDocumentSchemaDescriptor,
value: TDocument,
options?: { parse?: (value: unknown) => TDocument; context?: CultMeshQueryContext | string },
public replace<TDocument = never, TSchema extends CultMeshDocumentSchemaDescriptor = CultMeshDocumentSchemaDescriptor>(
schema: TSchema,
value: CultMeshCatalogValue<TDocument, TSchema>,
options?: { context?: CultMeshQueryContext | string },
): Promise<void>;
public replace<TDocument>(
schema: CultMeshDocumentSchemaDescriptor,
public replace<TDocument = never, TSchema extends CultMeshDocumentSchemaDescriptor = CultMeshDocumentSchemaDescriptor>(
schema: TSchema,
context: CultMeshQueryContext | string,
value: TDocument,
options?: { parse?: (value: unknown) => TDocument },
value: CultMeshCatalogValue<TDocument, TSchema>,
): Promise<void>;
public replace<TDocument>(
public replace(
schema: CultMeshDocumentSchemaDescriptor,
contextOrValue: CultMeshQueryContext | string | TDocument,
valueOrOptions?: TDocument | { parse?: (value: unknown) => TDocument; context?: CultMeshQueryContext | string },
maybeOptions: { parse?: (value: unknown) => TDocument } = {},
contextOrValue: unknown,
valueOrOptions?: unknown,
): Promise<void> {
const hasContext =
typeof contextOrValue === "string" || isCultMeshQueryContext(contextOrValue);
const context = hasContext
? contextOrValue as CultMeshQueryContext | string
: (valueOrOptions as { context?: CultMeshQueryContext | string } | undefined)?.context ?? "local";
const value = hasContext ? valueOrOptions as TDocument : contextOrValue as TDocument;
const options = hasContext ? maybeOptions : valueOrOptions as { parse?: (value: unknown) => TDocument } | undefined;
return this.document(schema, options).replace(context, value);
const [context, value] = cultMeshCatalogWriteArguments(contextOrValue, valueOrOptions);
return this.#require(schema).replace(context, value);
}
public canSubmitPrediction(schema: CultMeshDocumentSchemaDescriptor): boolean {
return this.tryDocument(schema)?.canSubmitPrediction ?? false;
}
public submitPrediction<TDocument>(
schema: CultMeshDocumentSchemaDescriptor,
value: TDocument,
options?: { parse?: (value: unknown) => TDocument; context?: CultMeshQueryContext | string },
public submitPrediction<TDocument = never, TSchema extends CultMeshDocumentSchemaDescriptor = CultMeshDocumentSchemaDescriptor>(
schema: TSchema,
value: CultMeshCatalogValue<TDocument, TSchema>,
options?: { context?: CultMeshQueryContext | string },
): Promise<void>;
public submitPrediction<TDocument>(
schema: CultMeshDocumentSchemaDescriptor,
public submitPrediction<TDocument = never, TSchema extends CultMeshDocumentSchemaDescriptor = CultMeshDocumentSchemaDescriptor>(
schema: TSchema,
context: CultMeshQueryContext | string,
value: TDocument,
options?: { parse?: (value: unknown) => TDocument },
value: CultMeshCatalogValue<TDocument, TSchema>,
): Promise<void>;
public submitPrediction<TDocument>(
public submitPrediction(
schema: CultMeshDocumentSchemaDescriptor,
contextOrValue: CultMeshQueryContext | string | TDocument,
valueOrOptions?: TDocument | { parse?: (value: unknown) => TDocument; context?: CultMeshQueryContext | string },
maybeOptions: { parse?: (value: unknown) => TDocument } = {},
contextOrValue: unknown,
valueOrOptions?: unknown,
): Promise<void> {
const hasContext =
typeof contextOrValue === "string" || isCultMeshQueryContext(contextOrValue);
const context = hasContext
? contextOrValue as CultMeshQueryContext | string
: (valueOrOptions as { context?: CultMeshQueryContext | string } | undefined)?.context ?? "local";
const value = hasContext ? valueOrOptions as TDocument : contextOrValue as TDocument;
const options = hasContext ? maybeOptions : valueOrOptions as { parse?: (value: unknown) => TDocument } | undefined;
return this.document(schema, options).submitPrediction(context, value);
const [context, value] = cultMeshCatalogWriteArguments(contextOrValue, valueOrOptions);
return this.#require(schema).submitPrediction(context, value);
}
public authoritativeWriter<TDocument extends object>(
schema: CultMeshDocumentSchemaDescriptor,
options: { parse?: (value: unknown) => TDocument; context?: CultMeshQueryContext | string } = {},
): CultMeshDocumentWriter<TDocument> {
return this.document(schema, options).authoritativeWriter(options.context ?? "local");
public authoritativeWriter<TDocument extends object = never, TSchema extends CultMeshDocumentSchemaDescriptor = CultMeshDocumentSchemaDescriptor>(
schema: TSchema,
options: { context?: CultMeshQueryContext | string } = {},
): CultMeshDocumentWriter<CultMeshCatalogValue<TDocument, TSchema> & object> {
return this.#require(schema).authoritativeWriter(options.context ?? "local");
}
public predictionWriter<TDocument extends object>(
schema: CultMeshDocumentSchemaDescriptor,
options: { parse?: (value: unknown) => TDocument; context?: CultMeshQueryContext | string } = {},
): CultMeshDocumentWriter<TDocument> {
return this.document(schema, options).predictionWriter(options.context ?? "local");
public predictionWriter<TDocument extends object = never, TSchema extends CultMeshDocumentSchemaDescriptor = CultMeshDocumentSchemaDescriptor>(
schema: TSchema,
options: { context?: CultMeshQueryContext | string } = {},
): CultMeshDocumentWriter<CultMeshCatalogValue<TDocument, TSchema> & object> {
return this.#require(schema).predictionWriter(options.context ?? "local");
}
public watch<TDocument>(
schema: CultMeshDocumentSchemaDescriptor,
callback: (value: TDocument) => void,
options: { parse?: (value: unknown) => TDocument; context?: CultMeshQueryContext | string } = {},
public watch<TDocument = never, TSchema extends CultMeshDocumentSchemaDescriptor = CultMeshDocumentSchemaDescriptor>(
schema: TSchema,
callback: (value: CultMeshCatalogValue<TDocument, TSchema>) => void,
options: { context?: CultMeshQueryContext | string } = {},
): CultMeshUnsubscribe {
return this.document(schema, options).watch(options.context ?? "local", callback);
return this.#require(schema).watch(options.context ?? "local", callback);
}
#require(schema: CultMeshDocumentSchemaDescriptor): CultMeshDocumentHandle<any> {
const document = this.tryDocument(schema);
if (!document) {
throw new Error(`Document catalog has no document for ${cultMeshSchemaLabel(schema)}.`);
}
return document;
}
}
@ -1711,35 +1676,6 @@ export class CultMeshCollectionHandle<TDocument> {
return this.watchCollection(this.resolveContext(context), callback);
}
public asSchemaAlias<TAlias>(
schema: CultMeshDocumentSchemaDescriptor,
options: { parse?: (value: unknown) => TAlias } = {},
): CultMeshCollectionHandle<TAlias> {
const aliasSchema = normalizeCultMeshDocumentSchema(schema);
if (!cultMeshSchemasAreCompatible(this.schema, aliasSchema)) {
throw new Error(
`Collection '${this.collectionId}' schema ${cultMeshSchemaLabel(this.schema)} is not compatible with alias ${cultMeshSchemaLabel(aliasSchema)}.`,
);
}
const parse = options.parse ?? ((value: unknown) => value as TAlias);
return new CultMeshCollectionHandle<TAlias>(
this.collectionId,
aliasSchema,
async (context) => (await this.latest(context)).map(value => parse(value)),
{
sources: this.sources,
routeHint: this.routeHint,
watchCollection: this.watchCollection
? (context, callback) => this.watchChanges(context, change => callback({
...change,
value: change.value === undefined ? undefined : parse(change.value),
}))
: undefined,
},
);
}
public bind(verse: CultMeshVerseContext | CultMeshVerse): CultMeshBoundCollectionHandle<TDocument> {
return cultMeshBindCollection(verse, this);
}
@ -1772,6 +1708,7 @@ export class CultMeshCollectionCatalog {
}
public add<TDocument>(collection: CultMeshCollectionHandle<TDocument>): this {
cultMeshRefuseSecondCatalogType(collection.schema, this.#byType, this.#bySchemaId, this.#bySchemaNameVersion);
this.#byCollectionId.set(collection.collectionId, collection as CultMeshCollectionHandle<any>);
if (collection.schema.type) {
this.#byType.set(collection.schema.type, collection as CultMeshCollectionHandle<any>);
@ -1786,54 +1723,47 @@ export class CultMeshCollectionCatalog {
return this;
}
public tryCollection<TDocument>(
schema: CultMeshDocumentSchemaDescriptor,
options: { parse?: (value: unknown) => TDocument } = {},
): CultMeshCollectionHandle<TDocument> | undefined {
public tryCollection<TDocument = never, TSchema extends CultMeshDocumentSchemaDescriptor = CultMeshDocumentSchemaDescriptor>(
schema: TSchema,
): CultMeshCollectionHandle<CultMeshCatalogValue<TDocument, TSchema>> | undefined {
const descriptor = normalizeCultMeshDocumentSchema(schema);
const exact =
(descriptor.type ? this.#byType.get(descriptor.type) : undefined) ??
(descriptor.schemaId ? this.#bySchemaId.get(descriptor.schemaId) : undefined) ??
(() => {
const key = cultMeshSchemaNameVersionKey(descriptor);
return key ? this.#bySchemaNameVersion.get(key) : undefined;
})();
if (!exact) {
return undefined;
// One type owns a schema, so a typed lookup is answered by that type's handle or not at all.
if (descriptor.type) {
return this.#byType.get(descriptor.type);
}
return cultMeshSchemasAreCompatible(exact.schema, descriptor)
? exact.asSchemaAlias(descriptor, options)
: undefined;
const key = cultMeshSchemaNameVersionKey(descriptor);
return (descriptor.schemaId ? this.#bySchemaId.get(descriptor.schemaId) : undefined) ??
(key ? this.#bySchemaNameVersion.get(key) : undefined);
}
public collection<TDocument>(
schema: CultMeshDocumentSchemaDescriptor,
options: { parse?: (value: unknown) => TDocument } = {},
): CultMeshCollectionHandle<TDocument> {
const collection = this.tryCollection(schema, options);
public collection<TDocument = never, TSchema extends CultMeshDocumentSchemaDescriptor = CultMeshDocumentSchemaDescriptor>(
schema: TSchema,
): CultMeshCollectionHandle<CultMeshCatalogValue<TDocument, TSchema>> {
return this.#require(schema);
}
public latest<TDocument = never, TSchema extends CultMeshDocumentSchemaDescriptor = CultMeshDocumentSchemaDescriptor>(
schema: TSchema,
context: CultMeshQueryContext | string = "local",
): Promise<CultMeshCollectionSnapshot<CultMeshCatalogValue<TDocument, TSchema>>> {
return this.#require(schema).latest(context);
}
public watchChanges<TDocument = never, TSchema extends CultMeshDocumentSchemaDescriptor = CultMeshDocumentSchemaDescriptor>(
schema: TSchema,
callback: (change: CultMeshCollectionChange<CultMeshCatalogValue<TDocument, TSchema>>) => void,
options: { context?: CultMeshQueryContext | string } = {},
): CultMeshUnsubscribe {
return this.#require(schema).watchChanges(options.context ?? "local", callback);
}
#require(schema: CultMeshDocumentSchemaDescriptor): CultMeshCollectionHandle<any> {
const collection = this.tryCollection(schema);
if (!collection) {
throw new Error(`Collection catalog has no collection for ${cultMeshSchemaLabel(schema)}.`);
}
return collection;
}
public latest<TDocument>(
schema: CultMeshDocumentSchemaDescriptor,
context: CultMeshQueryContext | string = "local",
options: { parse?: (value: unknown) => TDocument } = {},
): Promise<CultMeshCollectionSnapshot<TDocument>> {
return this.collection(schema, options).latest(context);
}
public watchChanges<TDocument>(
schema: CultMeshDocumentSchemaDescriptor,
callback: (change: CultMeshCollectionChange<TDocument>) => void,
options: { parse?: (value: unknown) => TDocument; context?: CultMeshQueryContext | string } = {},
): CultMeshUnsubscribe {
return this.collection(schema, options).watchChanges(options.context ?? "local", callback);
}
}
export class CultMeshBoundCollectionHandle<TDocument> {
@ -1858,12 +1788,6 @@ export class CultMeshBoundCollectionHandle<TDocument> {
return this.collection.watchChanges(cultMeshQueryContextFromVerse(this.verse), callback);
}
public asSchemaAlias<TAlias>(
schema: CultMeshDocumentSchemaDescriptor,
options: { parse?: (value: unknown) => TAlias } = {},
): CultMeshBoundCollectionHandle<TAlias> {
return cultMeshBindCollection(this.verse, this.collection.asSchemaAlias(schema, options));
}
}
export class CultMeshStatePointer<T> {
@ -2415,6 +2339,17 @@ function resolveCultMeshVerseContext(verse: CultMeshVerseContext | CultMeshVerse
return verse instanceof CultMeshVerse ? verse.context : verse;
}
// A catalog write takes (value, options?) or (context, value); both resolve to one context and one value.
function cultMeshCatalogWriteArguments(
contextOrValue: unknown,
valueOrOptions: unknown,
): [CultMeshQueryContext | string, unknown] {
if (typeof contextOrValue === "string" || isCultMeshQueryContext(contextOrValue)) {
return [contextOrValue, valueOrOptions];
}
return [(valueOrOptions as { context?: CultMeshQueryContext | string } | undefined)?.context ?? "local", contextOrValue];
}
function isCultMeshQueryContext(value: unknown): value is CultMeshQueryContext {
if (typeof value !== "object" || value === null) {
return false;
@ -2672,19 +2607,11 @@ export function cultMeshDocumentFromPeerSnapshot<TDefinition extends AnyCultCach
const read = async (): Promise<CultCacheDocumentValue<TDefinition>> => {
const record = await requestCultNetRawSnapshotDocument(
peer,
resolvedSchemaId,
cultMeshAcceptedSchemaIds(definition),
recordKey,
{
timeoutMs: options.timeoutMs,
messageIdPrefix: options.messageIdPrefix ?? documentId,
accepts: candidate => {
try {
decodeDocumentPayload(definition, candidate.payload);
return true;
} catch {
return false;
}
},
},
);
return decodeDocumentPayload(definition, record.payload);
@ -3872,11 +3799,7 @@ export class CultMeshNode {
definition: TDefinition,
key: string,
): CultCacheDocumentValue<TDefinition> | undefined {
const registered = this.resolveCacheDefinition(definition);
const value = this.cache.get(registered, key);
return value === undefined
? undefined
: this.parseDocumentValue(definition, value);
return this.cache.get(definition, key);
}
public getRequired<TDefinition extends AnyCultCacheDocumentDefinition>(
@ -3896,17 +3819,14 @@ export class CultMeshNode {
key: string,
value: CultCacheDocumentValue<TDefinition>,
): Promise<CultCacheDocumentValue<TDefinition>> {
const registered = this.resolveCacheDefinition(definition);
return this.cache
.put(registered, key, this.parseDocumentValue(definition, value) as never)
.then(stored => this.parseDocumentValue(definition, stored));
return this.cache.put(definition, key, value);
}
public delete<TDefinition extends AnyCultCacheDocumentDefinition>(
definition: TDefinition,
key: string,
): Promise<boolean> {
return this.cache.delete(this.resolveCacheDefinition(definition), key);
return this.cache.delete(definition, key);
}
public document<TDefinition extends AnyCultCacheDocumentDefinition>(
@ -3919,15 +3839,7 @@ export class CultMeshNode {
} = {},
): CultMeshDocumentHandle<CultCacheDocumentValue<TDefinition>> {
const documentId = options.documentId ?? `${definition.type}:${key}`;
return this.asRequestedDocument(
cultMeshDocumentFromCache(
this.cache,
this.resolveCacheDefinition(definition),
key,
{ ...options, documentId },
),
definition,
);
return cultMeshDocumentFromCache(this.cache, definition, key, { ...options, documentId });
}
public async syncDocumentFromPeerSnapshot<TDefinition extends AnyCultCacheDocumentDefinition>(
@ -3939,7 +3851,6 @@ export class CultMeshNode {
messageIdPrefix?: string;
} = {},
): Promise<CultCacheDocumentValue<TDefinition>> {
const registered = this.resolveCacheDefinition(definition);
const schema = cultMeshSchemaFromDefinition(definition);
const schemaId = schema.schemaId ?? schema.type;
if (!schemaId) {
@ -3947,19 +3858,11 @@ export class CultMeshNode {
}
const document = await requestCultNetRawSnapshotDocument(
peer,
schemaId,
[schemaId],
key,
{
timeoutMs: options.timeoutMs,
messageIdPrefix: options.messageIdPrefix ?? key,
accepts: record => {
try {
decodeDocumentPayload(definition, record.payload);
return true;
} catch {
return false;
}
},
},
);
await this.documents.applyRawDocumentPutMessage(this.cache, {
@ -3967,7 +3870,7 @@ export class CultMeshNode {
messageId: options.messageIdPrefix ?? `sync:${key}`,
document,
});
return this.parseDocumentValue(definition, this.cache.getRequired(registered, key));
return this.cache.getRequired(definition, key);
}
public syncDocumentFromPublication<TDefinition extends AnyCultCacheDocumentDefinition>(
@ -4004,14 +3907,7 @@ export class CultMeshNode {
} = {},
): CultMeshDocumentHandle<CultCacheDocumentValue<TDefinition>> {
const documentId = options.documentId ?? `${definition.type}:global`;
return this.asRequestedDocument(
cultMeshGlobalDocumentFromCache(
this.cache,
this.resolveCacheDefinition(definition),
{ ...options, documentId },
),
definition,
);
return cultMeshGlobalDocumentFromCache(this.cache, definition, { ...options, documentId });
}
public collection<TDefinition extends AnyCultCacheDocumentDefinition>(
@ -4023,46 +3919,12 @@ export class CultMeshNode {
} = {},
): CultMeshCollectionHandle<CultCacheDocumentValue<TDefinition>> {
const collectionId = options.collectionId ?? definition.type;
return cultMeshCollectionFromCache(
this.cache,
this.resolveCacheDefinition(definition),
{ ...options, collectionId },
).asSchemaAlias<CultCacheDocumentValue<TDefinition>>(
cultMeshSchemaFromDefinition(definition),
{ parse: value => this.parseDocumentValue(definition, value) },
);
return cultMeshCollectionFromCache(this.cache, definition, { ...options, collectionId });
}
public async flush(soft = false): Promise<void> {
await this.store.pushAll?.(this.cache.snapshot(), { soft });
}
private resolveCacheDefinition<TDefinition extends AnyCultCacheDocumentDefinition>(
definition: TDefinition,
): AnyCultCacheDocumentDefinition {
const schema = cultMeshSchemaFromDefinition(definition);
return (
this.documents.get(definition.type) ??
(schema.schemaId ? this.documents.getBySchemaId(schema.schemaId) : undefined)
)?.definition ?? definition;
}
private asRequestedDocument<TDefinition extends AnyCultCacheDocumentDefinition>(
document: CultMeshDocumentHandle<unknown>,
definition: TDefinition,
): CultMeshDocumentHandle<CultCacheDocumentValue<TDefinition>> {
return document.asSchemaAlias<CultCacheDocumentValue<TDefinition>>(
cultMeshSchemaFromDefinition(definition),
{ parse: value => this.parseDocumentValue(definition, value) },
);
}
private parseDocumentValue<TDefinition extends AnyCultCacheDocumentDefinition>(
definition: TDefinition,
value: unknown,
): CultCacheDocumentValue<TDefinition> {
return definition.schema.parse(value) as CultCacheDocumentValue<TDefinition>;
}
}
export class CultMeshVerseCatalog<TVerse = unknown> {
@ -6281,12 +6143,8 @@ export class CultMesh {
function cultMeshSchemaFromDefinition(
definition: AnyCultCacheDocumentDefinition,
): CultMeshDocumentSchemaDescriptor {
return {
type: definition.type,
schemaId: definition.schemaId ?? definition.type,
schemaName: definition.schemaName,
schemaVersion: definition.schemaVersion,
};
const { schemaId, schemaName, schemaVersion } = schemaIdentityOf(definition);
return { type: definition.type, schemaId, schemaName, schemaVersion };
}
function normalizeCultMeshDocumentSchema(
@ -6308,6 +6166,32 @@ function normalizeCultMeshDocumentSchema(
return normalized;
}
// One type owns a schema in a catalog, as in a registry. A handle whose type differs from the one already holding its
// schema id or schema name and version, or whose type already holds another schema, is refused. Handles of one type
// for one schema may repeat (one per record), and a handle without a type claims no type.
function cultMeshRefuseSecondCatalogType(
schema: CultMeshDocumentSchemaDescriptor,
byType: ReadonlyMap<string, { readonly schema: CultMeshDocumentSchemaDescriptor }>,
bySchemaId: ReadonlyMap<string, { readonly schema: CultMeshDocumentSchemaDescriptor }>,
bySchemaNameVersion: ReadonlyMap<string, { readonly schema: CultMeshDocumentSchemaDescriptor }>,
): void {
if (!schema.type) {
return;
}
const key = cultMeshSchemaNameVersionKey(schema);
const sameType = byType.get(schema.type)?.schema;
const holder =
(sameType?.schemaId && schema.schemaId && sameType.schemaId !== schema.schemaId ? sameType : undefined) ??
[schema.schemaId ? bySchemaId.get(schema.schemaId) : undefined, key ? bySchemaNameVersion.get(key) : undefined]
.map(held => held?.schema)
.find(held => held?.type !== undefined && held.type !== schema.type);
if (holder) {
const name = (held: CultMeshDocumentSchemaDescriptor) => held.schemaName ?? held.type ?? held.schemaId ?? "";
throw new SchemaConflictError(schema.schemaId ?? schema.type, [name(holder), name(schema)], "",
`CultMesh catalog schema ${cultMeshSchemaLabel(schema)} is held by type "${holder.type}" and cannot also be held by type "${schema.type}".`);
}
}
function cultMeshSchemaNameVersionKey(
schema: CultMeshDocumentSchemaDescriptor,
): string | undefined {
@ -6316,26 +6200,6 @@ function cultMeshSchemaNameVersionKey(
: undefined;
}
function cultMeshSchemasAreCompatible(
left: CultMeshDocumentSchemaDescriptor,
right: CultMeshDocumentSchemaDescriptor,
): boolean {
if (left.schemaId && right.schemaId && left.schemaId === right.schemaId) {
return true;
}
if (
left.schemaName &&
right.schemaName &&
left.schemaName === right.schemaName &&
(left.schemaVersion ?? "") === (right.schemaVersion ?? "")
) {
return true;
}
return Boolean(left.type && right.type && left.type === right.type);
}
function inferCultMeshSchemaName(schemaId: string | undefined): string | undefined {
if (!schemaId) {
return undefined;
@ -6348,53 +6212,23 @@ function inferCultMeshSchemaName(schemaId: string | undefined): string | undefin
return /^\d+$/.test(version) ? schemaId.slice(0, marker) : undefined;
}
// The record a store document reads: the one under the handle's schema id, else one under an id the definition lists as
// compatible, as CultCache resolves it. A record under any other id is not this document, whatever
// its payload parses as.
function resolveCultMeshStoreDocumentRecord(
records: readonly CultCacheEnvelope[],
schema: CultMeshDocumentSchemaDescriptor,
definition: AnyCultCacheDocumentDefinition,
): CultCacheEnvelope | undefined {
const schemaId = schema.schemaId ?? schema.type;
return (
records.find(candidate => schemaId !== undefined && candidate.schemaId === schemaId) ??
records.find(candidate =>
cultMeshEnvelopeMatchesSchema(candidate, schema) && cultMeshEnvelopeParsesAsDefinition(candidate, definition)) ??
records.find(candidate => cultMeshEnvelopeParsesAsDefinition(candidate, definition))
);
const own = schema.schemaId ?? schema.type;
const accepted = [own, ...cultMeshAcceptedSchemaIds(definition)];
return records.find(candidate => candidate.schemaId === own)
?? records.find(candidate => candidate.schemaId !== undefined && accepted.includes(candidate.schemaId));
}
function cultMeshEnvelopeMatchesSchema(
envelope: CultCacheEnvelope,
schema: CultMeshDocumentSchemaDescriptor,
): boolean {
const entry = envelope.catalogEntry;
if (!entry) {
return false;
}
if (
schema.schemaId &&
(entry.schemaId === schema.schemaId ||
envelope.schemaId === schema.schemaId ||
entry.compatibleSchemaIds?.includes(schema.schemaId))
) {
return true;
}
return Boolean(
schema.schemaName &&
entry.schemaName === schema.schemaName &&
(schema.schemaVersion ?? "") === (entry.schemaVersion ?? ""),
);
}
function cultMeshEnvelopeParsesAsDefinition(
envelope: CultCacheEnvelope,
definition: AnyCultCacheDocumentDefinition,
): boolean {
try {
decodeDocumentPayload(definition, envelope.payload);
return true;
} catch {
return false;
}
// The ids a definition's records may be stored under: its own first, then the ids it declares compatible.
function cultMeshAcceptedSchemaIds(definition: AnyCultCacheDocumentDefinition): readonly string[] {
return schemaIdentityOf(definition).compatibleSchemaIds;
}
function cultMeshSchemaLabel(schema: CultMeshDocumentSchemaDescriptor): string {
@ -6441,14 +6275,15 @@ function normalizeRudpDocumentPut(
};
}
// The peer's record at the key under the first schema id, else under another id in the list. A record at the key under any
// other id is another type's, and is never taken in its place, whatever its payload parses as; none is not-found.
async function requestCultNetRawSnapshotDocument(
peerOrProvider: CultNetPeer | (() => CultNetPeer | Promise<CultNetPeer>),
schemaId: string,
schemaIds: readonly string[],
recordKey: string,
options: {
timeoutMs?: number;
messageIdPrefix?: string;
accepts?: (document: CultNetRawDocumentRecord) => boolean;
} = {},
): Promise<CultNetRawDocumentRecord> {
const peer = await resolveCultNetPeer(peerOrProvider);
@ -6457,13 +6292,11 @@ async function requestCultNetRawSnapshotDocument(
messageId,
timeoutMs: options.timeoutMs,
});
const candidates = response.documents.filter(candidate => candidate.recordKey === recordKey);
const document =
candidates.find(candidate => candidate.schemaId === schemaId) ??
(options.accepts ? candidates.find(options.accepts) : undefined) ??
candidates[0];
const atKey = response.documents.filter(candidate => candidate.recordKey === recordKey);
const document = atKey.find(candidate => candidate.schemaId === schemaIds[0])
?? atKey.find(candidate => schemaIds.includes(candidate.schemaId));
if (!document) {
throw new Error(`CultMesh peer snapshot did not return ${schemaId} at ${recordKey}.`);
throw new Error(`CultMesh peer snapshot did not return ${schemaIds[0]} at ${recordKey}.`);
}
return document;
}

56
packages/cultmesh-ts/test/codec-documents.test.ts
View file

@ -15,13 +15,12 @@ import {
documentPayloadFromBytes,
encodeDocumentPayload,
} from "@gamecult/cultcache-ts";
import type { CacheBackingStore, CultCacheEnvelope } from "@gamecult/cultcache-ts";
import {
CultNetDocumentRegistry,
defineCultNetDocumentBinding,
type CultNetPeer,
} from "cultnet-ts";
import { CultMesh, cultMeshDocumentFromSingleFile, cultMeshDocumentFromStore, type CultMeshRudpDocumentPut } from "../src/index";
import { CultMesh, cultMeshDocumentFromSingleFile, type CultMeshRudpDocumentPut } from "../src/index";
// A document written in the reference's positional shape, and a plain map document beside it. Every mesh reader of a
// payload reads it the way the cache does, so both read through the mesh.
@ -47,15 +46,6 @@ const codecReceipt = defineDocumentType({
}, "Tests.CodecReceipt"),
});
// The same record under another schema id: the store holds it as codec_note, so it is found by parsing, not by id.
const codecNoteAlias = defineDocumentType({
type: "cultmesh.codec_note_alias",
schemaId: "cultmesh.codec_note_alias.v0",
schema: z.object({ noteId: z.string(), body: z.string() }),
name: "noteId",
codec: codecNote.codec,
});
const plainNote = defineDocumentType({
type: "cultmesh.plain_note",
schemaId: "cultmesh.plain_note.v0",
@ -73,17 +63,6 @@ test("the mesh reads a stored codec document as the cache does, and a plain docu
await plainNode.put(plainNote, "n2", { noteId: "n2", body: "map" });
assert.deepEqual(await cultMeshDocumentFromSingleFile(codecPath, codecNote).latest(), { noteId: "n1", body: "positional" });
assert.deepEqual(await cultMeshDocumentFromSingleFile(codecPath, codecNoteAlias).latest(), { noteId: "n1", body: "positional" });
// A store holding a plain record and a codec record, the alias found by parsing takes the codec one whichever the
// single-file store (which sorts records by key) puts first: the probe has to refuse the plain record, not meet it
// second.
for (const [plainKey, codecKey] of [["a-plain", "z-codec"], ["z-plain", "a-codec"]] as const) {
const bothPath = join(dir, `both-${plainKey}.ccmp`);
const bothNode = await CultMesh.startNode(bothPath, { documents: [plainNote, codecNote] });
await bothNode.put(plainNote, plainKey, { noteId: plainKey, body: "map" });
await bothNode.put(codecNote, codecKey, { noteId: codecKey, body: "positional" });
assert.deepEqual(await cultMeshDocumentFromSingleFile(bothPath, codecNoteAlias).latest(), { noteId: codecKey, body: "positional" });
}
assert.deepEqual(await cultMeshDocumentFromSingleFile(plainPath, plainNote).latest(), { noteId: "n2", body: "map" });
});
@ -92,10 +71,6 @@ test("the mesh syncs a codec document from a peer snapshot and returns a codec r
const dir = await mkdtemp(join(tmpdir(), "cultmesh-ts-codec-sync-"));
const source = await CultMesh.startNode(join(dir, "source.ccmp"), { documents: [codecNote, plainNote] });
await source.put(codecNote, "n1", { noteId: "n1", body: "synced positional" });
// Two records at one key, in two shapes: a peer snapshot handle for the alias schema (which neither holds) accepts the
// one that parses as the codec.
await source.put(plainNote, "n3", { noteId: "n3", body: "map" });
await source.put(codecNote, "n3", { noteId: "n3", body: "positional" });
const target = await CultMesh.startNode(join(dir, "target.ccmp"), { documents: [codecNote] });
const commandBinding = defineCultNetDocumentBinding({ definition: codecNote });
const plainBinding = defineCultNetDocumentBinding({ definition: plainNote });
@ -117,18 +92,12 @@ test("the mesh syncs a codec document from a peer snapshot and returns a codec r
peer = await CultMesh.createRudpPeer("cultmesh-ts-codec-client", connectionId, `rudp://127.0.0.1:${server.bind.port}`, {
resendDelayMs: 25, resendPollMs: 5, connectTimeoutMs: 1_000,
});
// The accepts probe reads the snapshot's payload as the codec: a plain decode would reject every codec snapshot.
const synced = await target.syncDocumentFromPeerSnapshot(peer, codecNote, "n1", { timeoutMs: 1_000 });
assert.deepEqual(synced, { noteId: "n1", body: "synced positional" });
assert.deepEqual(target.getRequired(codecNote, "n1"), { noteId: "n1", body: "synced positional" });
// A peer snapshot handle reads the same way, and accepts the snapshot by parsing it as the codec.
// A peer snapshot handle reads the same way.
assert.deepEqual(await CultMesh.documentFromPeerSnapshot(peer, codecNote, "n1", { timeoutMs: 1_000 }).latest(), { noteId: "n1", body: "synced positional" });
assert.deepEqual(
await CultMesh.documentFromPeerSnapshot(peer, codecNoteAlias, "n3", { timeoutMs: 1_000 }).latest(),
{ noteId: "n3", body: "positional" },
);
const receipt = await CultMesh.publishRudpDocumentAndWaitForReceipt(
"cultmesh-ts-codec-receipt", connectionId, `rudp://127.0.0.1:${server.bind.port}`,
commandBinding, "n2", { noteId: "n2", body: "command" }, receiptBinding,
@ -192,27 +161,6 @@ test("a RUDP put naming a schema the server has no definition for reaches the ha
}
});
test("a store read takes the record the catalog matches to the schema and the definition parses, not the first that merely parses", async () => {
const target = defineDocumentType({ type: "cultmesh.pick", schemaId: "cultmesh.pick.v1", schema: z.object({ n: z.number() }) });
const lenient = defineDocumentType({ type: "cultmesh.pick", schemaId: "cultmesh.pick.v1", schema: { parse: (value: unknown) => value } });
const entry = (schemaId: string, compatibleSchemaIds?: readonly string[]) => ({
schemaId, schemaName: schemaId, schemaVersion: `${schemaId}.v1`, contentHash: "h", canonicalSchemaJson: "{}", compatibleSchemaIds,
});
const envelope = (key: string, schemaId: string, value: unknown, compatibleSchemaIds?: readonly string[]): CultCacheEnvelope => ({
key, type: "cultmesh.pick", schemaId, storedAt: "2026-10-11T00:00:00.000Z",
payload: encodeDocumentPayload(lenient, value), catalogEntry: entry(schemaId, compatibleSchemaIds),
});
// None carries the definition's own schema id, so the exact-id find misses. The first parses and is foreign; the
// second is matched by the catalog and does not parse; the third is matched and parses.
const records = [
envelope("a", "foreign.v1", { n: 1 }),
envelope("b", "alias-bad.v1", { n: "x" }, ["cultmesh.pick.v1"]),
envelope("c", "alias-good.v1", { n: 2 }, ["cultmesh.pick.v1"]),
];
const store: CacheBackingStore = { pullAll: async () => records, push: async () => undefined, delete: async () => undefined };
assert.deepEqual(await cultMeshDocumentFromStore(store, target).latest(), { n: 2 });
});
// A payload nested past the limit: MAX_PAYLOAD_DEPTH + 1 one-element arrays around a nil.
const pastLimitPayload = () => documentPayloadFromBytes(Uint8Array.from([...new Array(MAX_PAYLOAD_DEPTH + 1).fill(0x91), 0xc0]));

503
packages/cultmesh-ts/test/cultmesh.test.ts
View file

@ -6,7 +6,7 @@ import test from "node:test";
import assert from "node:assert/strict";
import { decodeWireValue, encodeWireValue } from "cultnet-ts/wire";
import { z } from "zod";
import { defineDocumentType, documentPayloadBytes, documentPayloadFromBytes } from "@gamecult/cultcache-ts";
import { SchemaConflictError, defineDocumentType, documentPayloadBytes, documentPayloadFromBytes } from "@gamecult/cultcache-ts";
import {
CultNetDocumentRegistry,
CultNetPeer,
@ -35,7 +35,11 @@ const noteDocument = defineDocumentType({
name: "noteId",
});
const noteAliasDocument = defineDocumentType({
type Note = z.infer<typeof noteDocument.schema>;
type CultMeshDocumentSchemaDescriptorLike = { type?: string; schemaId?: string; schemaName?: string; schemaVersion?: string };
// Another type claiming noteDocument's schema. It is never registered beside noteDocument: one type owns a schema.
const otherTypeForNoteSchema = defineDocumentType({
type: "cultmesh.note.ui",
schemaId: "cultmesh.note.v0",
schema: z.object({
@ -134,36 +138,142 @@ test("CultMesh TS document handles hide local cache plumbing behind typed reacti
await waitFor(() => observed.includes("updated"), "document handle update");
unsubscribe();
const alias = bound.asSchemaAlias(noteAliasDocument, {
parse: value => noteAliasDocument.schema.parse(value),
});
assert.equal((await alias.latest()).body, "updated");
const catalog = CultMesh.documents(document);
assert.equal(catalog.canReplace(noteAliasDocument), true);
await catalog.authoritativeWriter(noteAliasDocument, {
parse: value => noteAliasDocument.schema.parse(value),
assert.equal(catalog.canReplace(noteDocument), true);
await catalog.authoritativeWriter(noteDocument, {
context: "browser-client",
}).write({
noteId: "note:1",
body: "catalog-updated",
});
assert.equal(
(await catalog.latest(noteAliasDocument, "browser-client", {
parse: value => noteAliasDocument.schema.parse(value),
})).body,
(await catalog.latest(noteDocument, "browser-client")).body,
"catalog-updated",
);
// A typed lookup is answered by its own type's handle; a handle of another type for the same schema is not it.
assert.equal(catalog.tryDocument(otherTypeForNoteSchema), undefined);
assert.equal(catalog.canReplace(otherTypeForNoteSchema), false);
assert.throws(() => catalog.document(otherTypeForNoteSchema), /no document for/);
const incremented = await bound.authoritativeWriter().update(value => ({
...value,
body: `${value.body}:updated-through-bound-set`,
}));
assert.equal(incremented.body, "catalog-updated:updated-through-bound-set");
assert.equal((await document.latest()).body, "catalog-updated:updated-through-bound-set");
assert.throws(
() => document.asSchemaAlias({ schemaId: "cultmesh.other.v0" }),
/not compatible/,
);
});
test("CultMesh TS catalogs infer a lookup's value type from its document definition", async () => {
const documents = CultMesh.documents(CultMesh.document("note:typed", noteDocument, async () => ({ noteId: "note:typed", body: "typed" })));
const collections = CultMesh.collections(CultMesh.collection("notes:typed", noteDocument, async () => [{ noteId: "note:typed", body: "typed" }]));
const inferred: Note = await documents.latest(noteDocument);
const inferredRows: readonly Note[] = await collections.latest(noteDocument);
assert.equal(inferred.body, "typed");
assert.equal(inferredRows[0]?.body, "typed");
// @ts-expect-error a definition names its value type: body is a string
const wrong: number = (await documents.latest(noteDocument)).body;
// @ts-expect-error a collection lookup infers its rows the same way
const wrongRows: readonly number[] = await collections.latest(noteDocument);
// @ts-expect-error a bare schema descriptor yields unknown
void (await documents.latest({ schemaId: "cultmesh.note.v0" })).body;
const explicit: { body: string } = await documents.latest<{ body: string }>({ schemaId: "cultmesh.note.v0" });
assert.equal(explicit.body, "typed");
void wrong;
void wrongRows;
// Writes are checked against the definition's value type. Never called: the handle is read-only.
const typeOnly = () => {
// @ts-expect-error replace checks the value against the definition
void documents.replace(noteDocument, { noteId: "note:typed", body: 1 });
// @ts-expect-error so does a prediction, with a context
void documents.submitPrediction(noteDocument, "local", { noteId: "note:typed" });
documents.watch(noteDocument, value => {
// @ts-expect-error a watched value is the definition's value
const n: number = value.body;
void n;
});
// @ts-expect-error a writer writes the definition's value
void documents.authoritativeWriter(noteDocument).write({ noteId: 1, body: "x" });
};
void typeOnly;
});
test("CultMesh TS catalogs hold one type per schema and refuse a second, typed", () => {
const sameNameVersion = { type: "cultmesh.note.named", schemaId: "sha256:note-named", schemaName: "cultmesh.note", schemaVersion: "cultmesh.note.v0" };
const sameTypeOtherSchema = { type: "cultmesh.note", schemaId: "cultmesh.note.v9" };
const note = { noteId: "note:held", body: "held" };
const kinds = [
{
catalog: (...schemas: CultMeshDocumentSchemaDescriptorLike[]) =>
CultMesh.documents(...schemas.map((schema, index) => CultMesh.document(`doc:${index}`, schema, async () => note))),
handle: (schema: CultMeshDocumentSchemaDescriptorLike, id: string) => CultMesh.document(id, schema, async () => note),
find: (catalog: any, schema: CultMeshDocumentSchemaDescriptorLike) => catalog.document(schema).documentId,
},
{
catalog: (...schemas: CultMeshDocumentSchemaDescriptorLike[]) =>
CultMesh.collections(...schemas.map((schema, index) => CultMesh.collection(`doc:${index}`, schema, async () => [note]))),
handle: (schema: CultMeshDocumentSchemaDescriptorLike, id: string) => CultMesh.collection(id, schema, async () => [note]),
find: (catalog: any, schema: CultMeshDocumentSchemaDescriptorLike) => catalog.collection(schema).collectionId,
},
];
for (const kind of kinds) {
for (const claimant of [otherTypeForNoteSchema, sameNameVersion, sameTypeOtherSchema]) {
for (const [first, second] of [[noteDocument, claimant], [claimant, noteDocument]] as const) {
const catalog = kind.catalog(first);
assert.throws(() => catalog.add(kind.handle(second, "doc:second") as any), (error: unknown) => {
assert.ok(error instanceof SchemaConflictError, String(error));
assert.equal(error.schemaId, second.schemaId);
assert.deepEqual(error.schemaNames, ["cultmesh.note", "cultmesh.note"]);
assert.ok(error.message.includes(`type "${second.type}"`) && error.message.includes(`type "${first.type}"`), error.message);
assert.equal(error.recordKey, "");
return true;
});
assert.throws(() => kind.catalog(first, second), SchemaConflictError);
// The refused handle left nothing behind: the first type still answers.
assert.equal(kind.find(catalog, first), "doc:0");
assert.equal(kind.find(catalog, { schemaId: first.schemaId }), "doc:0");
}
}
// Handles of one type for one schema may repeat, the same handle may be added again, and an untyped handle claims no type.
const held = kind.handle(noteDocument, "doc:held");
const catalog = kind.catalog(noteDocument);
catalog.add(held as any).add(held as any);
catalog.add(kind.handle({ schemaId: "cultmesh.note.v0" }, "doc:untyped") as any);
assert.equal(kind.find(catalog, noteDocument), "doc:held");
}
});
test("CultMesh TS catalog writes take (value, options) or (context, value)", async () => {
const writes: string[] = [];
let current = { noteId: "note:writes", body: "initial" };
const record = (kind: string) => async (context: { runtimeId: string }, value: Note) => {
writes.push(`${kind}:${context.runtimeId}:${value.body}`);
current = value;
};
const catalog = CultMesh.documents(CultMesh.document("note:writes", noteDocument, async () => current, {
replaceDocument: record("replace"),
submitPrediction: record("predict"),
}));
const note = (body: string) => ({ noteId: "note:writes", body });
await catalog.replace(noteDocument, note("r1"));
await catalog.replace(noteDocument, note("r2"), {});
await catalog.replace(noteDocument, note("r3"), { context: "pilot-a" });
await catalog.replace(noteDocument, "pilot-b", note("r4"));
await catalog.replace(noteDocument, CultMesh.queryContext("pilot-c"), note("r5"));
await catalog.submitPrediction(noteDocument, note("p1"));
await catalog.submitPrediction(noteDocument, note("p2"), { context: "pilot-a" });
await catalog.submitPrediction(noteDocument, "pilot-b", note("p3"));
assert.deepEqual(writes, [
"replace:local:r1",
"replace:local:r2",
"replace:pilot-a:r3",
"replace:pilot-b:r4",
"replace:pilot-c:r5",
"predict:local:p1",
"predict:pilot-a:p2",
"predict:pilot-b:p3",
]);
});
test("CultMesh TS document handles submit predictions through configured authority hooks", async () => {
@ -193,10 +303,10 @@ test("CultMesh TS document handles submit predictions through configured authori
assert.equal(document.canReplace, false);
assert.equal(document.canSubmitPrediction, true);
assert.equal(bound.canSubmitPrediction, true);
assert.equal(catalog.canSubmitPrediction(noteAliasDocument), true);
assert.equal(catalog.canSubmitPrediction(noteDocument), true);
assert.equal(catalog.canSubmitPrediction(otherTypeForNoteSchema), false);
await catalog.predictionWriter(noteAliasDocument, {
parse: value => noteAliasDocument.schema.parse(value),
await catalog.predictionWriter(noteDocument, {
context: "pilot-a",
}).write({
noteId: "note:prediction",
@ -207,23 +317,19 @@ test("CultMesh TS document handles submit predictions through configured authori
assert.deepEqual(predictions, ["predicted"]);
assert.equal((await bound.latest()).body, "predicted");
const alias = bound.asSchemaAlias(noteAliasDocument, {
parse: value => noteAliasDocument.schema.parse(value),
});
assert.equal(alias.canSubmitPrediction, true);
await alias.predictionWriter().write({
await bound.predictionWriter().write({
noteId: "note:prediction",
body: "alias-predicted",
body: "bound-predicted",
});
assert.deepEqual(predictions, ["predicted", "alias-predicted"]);
assert.equal((await document.latest("pilot-a")).body, "alias-predicted");
assert.deepEqual(predictions, ["predicted", "bound-predicted"]);
assert.equal((await document.latest("pilot-a")).body, "bound-predicted");
const updated = await document.predictionWriter("pilot-a").update(value => ({
...value,
body: "updated-as-prediction",
}));
assert.equal(updated.body, "updated-as-prediction");
assert.deepEqual(predictions, ["predicted", "alias-predicted", "updated-as-prediction"]);
assert.deepEqual(predictions, ["predicted", "bound-predicted", "updated-as-prediction"]);
assert.throws(
() => document.replace({
@ -262,57 +368,42 @@ test("CultMesh TS documents with both capabilities require an immutable writer c
assert.deepEqual(writes, ["authoritative:canonical", "prediction:predicted"]);
});
test("CultMesh TS store document handles choose compatible foreign schema payloads", async () => {
const store = {
async pullAll() {
return [
{
key: "note:foreign",
type: "runtime.generated.incompatible-note",
schemaId: "runtime.generated.incompatible-note.v1",
storedAt: "2026-06-27T00:00:00Z",
payload: payloadOf({ nope: "not the requested shape" }),
catalogEntry: {
schemaId: "runtime.generated.incompatible-note.v1",
schemaName: "runtime.generated.incompatible-note",
schemaVersion: "runtime.generated.incompatible-note.v1",
contentHash: "runtime.generated.incompatible-note.v1",
canonicalSchemaJson: "{}",
compatibleSchemaIds: ["runtime.generated.incompatible-note.v1"],
},
},
{
key: "note:foreign",
type: "runtime.generated.cultmesh.note",
schemaId: "runtime.generated.cultmesh.note.ui.42",
storedAt: "2026-06-27T00:00:01Z",
payload: payloadOf({
noteId: "note:foreign",
body: "foreign schema id, local store shape",
}),
catalogEntry: {
schemaId: "runtime.generated.cultmesh.note.ui.42",
schemaName: "cultmesh.note",
schemaVersion: "cultmesh.note.v1",
contentHash: "runtime.generated.cultmesh.note.ui.42",
canonicalSchemaJson: "{}",
compatibleSchemaIds: ["runtime.generated.cultmesh.note.ui.42"],
},
},
];
},
// A store document reads the record under its own schema id, or under an id its definition lists as compatible. A record
// under any other id is another type's, and is never read in its place because its payload happens to parse.
test("CultMesh TS store document handles read only records under an id their definition declares", async () => {
const envelope = (schemaId: string, value: unknown) => ({
key: "note:store",
type: schemaId,
schemaId,
storedAt: "2026-06-27T00:00:00Z",
payload: payloadOf(value),
});
const storeOf = (...records: ReturnType<typeof envelope>[]) => ({
async pullAll() { return records; },
async push() {},
async delete() {},
};
const document = CultMesh.documentFromStore(store, noteAliasDocument, {
documentId: "note:foreign.store",
});
const foreign = envelope("runtime.generated.cultmesh.note.ui.42", { noteId: "note:store", body: "foreign id, local shape" });
assert.deepEqual(await document.latest(), {
noteId: "note:foreign",
body: "foreign schema id, local store shape",
await assert.rejects(
CultMesh.documentFromStore(storeOf(foreign), noteDocument).latest(),
/did not contain schema cultmesh\.note\.v0/,
);
const own = envelope("cultmesh.note.v0", { noteId: "note:store", body: "own id" });
assert.equal((await CultMesh.documentFromStore(storeOf(foreign, own), noteDocument).latest()).body, "own id");
const renamed = defineDocumentType({
type: "cultmesh.note-renamed",
schemaId: "cultmesh.note-renamed.v1",
compatibleSchemaIds: ["cultmesh.note.v0"],
schema: noteDocument.schema,
});
assert.equal((await CultMesh.documentFromStore(storeOf(foreign, own), renamed).latest()).body, "own id");
await assert.rejects(CultMesh.documentFromStore(storeOf(foreign), renamed).latest(), /did not contain schema/);
// A record under the definition's own id wins over one under an id it lists, wherever each sits in the store.
const renamedOwn = envelope("cultmesh.note-renamed.v1", { noteId: "note:store", body: "renamed own id" });
assert.equal((await CultMesh.documentFromStore(storeOf(own, renamedOwn), renamed).latest()).body, "renamed own id");
});
test("CultMesh TS reactive documents submit predictions from explicit updates", async () => {
@ -401,7 +492,7 @@ test("CultMesh TS reactive updates publish nested edits while snapshot mutation
reactive.dispose();
});
test("CultMesh TS node opens same-schema aliases with one call", async () => {
test("CultMesh TS node opens its registered document with one call", async () => {
const filePath = join(await mkdtemp(join(tmpdir(), "cultmesh-ts-node-reactive-")), "node.ccmp");
const node = await CultMesh.startNode(filePath, {
documents: [noteDocument],
@ -411,34 +502,25 @@ test("CultMesh TS node opens same-schema aliases with one call", async () => {
body: "initial",
});
const aliasHandle = node.document(noteAliasDocument, "note:node-reactive", {
const handle = node.document(noteDocument, "note:node-reactive", {
pollMs: 5,
});
assert.equal(aliasHandle.documentId, "cultmesh.note.ui:note:node-reactive");
assert.equal((await aliasHandle.latest()).body, "initial");
await aliasHandle.authoritativeWriter().write({
assert.equal(handle.documentId, "cultmesh.note:note:node-reactive");
assert.equal((await handle.latest()).body, "initial");
await handle.authoritativeWriter().write({
noteId: "note:node-reactive",
body: "edited-through-alias-handle",
});
assert.equal(
node.getRequired(noteAliasDocument, "note:node-reactive").body,
"edited-through-alias-handle",
);
await node.put(noteAliasDocument, "note:node-reactive", {
noteId: "note:node-reactive",
body: "edited-through-alias-put",
body: "edited-through-handle",
});
assert.equal(
node.getRequired(noteDocument, "note:node-reactive").body,
"edited-through-alias-put",
"edited-through-handle",
);
assert.deepEqual(
(await node.collection(noteAliasDocument).latest()).map(note => note.body),
["edited-through-alias-put"],
(await node.collection(noteDocument).latest()).map(note => note.body),
["edited-through-handle"],
);
const reactive = aliasHandle.authoritativeWriter(
const reactive = handle.authoritativeWriter(
CultMesh.queryContext("browser-client"),
).reactive({
watch: false,
@ -449,18 +531,12 @@ test("CultMesh TS node opens same-schema aliases with one call", async () => {
});
await waitFor(
() => node.getRequired(noteDocument, "note:node-reactive").body === "edited-through-node-helper",
"node-level reactive alias write",
"node-level reactive write",
);
assert.equal(reactive.snapshot.noteId, "note:node-reactive");
assert.equal(
node.document(noteDocument, "note:node-reactive").asSchemaAlias(noteAliasDocument, {
parse: value => noteAliasDocument.schema.parse(value),
}).canReplace,
true,
);
assert.equal(
(await node.document(noteAliasDocument, "note:node-reactive").observe({
(await node.document(noteDocument, "note:node-reactive").observe({
watch: false,
}).ready).body,
"edited-through-node-helper",
@ -468,6 +544,33 @@ test("CultMesh TS node opens same-schema aliases with one call", async () => {
reactive.dispose();
});
test("CultMesh TS node refuses a type it did not register, even one claiming a registered schema", async () => {
const filePath = join(await mkdtemp(join(tmpdir(), "cultmesh-ts-node-other-type-")), "node.ccmp");
const node = await CultMesh.startNode(filePath, {
documents: [noteDocument],
});
await node.put(noteDocument, "note:owned", { noteId: "note:owned", body: "owned" });
const notRegistered = /"cultmesh\.note\.ui" is not registered/;
assert.throws(() => node.get(otherTypeForNoteSchema, "note:owned"), notRegistered);
await assert.rejects(
node.put(otherTypeForNoteSchema, "note:owned", { noteId: "note:owned", body: "other" }),
notRegistered,
);
await assert.rejects(node.delete(otherTypeForNoteSchema, "note:owned"), notRegistered);
await assert.rejects(node.document(otherTypeForNoteSchema, "note:owned").latest(), notRegistered);
await assert.rejects(node.collection(otherTypeForNoteSchema).latest(), notRegistered);
assert.equal(node.getRequired(noteDocument, "note:owned").body, "owned");
// The node cannot come to hold the other type beside its owner either.
await assert.rejects(
CultMesh.startNode(join(await mkdtemp(join(tmpdir(), "cultmesh-ts-node-two-types-")), "node.ccmp"), {
documents: [noteDocument, otherTypeForNoteSchema],
}),
(error: unknown) => error instanceof SchemaConflictError && error.schemaId === "cultmesh.note.v0",
);
});
test("CultMesh TS reactive documents store reconciliation deltas after misprediction", async () => {
let current = {
noteId: "note:reconcile",
@ -949,15 +1052,13 @@ test("CultMesh TS binds publication document catalogs from source resolvers", as
["daemon:first", "daemon:second"],
);
assert.equal(
(await catalog.document(noteAliasDocument, {
parse: value => noteAliasDocument.schema.parse(value),
}).latest()).body,
(await catalog.document(noteDocument).latest()).body,
"second source",
);
assert.equal(catalog.document(noteDocument).routeHint.description, "publication catalog");
});
test("CultMesh TS syncs configured publications into local node aliases", async () => {
test("CultMesh TS syncs configured publications into a local node", async () => {
const sourcePath = join(await mkdtemp(join(tmpdir(), "cultmesh-ts-publication-sync-source-")), "source.ccmp");
const targetPath = join(await mkdtemp(join(tmpdir(), "cultmesh-ts-publication-sync-target-")), "target.ccmp");
const source = await CultMesh.startNode(sourcePath, {
@ -965,7 +1066,7 @@ test("CultMesh TS syncs configured publications into local node aliases", async
});
await source.put(noteDocument, "note:published", {
noteId: "note:published",
body: "publication source hydrates local alias",
body: "publication source hydrates the local node",
});
await source.flush();
const target = await CultMesh.startNode(targetPath, {
@ -977,7 +1078,7 @@ test("CultMesh TS syncs configured publications into local node aliases", async
kind: "single-file",
path: sourcePath,
},
noteAliasDocument,
noteDocument,
"note:published",
);
const facadeSynced = await CultMesh.syncDocumentFromPublication(
@ -986,21 +1087,20 @@ test("CultMesh TS syncs configured publications into local node aliases", async
kind: "single-file",
path: sourcePath,
},
noteAliasDocument,
noteDocument,
"note:published",
);
assert.deepEqual(synced, {
noteId: "note:published",
body: "publication source hydrates local alias",
body: "publication source hydrates the local node",
});
assert.deepEqual(facadeSynced, synced);
assert.equal(target.getRequired(noteDocument, "note:published").body, "publication source hydrates local alias");
assert.equal(target.getRequired(noteAliasDocument, "note:published").body, "publication source hydrates local alias");
assert.equal((await target.document(noteAliasDocument, "note:published").observe({ watch: false }).ready).body, "publication source hydrates local alias");
assert.equal(target.getRequired(noteDocument, "note:published").body, "publication source hydrates the local node");
assert.equal((await target.document(noteDocument, "note:published").observe({ watch: false }).ready).body, "publication source hydrates the local node");
});
test("CultMesh TS syncs configured publication catalogs into local node aliases", async () => {
test("CultMesh TS syncs configured publication catalogs into a local node", async () => {
const firstPath = join(await mkdtemp(join(tmpdir(), "cultmesh-ts-publication-sync-catalog-first-")), "first.ccmp");
const secondPath = join(await mkdtemp(join(tmpdir(), "cultmesh-ts-publication-sync-catalog-second-")), "second.ccmp");
const targetPath = join(await mkdtemp(join(tmpdir(), "cultmesh-ts-publication-sync-catalog-target-")), "target.ccmp");
@ -1027,7 +1127,7 @@ test("CultMesh TS syncs configured publication catalogs into local node aliases"
CultMesh.publicationDocument(noteDocument, "note:first-sync", {
documentId: "local:first",
}),
CultMesh.publicationDocument(noteAliasDocument, "note:second-sync", {
CultMesh.publicationDocument(noteDocument, "note:second-sync", {
documentId: "local:second",
source: {
kind: "single-file",
@ -1056,37 +1156,27 @@ test("CultMesh TS syncs configured publication catalogs into local node aliases"
bindings,
);
// Two documents of one type, each addressed by its own id whatever order the catalog lists them in.
assert.deepEqual(
catalog.documents.map(document => document.documentId),
catalog.documents.map(document => document.documentId).sort(),
["local:first", "local:second"],
);
assert.equal(
await catalog.document(noteDocument, {
parse: value => noteDocument.schema.parse(value),
}).latest().then(value => value.body),
"first synced publication",
);
assert.equal(
await catalog.document(noteAliasDocument, {
parse: value => noteAliasDocument.schema.parse(value),
}).latest().then(value => value.body),
"second synced publication",
);
assert.equal(catalog.document(noteAliasDocument).routeHint.description, "publication sync catalog");
const secondSynced = catalog.documentById<Note>("local:second");
assert.equal((await catalog.documentById<Note>("local:first").latest()).body, "first synced publication");
assert.equal((await secondSynced.latest()).body, "second synced publication");
assert.equal(secondSynced.documentId, "local:second");
assert.equal(secondSynced.routeHint.description, "publication sync catalog");
assert.equal(target.getRequired(noteDocument, "note:second-sync").body, "second synced publication");
assert.equal(target.getRequired(noteAliasDocument, "note:second-sync").body, "second synced publication");
assert.equal(
await facadeCatalog.document(noteAliasDocument, {
parse: value => noteAliasDocument.schema.parse(value),
}).latest().then(value => value.body),
"second synced publication",
);
assert.equal((await facadeCatalog.documentById<Note>("local:second").latest()).body, "second synced publication");
assert.equal((await facadeCatalog.documentById<Note>("local:first").latest()).body, "first synced publication");
assert.equal(catalog.tryDocumentById("note:second-sync"), undefined);
assert.throws(() => catalog.documentById("local:third"), /no document with id 'local:third'/);
});
test("CultMesh TS document catalogs resolve semantic schema versions passed as schema ids", async () => {
const current = {
noteId: "note:semantic",
body: "semantic alias",
body: "semantic version",
};
const document = CultMesh.document(
"daemon:semantic-note",
@ -1097,18 +1187,18 @@ test("CultMesh TS document catalogs resolve semantic schema versions passed as s
},
async () => current,
{
routeHint: CultMesh.routeHint("shared-memory", "semantic alias catalog"),
routeHint: CultMesh.routeHint("shared-memory", "semantic version catalog"),
},
);
const catalog = CultMesh.documents(document);
const alias = catalog.document({
const bySchemaVersion = catalog.document({
schemaId: "cultmesh.note.v1",
});
assert.equal(alias.documentId, "daemon:semantic-note");
assert.equal(alias.routeHint.description, "semantic alias catalog");
assert.deepEqual(await alias.latest(), current);
assert.equal(bySchemaVersion.documentId, "daemon:semantic-note");
assert.equal(bySchemaVersion.routeHint.description, "semantic version catalog");
assert.deepEqual(await bySchemaVersion.latest(), current);
});
test("CultMesh TS collection handles expose typed snapshots and reset watches", async () => {
@ -1138,19 +1228,14 @@ test("CultMesh TS collection handles expose typed snapshots and reset watches",
await waitFor(() => changes.length >= 2, "collection reset after update");
unsubscribe();
assert.deepEqual(
(await collection.asSchemaAlias(noteAliasDocument, {
parse: value => noteAliasDocument.schema.parse(value),
}).latest()).map(note => note.body).sort(),
["alpha", "bravo"],
);
const catalog = CultMesh.collections(collection);
assert.deepEqual(
(await catalog.latest(noteAliasDocument, "local", {
parse: value => noteAliasDocument.schema.parse(value),
})).map(note => note.body).sort(),
(await catalog.latest(noteDocument, "local")).map(note => note.body).sort(),
["alpha", "bravo"],
);
// A typed lookup is answered by its own type's handle; a handle of another type for the same schema is not it.
assert.equal(catalog.tryCollection(otherTypeForNoteSchema), undefined);
assert.throws(() => catalog.collection(otherTypeForNoteSchema), /no collection for/);
assert.equal(
catalog.collection({ schemaId: "cultmesh.note.v0" }).collectionId,
collection.collectionId,
@ -1159,7 +1244,7 @@ test("CultMesh TS collection handles expose typed snapshots and reset watches",
assert.ok(changes.every(kind => kind === "reset"));
});
test("CultMesh TS collection catalogs resolve same-schema alias watches", async () => {
test("CultMesh TS collection catalogs resolve watches by schema version", async () => {
const current = [{
noteId: "note:catalog-watch",
body: "initial",
@ -1185,27 +1270,25 @@ test("CultMesh TS collection catalogs resolve same-schema alias watches", async
);
const catalog = CultMesh.collections(collection);
const observed: string[] = [];
const semanticAlias = { schemaId: "cultmesh.note.v1" };
const unsubscribe = catalog.watchChanges<z.infer<typeof noteAliasDocument.schema>>(semanticAlias, change => {
const bySchemaVersion = { schemaId: "cultmesh.note.v1" };
const unsubscribe = catalog.watchChanges<z.infer<typeof noteDocument.schema>>(bySchemaVersion, change => {
if (change.value) {
observed.push(change.value.body);
}
}, {
parse: value => noteAliasDocument.schema.parse(value),
});
watcher?.({
kind: "updated",
value: {
noteId: "note:catalog-watch",
body: "through-catalog-alias",
body: "through-catalog",
},
});
unsubscribe();
assert.deepEqual(observed, ["through-catalog-alias"]);
assert.deepEqual(observed, ["through-catalog"]);
assert.equal(watcher, undefined);
assert.equal(catalog.collection(semanticAlias).routeHint.description, "collection catalog");
assert.equal(catalog.collection(bySchemaVersion).routeHint.description, "collection catalog");
});
test("CultMesh TS local authority leases do not trust peer cards by contact alone", () => {
@ -3021,33 +3104,31 @@ test("CultMesh TS reads remote RUDP snapshots through document handles", async (
body: "remote handles feel local",
});
assert.equal(document.schema.type, "cultmesh.note");
const alias = CultMesh.documentFromPeerSnapshot(
peer,
defineDocumentType({
type: "cultmesh.note.alias",
schemaId: "cultmesh.note.alias.v1",
schema: noteDocument.schema,
name: "noteId",
}),
"note:remote",
{
documentId: "note:remote.alias",
timeoutMs: 1_000,
},
);
assert.deepEqual(await alias.latest(), {
noteId: "note:remote",
body: "remote handles feel local",
});
} finally {
peer?.close();
server.close();
}
});
test("CultMesh TS syncs remote RUDP snapshots into a local node with same-schema aliases", async () => {
// What ships (F8): a node's reads and puts return the object its cache holds, not a copy, and the README says that object
// belongs to the cache. The caller's own value is never the one held.
test("CultMesh TS node reads and puts return the cache's own object", async () => {
const node = await CultMesh.startNode(
join(await mkdtemp(join(tmpdir(), "cultmesh-ts-held-object-")), "node.ccmp"),
{ documents: [noteDocument] },
);
const written = { noteId: "note:held", body: "held" };
const put = await node.put(noteDocument, "note:held", written);
assert.notEqual(put, written);
assert.deepEqual(put, written);
assert.equal(node.get(noteDocument, "note:held"), put);
assert.equal(node.getRequired(noteDocument, "note:held"), put);
assert.equal(node.get(noteDocument, "note:held"), node.get(noteDocument, "note:held"));
written.body = "changed by the caller";
assert.equal(node.getRequired(noteDocument, "note:held").body, "held");
});
test("CultMesh TS syncs remote RUDP snapshots into a local node", async () => {
const connectionId = 0x1020304a;
const source = await CultMesh.startNode(
join(await mkdtemp(join(tmpdir(), "cultmesh-ts-rudp-sync-source-")), "node.ccmp"),
@ -3101,7 +3182,7 @@ test("CultMesh TS syncs remote RUDP snapshots into a local node with same-schema
const synced = await target.syncDocumentFromPeerSnapshot(
peer,
noteAliasDocument,
noteDocument,
"note:sync-remote",
{
timeoutMs: 1_000,
@ -3110,7 +3191,7 @@ test("CultMesh TS syncs remote RUDP snapshots into a local node with same-schema
const facadeSynced = await CultMesh.syncDocumentFromPeerSnapshot(
target,
peer,
noteAliasDocument,
noteDocument,
"note:sync-remote",
{
timeoutMs: 1_000,
@ -3123,29 +3204,29 @@ test("CultMesh TS syncs remote RUDP snapshots into a local node with same-schema
});
assert.deepEqual(facadeSynced, synced);
assert.equal(target.getRequired(noteDocument, "note:sync-remote").body, "synced once, read as local");
assert.equal(target.getRequired(noteAliasDocument, "note:sync-remote").body, "synced once, read as local");
assert.equal((await target.document(noteAliasDocument, "note:sync-remote").latest()).body, "synced once, read as local");
assert.equal((await target.document(noteDocument, "note:sync-remote").latest()).body, "synced once, read as local");
} finally {
peer?.close();
server.close();
}
});
test("CultMesh TS peer snapshot handles choose compatible foreign schema payloads", async () => {
// A peer snapshot is read, and synced, only under the requested type's schema id. A record the peer holds at the key under
// another id is another type's: it is not-found for this type, and syncing never writes it anywhere.
test("CultMesh TS peer snapshots never take another type's record at the key", async () => {
const connectionId = 0x10203049;
const node = await CultMesh.startNode(
const source = await CultMesh.startNode(
join(await mkdtemp(join(tmpdir(), "cultmesh-ts-rudp-foreign-snapshot-")), "node.ccmp"),
{
documents: [incompatibleNoteDocument, foreignNoteDocument],
},
{ documents: [incompatibleNoteDocument, foreignNoteDocument] },
);
await node.put(incompatibleNoteDocument, "note:foreign", {
nope: "not the requested shape",
});
await node.put(foreignNoteDocument, "note:foreign", {
noteId: "note:foreign",
body: "foreign schema id, local shape",
});
await source.put(incompatibleNoteDocument, "note:b", { nope: "remote b" });
// Parses as a note: shape routing would store it as one.
await source.put(foreignNoteDocument, "note:c", { noteId: "note:c", body: "remote c, note-shaped" });
const target = await CultMesh.startNode(
join(await mkdtemp(join(tmpdir(), "cultmesh-ts-rudp-foreign-target-")), "node.ccmp"),
{ documents: [noteDocument, incompatibleNoteDocument] },
);
await target.put(incompatibleNoteDocument, "note:b", { nope: "local b" });
const server = CultMesh.createRudpDocumentServer(
"cultmesh-ts-rudp-foreign-snapshot-server",
@ -3155,7 +3236,7 @@ test("CultMesh TS peer snapshot handles choose compatible foreign schema payload
defineCultNetDocumentBinding({ definition: incompatibleNoteDocument }),
defineCultNetDocumentBinding({ definition: foreignNoteDocument }),
]),
getCache: () => node.cache,
getCache: () => source.cache,
bindHost: "127.0.0.1",
bindPort: 0,
resendDelayMs: 25,
@ -3181,20 +3262,22 @@ test("CultMesh TS peer snapshot handles choose compatible foreign schema payload
},
);
const alias = CultMesh.documentFromPeerSnapshot(
peer,
noteAliasDocument,
"note:foreign",
{
documentId: "note:foreign.alias",
timeoutMs: 1_000,
},
for (const key of ["note:b", "note:c"]) {
const notFound = new RegExp(`did not return cultmesh\\.note\\.v0 at ${key}`);
await assert.rejects(
CultMesh.documentFromPeerSnapshot(peer, noteDocument, key, { timeoutMs: 1_000 }).latest(),
notFound,
);
await assert.rejects(target.syncDocumentFromPeerSnapshot(peer, noteDocument, key, { timeoutMs: 1_000 }), notFound);
assert.equal(target.get(noteDocument, key), undefined, key);
}
// The failed sync of a note at "note:b" left the local record of the other type there untouched.
assert.deepEqual(target.getRequired(incompatibleNoteDocument, "note:b"), { nope: "local b" });
// Its own type's record under its own id still syncs.
assert.deepEqual(
await target.syncDocumentFromPeerSnapshot(peer, incompatibleNoteDocument, "note:b", { timeoutMs: 1_000 }),
{ nope: "remote b" },
);
assert.deepEqual(await alias.latest(), {
noteId: "note:foreign",
body: "foreign schema id, local shape",
});
} finally {
peer?.close();
server.close();

9
packages/cultnet-py/CHANGELOG.md Normal file
View file

@ -0,0 +1,9 @@
# Changelog
## Unreleased (breaking; version 0.x, so the next minor)
- Removed `resolve_document_and_schema_id_for_raw_record` from the package exports. A raw record resolves to one local schema id, the one
its document registers, so there is no wire id to return. Use `resolve_document_for_raw_record(documents_by_schema_id, schema_id, record)`,
which returns the document; take the schema id from `document.catalog_entry().schema_id`.
- `apply_raw_document_record`, `apply_raw_snapshot` and `apply_shard_log_response` report and stamp the local schema id, also for a record
that arrived under a compatible id or the schema name.

4
packages/cultnet-py/src/cultnet_py/__init__.py
View file

@ -66,7 +66,7 @@ from .replication import (
apply_raw_document_record,
apply_raw_snapshot,
apply_shard_log_response,
resolve_document_and_schema_id_for_raw_record,
resolve_document_for_raw_record,
schema_document_map,
)
from .schema_catalog import (
@ -228,7 +228,7 @@ __all__ = [
"apply_raw_document_record",
"apply_raw_snapshot",
"apply_shard_log_response",
"resolve_document_and_schema_id_for_raw_record",
"resolve_document_for_raw_record",
"compute_simulation_claim_hash",
"compute_reconnect_delay_ms",
"create_reconnect_policy",

3
packages/cultnet-py/src/cultnet_py/interop_peer.py
View file

@ -1316,6 +1316,7 @@ def witness_artifact_bundle_schema_json() -> str:
"required": ["role", "schemaId"],
"additionalProperties": False,
"properties": {
"id": {"type": "string", "minLength": 1},
"role": {"type": "string", "minLength": 1},
"schemaId": {"type": "string", "minLength": 1},
"schemaVersion": {"type": "string", "minLength": 1},
@ -1327,6 +1328,7 @@ def witness_artifact_bundle_schema_json() -> str:
"required": ["role", "uri", "mediaType"],
"additionalProperties": False,
"properties": {
"id": {"type": "string", "minLength": 1},
"role": {"type": "string", "minLength": 1},
"uri": {"type": "string", "minLength": 1},
"mediaType": {"type": "string", "minLength": 1},
@ -1340,6 +1342,7 @@ def witness_artifact_bundle_schema_json() -> str:
"required": ["stage", "startedAt", "completedAt", "latencyMs"],
"additionalProperties": False,
"properties": {
"id": {"type": "string", "minLength": 1},
"stage": {"type": "string", "minLength": 1},
"startedAt": {"type": "string", "minLength": 1},
"completedAt": {"type": "string", "minLength": 1},

29
packages/cultnet-py/src/cultnet_py/replication.py
View file

@ -37,11 +37,9 @@ def apply_raw_document_record(
) -> CultNetAppliedRecord:
wire = record.to_wire() if isinstance(record, CultNetRawDocumentRecord) else record
schema_id = str(wire["schemaId"])
document, resolved_schema_id = resolve_document_and_schema_id_for_raw_record(
documents_by_schema_id,
schema_id,
wire,
)
document = resolve_document_for_raw_record(documents_by_schema_id, schema_id, wire)
# The record is applied, stamped and reported under the local schema id, as the cache stores it and a delete reports it.
resolved_schema_id = document.catalog_entry().schema_id
envelope = raw_record_to_envelope(document, resolved_schema_id, wire)
value = cache.put_envelope(document, envelope)
return CultNetAppliedRecord(
@ -82,11 +80,8 @@ def apply_raw_snapshot(
if not isinstance(record, dict):
continue
schema_id = str(record["schemaId"])
document, resolved_schema_id = resolve_document_and_schema_id_for_raw_record(
documents_by_schema_id,
schema_id,
record,
)
document = resolve_document_for_raw_record(documents_by_schema_id, schema_id, record)
resolved_schema_id = document.catalog_entry().schema_id
envelope = raw_record_to_envelope(document, resolved_schema_id, record)
batch = batches.setdefault(document.type, (document, []))
batch[1].append((resolved_schema_id, envelope))
@ -159,27 +154,19 @@ def can_resolve_document_for_raw_record(
return False
# routes a replicated raw record to a local binding; the store always persists the local schema id; this is not store-reader recovery.
def resolve_document_for_raw_record(
documents_by_schema_id: dict[str, DocumentDefinition[Any]],
schema_id: str,
record: dict[str, Any],
) -> DocumentDefinition[Any]:
return resolve_document_and_schema_id_for_raw_record(documents_by_schema_id, schema_id, record)[0]
def resolve_document_and_schema_id_for_raw_record(
documents_by_schema_id: dict[str, DocumentDefinition[Any]],
schema_id: str,
record: dict[str, Any],
) -> tuple[DocumentDefinition[Any], str]:
document = documents_by_schema_id.get(schema_id)
if document is not None:
return document, schema_id
return document
schema_version = _infer_schema_version_from_payload(bytes(record["payload"]))
schema_name = _infer_schema_name(schema_version) if schema_version is not None else None
if schema_name is not None and schema_name in documents_by_schema_id:
document = documents_by_schema_id[schema_name]
return document, document.catalog_entry().schema_id
return documents_by_schema_id[schema_name]
raise KeyError(schema_id)

55
packages/cultnet-py/tests/test_cultnet.py
View file

@ -2102,6 +2102,51 @@ class CultNetTests(unittest.TestCase):
self.assertEqual(applied_delete[0].change_kind, "removed")
self.assertIsNone(cache.get(document, "policy:1"))
# A compatible schema id applies through the real replication function, the record is stamped with the registered id, and
# the store it leaves reopens and takes the next write.
def test_cultnet_raw_replication_under_a_compatible_schema_id_leaves_a_store_that_reopens(self) -> None:
import tempfile
from pathlib import Path
from cultcache_py import SingleFileMessagePackBackingStore
document = define_database_entry_type(
"replica.compatible-policy",
[("schema_version", 0), ("name", 1)],
schema_id="replica.compatible-policy.current",
schema_name="replica.compatible_policy",
schema_version="replica.compatible_policy.v1",
compatible_schema_ids=["replica.compatible-policy.current", "replica.compatible-policy.older"],
)
value = {"schema_version": "replica.compatible_policy.v1", "name": "synced"}
record = {
"schemaId": "replica.compatible-policy.older",
"recordKey": "policy:1",
"storedAt": "2026-06-25T00:00:00Z",
"payloadEncoding": "messagepack",
"payload": document.encode_payload(value),
}
with tempfile.TemporaryDirectory() as tmp:
path = Path(tmp) / "store.msgpack"
def open_cache() -> CultCache:
cache = CultCache.builder().register_document_type(document).add_generic_store(
SingleFileMessagePackBackingStore(path)
).build()
cache.pull_all_backing_stores()
return cache
cache = open_cache()
applied = apply_raw_document_record(cache, schema_document_map([document]), record)
self.assertEqual(applied.schema_id, document.catalog_entry().schema_id)
self.assertEqual(
cache.get_required_envelope(document, "policy:1").schema_id, document.catalog_entry().schema_id
)
reopened = open_cache()
self.assertEqual(reopened.get_required(document, "policy:1")["name"], "synced")
reopened.put(document, "policy:2", {**value, "name": "next"})
self.assertEqual(open_cache().get_required(document, "policy:2")["name"], "next")
def test_cultnet_document_delete_helper_matches_schema_v0_shape(self) -> None:
message = document_delete(
message_id="delete-1",
@ -2425,9 +2470,9 @@ class CultNetTests(unittest.TestCase):
witness_kind="interop-proof",
captured_at="2026-06-13T00:00:03Z",
subject={"documentType": "cultnet.interop-note", "subjectId": "note:python"},
contracts=[{"role": "payload", "schemaId": "schema-a"}],
artifacts=[{"role": "log", "uri": "cultcache://bundle-1/log", "mediaType": "text/plain"}],
timing_witnesses=[{"stage": "roundtrip", "startedAt": "2026-06-13T00:00:03Z", "completedAt": "2026-06-13T00:00:04Z", "latencyMs": 1.0}],
contracts=[{"id": "c0e1a2b3c4d5", "role": "payload", "schemaId": "schema-a"}],
artifacts=[{"id": "a0e1a2b3c4d5", "role": "log", "uri": "cultcache://bundle-1/log", "mediaType": "text/plain"}],
timing_witnesses=[{"id": "e0e1a2b3c4d5", "stage": "roundtrip", "startedAt": "2026-06-13T00:00:03Z", "completedAt": "2026-06-13T00:00:04Z", "latencyMs": 1.0}],
provenance={"pipelineId": "interop", "runId": "run-1", "runtimeId": "python-test"},
)
payload = encode_witness_artifact_bundle_payload(bundle)
@ -2443,6 +2488,8 @@ class CultNetTests(unittest.TestCase):
self.assertEqual(slots[2], "2026-06-13T00:00:03Z")
self.assertEqual(slots[3]["subjectId"], "note:python")
self.assertEqual(slots[4][0]["schemaId"], "schema-a")
self.assertEqual([slots[4][0]["id"], slots[5][0]["id"], slots[6][0]["id"]], ["c0e1a2b3c4d5", "a0e1a2b3c4d5", "e0e1a2b3c4d5"])
self.assertEqual(typed_decoded.contracts[0]["id"], "c0e1a2b3c4d5")
self.assertEqual(slots[5][0]["uri"], "cultcache://bundle-1/log")
self.assertEqual(slots[6][0]["latencyMs"], 1.0)
self.assertEqual(slots[7]["runtimeId"], "python-test")
@ -2462,7 +2509,7 @@ class CultNetTests(unittest.TestCase):
runtime_kind="python",
display_name="Python Interop Test",
agent_id="python-interop-test-agent",
schema_path=str(Path("contracts/cultnet/interop/cultnet.interop-note.schema.json")),
schema_path=str(Path(__file__).resolve().parents[3] / "contracts" / "cultnet" / "interop" / "cultnet.interop-note.schema.json"),
)
binding = state.bindings["note"]
logged_value = {

12
packages/cultnet-rs/CHANGELOG.md Normal file
View file

@ -0,0 +1,12 @@
# Changelog
All notable changes to `cultnet-rs` are recorded here, following
[Keep a Changelog](https://keepachangelog.com/en/1.1.0/). Earlier releases are recorded
only in git history.
## Unreleased
### Breaking
- `CultNetDocumentRegistry::register` returns `Result<&mut Self>` where it returned `&mut Self`. A binding that claims a document type or a schema id another binding already holds is refused with `cultcache_rs::SchemaConflictError` (naming the new binding's schema id and both document types) and nothing is bound; binding a type to the schema id it already holds still replaces that binding. Callers chain `register(...)?`.
- `CultMeshCdnChunkRef` gains a public `id` field, the decimal text of `offset` (the reference's element id, slot 4). A manifest written before ids reads `""`; encoding writes the slot.

17
packages/cultnet-rs/examples/cultnet_interop_peer.rs
View file

@ -262,7 +262,7 @@ fn serve(config: PeerConfig) -> Result<()> {
cache.put(&note.document_id, &note)?;
let mut document_registry = CultNetDocumentRegistry::new();
register_capability_bindings(&mut document_registry, &schema_registration.schema_id);
register_capability_bindings(&mut document_registry, &schema_registration.schema_id)?;
let cache = Arc::new(Mutex::new(cache));
let document_registry = Arc::new(document_registry);
@ -384,7 +384,7 @@ fn dial(config: DialConfig) -> Result<()> {
cache.pull_all_backing_stores()?;
let mut document_registry = CultNetDocumentRegistry::new();
register_capability_bindings(&mut document_registry, &schema_registration.schema_id);
register_capability_bindings(&mut document_registry, &schema_registration.schema_id)?;
let transport_name = if config.target_rudp_port.is_some() {
"rudp"
@ -1227,38 +1227,39 @@ fn register_capability_entry_types(cache: &mut CultCache) -> Result<()> {
fn register_capability_bindings(
document_registry: &mut CultNetDocumentRegistry,
note_schema_id: &str,
) {
) -> Result<()> {
document_registry
.register(CultNetDocumentBinding::for_entry_with_schema_id::<
CultNetInteropNote,
>(
note_schema_id.to_string(),
INTEROP_SCHEMA_VERSION.to_string(),
))
))?
.register(CultNetDocumentBinding::for_entry_with_schema_id::<
CultNetInteropMutationIntent,
>(
MUTATION_INTENT_SCHEMA_ID.to_string(),
MUTATION_INTENT_SCHEMA_VERSION.to_string(),
))
))?
.register(CultNetDocumentBinding::for_entry_with_schema_id::<
CultNetInteropMutationReceipt,
>(
MUTATION_RECEIPT_SCHEMA_ID.to_string(),
MUTATION_RECEIPT_SCHEMA_VERSION.to_string(),
))
))?
.register(CultNetDocumentBinding::for_entry_with_schema_id::<
CultNetInteropFireCommand,
>(
FIRE_COMMAND_SCHEMA_ID.to_string(),
FIRE_COMMAND_SCHEMA_VERSION.to_string(),
))
))?
.register(CultNetDocumentBinding::for_entry_with_schema_id::<
CultNetInteropFireReceipt,
>(
FIRE_RECEIPT_SCHEMA_ID.to_string(),
FIRE_RECEIPT_SCHEMA_VERSION.to_string(),
));
))?;
Ok(())
}
fn interaction_contracts() -> Vec<CultNetDocumentMutationContract> {

8
packages/cultnet-rs/src/content.rs
View file

@ -27,6 +27,8 @@ pub struct CultMeshCdnChunkRef {
pub offset: i64,
pub size_bytes: i32,
pub record_key: String,
/// Element id (key 4): the decimal text of `offset`, never random. Absent in a manifest written before ids.
pub id: String,
}
/// `CultMeshCdnArtifactManifest`. Encodes as the reference's positional array (keys 0-9).
@ -50,7 +52,7 @@ pub struct CultMeshCdnArtifactManifest {
// The tuple mirrors: `to_vec_named` would write a map for a named-field struct, but a tuple
// struct is an array under every serde encoder.
#[derive(Serialize, Deserialize)]
struct ChunkRefWire(String, i64, i32, String);
struct ChunkRefWire(String, i64, i32, String, #[serde(default)] String);
#[derive(Serialize, Deserialize)]
struct ManifestWire(
@ -105,7 +107,7 @@ mod ordered_map {
impl From<CultMeshCdnChunkRef> for ChunkRefWire {
fn from(value: CultMeshCdnChunkRef) -> Self {
Self(value.chunk_hash, value.offset, value.size_bytes, value.record_key)
Self(value.chunk_hash, value.offset, value.size_bytes, value.record_key, value.id)
}
}
@ -116,6 +118,7 @@ impl From<ChunkRefWire> for CultMeshCdnChunkRef {
offset: value.1,
size_bytes: value.2,
record_key: value.3,
id: value.4,
}
}
}
@ -217,6 +220,7 @@ pub fn pack_content(
offset,
// A piece is at most `chunk_size` bytes, which fits an i32 (checked above).
size_bytes: piece.len() as i32,
id: offset.to_string(),
});
offset += piece.len() as i64;
chunks.push(CultMeshCdnChunk {

24
packages/cultnet-rs/src/replication.rs
View file

@ -3,6 +3,7 @@ use anyhow::anyhow;
use cultcache_rs::CultCache;
use cultcache_rs::CultCacheEnvelope;
use cultcache_rs::DatabaseEntry;
use cultcache_rs::SchemaConflictError;
use serde::Serialize;
use serde::de::DeserializeOwned;
use serde_json::Value;
@ -127,12 +128,31 @@ impl CultNetDocumentRegistry {
Self::default()
}
pub fn register(&mut self, binding: CultNetDocumentBinding) -> &mut Self {
/// Binds `binding`. One binding owns a document type and a schema id: a binding claiming a type or a schema id
/// another binding already holds is refused with [`SchemaConflictError`], naming the new binding's schema id and
/// both document types, and nothing is bound. Binding a type to the schema id it already holds replaces that binding.
pub fn register(&mut self, binding: CultNetDocumentBinding) -> Result<&mut Self> {
let holder = self
.bindings_by_type
.get(&binding.document_type)
.filter(|held| held.schema_id != binding.schema_id)
.or_else(|| {
self.bindings_by_schema_id
.get(&binding.schema_id)
.filter(|held| held.document_type != binding.document_type)
});
if let Some(holder) = holder {
return Err(anyhow::Error::new(SchemaConflictError {
schema_id: binding.schema_id.clone(),
schema_names: vec![holder.document_type.clone(), binding.document_type.clone()],
record_key: String::new(),
}));
}
self.bindings_by_schema_id
.insert(binding.schema_id.clone(), binding.clone());
self.bindings_by_type
.insert(binding.document_type.clone(), binding);
self
Ok(self)
}
pub fn binding(&self, document_type: &str) -> Option<&CultNetDocumentBinding> {

1
packages/cultnet-rs/tests/content.rs
View file

@ -547,6 +547,7 @@ fn chunk_ref(hash: &str, offset: i64, size: i32) -> cultnet_rs::CultMeshCdnChunk
offset,
size_bytes: size,
record_key: String::new(),
id: offset.to_string(),
}
}

90
packages/cultnet-rs/tests/cultnet.rs
View file

@ -1773,7 +1773,7 @@ fn document_registry_advertises_mutation_contracts_with_bindings() -> Result<()>
"ghostlight.agent_state.v0".to_string(),
)
.with_mutation_contract(contract.clone()),
);
)?;
assert_eq!(registry.mutation_contracts(), vec![contract]);
Ok(())
@ -1842,7 +1842,7 @@ fn document_registry_replicates_typed_cultcache_state() -> Result<()> {
let mut registry = CultNetDocumentRegistry::new();
registry.register(CultNetDocumentBinding::for_entry::<
GhostlightAgentStateFixture,
>("ghostlight.agent_state.v0".to_string()));
>("ghostlight.agent_state.v0".to_string()))?;
let mut origin = CultCache::new();
origin.register_entry_type::<GhostlightAgentStateFixture>()?;
@ -1899,7 +1899,7 @@ fn raw_snapshot_replication_preserves_messagepack_payload_bytes() -> Result<()>
let mut registry = CultNetDocumentRegistry::new();
registry.register(CultNetDocumentBinding::for_entry::<
GhostlightAgentStateFixture,
>("ghostlight.agent_state.v0".to_string()));
>("ghostlight.agent_state.v0".to_string()))?;
let mut origin = CultCache::new();
origin.register_entry_type::<GhostlightAgentStateFixture>()?;
@ -1945,7 +1945,7 @@ fn serve_read_only_raw_snapshot_omitted_v0_schema_list_is_no_filter_present_empt
let mut registry = CultNetDocumentRegistry::new();
registry.register(CultNetDocumentBinding::for_entry_with_schema_id::<
GhostlightAgentStateFixture,
>("ghostlight.agent-state".to_string(), None));
>("ghostlight.agent-state".to_string(), None))?;
let mut policy = CultNetReadOnlySnapshotPolicy::new();
policy.allow("ghostlight.agent-state", "row-1")?;
@ -2039,7 +2039,7 @@ fn serve_read_only_raw_snapshot_keeps_the_requested_record_key_order() -> Result
let mut registry = CultNetDocumentRegistry::new();
registry.register(CultNetDocumentBinding::for_entry_with_schema_id::<
GhostlightAgentStateFixture,
>("ghostlight.agent-state".to_string(), None));
>("ghostlight.agent-state".to_string(), None))?;
let mut policy = CultNetReadOnlySnapshotPolicy::new();
policy.allow("ghostlight.agent-state", "row-1")?;
@ -2099,10 +2099,64 @@ fn serve_read_only_raw_snapshot_keeps_the_requested_record_key_order() -> Result
}
#[test]
fn raw_snapshot_replication_hydrates_same_schema_rust_aliases() -> Result<()> {
fn document_registry_binds_one_type_per_schema_id_and_refuses_a_second_typed() -> Result<()> {
let canonical = |schema_id: &str| {
CultNetDocumentBinding::for_entry_with_schema_id::<GhostlightAgentStateFixture>(schema_id.to_string(), None)
};
let ui = |schema_id: &str| {
CultNetDocumentBinding::for_entry_with_schema_id::<GhostlightAgentStateUiFixture>(schema_id.to_string(), None)
};
let canonical_type = GhostlightAgentStateFixture::TYPE.to_string();
let ui_type = GhostlightAgentStateUiFixture::TYPE.to_string();
// Another type for a held schema id, and a held type for another schema id, in both registration orders.
let cases = [
(canonical("id.agent"), ui("id.agent"), "id.agent", [&canonical_type, &ui_type]),
(ui("id.agent"), canonical("id.agent"), "id.agent", [&ui_type, &canonical_type]),
(canonical("id.agent"), canonical("id.other"), "id.other", [&canonical_type, &canonical_type]),
(canonical("id.other"), canonical("id.agent"), "id.agent", [&canonical_type, &canonical_type]),
];
for (first, second, schema_id, names) in cases {
let mut registry = CultNetDocumentRegistry::new();
registry.register(first.clone())?;
let error = registry.register(second.clone()).err().expect("a second claimant is refused");
let conflict = error
.downcast_ref::<cultcache_rs::SchemaConflictError>()
.unwrap_or_else(|| panic!("{error:#}"));
assert_eq!(conflict.schema_id, schema_id);
assert_eq!(conflict.schema_names, names.map(|name| name.clone()).to_vec());
assert_eq!(conflict.record_key, "");
assert!(error.to_string().contains(schema_id), "{error:#}");
// The refused binding left nothing behind: the first binding still answers for its type and its schema id.
assert_eq!(registry.binding(&first.document_type).map(|held| held.schema_id.as_str()), Some(first.schema_id.as_str()));
assert_eq!(
registry.binding_by_schema_id(&first.schema_id).map(|held| held.document_type.as_str()),
Some(first.document_type.as_str())
);
if second.schema_id != first.schema_id {
assert!(registry.binding_by_schema_id(&second.schema_id).is_none());
}
}
// A type bound again to the schema id it holds replaces its binding.
let mut registry = CultNetDocumentRegistry::new();
registry.register(canonical("id.agent"))?;
registry.register(CultNetDocumentBinding::for_entry_with_schema_id::<GhostlightAgentStateFixture>(
"id.agent".to_string(),
Some("ghostlight.agent_state.v2".to_string()),
))?;
assert_eq!(
registry.binding(&canonical_type).and_then(|held| held.payload_schema_version.as_deref()),
Some("ghostlight.agent_state.v2")
);
Ok(())
}
#[test]
fn raw_snapshot_replication_hydrates_the_receivers_own_type_for_the_schema() -> Result<()> {
let temp = tempfile::tempdir()?;
let origin_store = temp.path().join("origin-alias.msgpack");
let target_store = temp.path().join("target-alias.msgpack");
let origin_store = temp.path().join("origin.msgpack");
let target_store = temp.path().join("target.msgpack");
let canonical = GhostlightAgentStateFixture {
schema_version: "ghostlight.agent_state.v0".to_string(),
agent_id: "epiphany.persona".to_string(),
@ -2115,7 +2169,7 @@ fn raw_snapshot_replication_hydrates_same_schema_rust_aliases() -> Result<()> {
>(
"ghostlight.agent_state.v0".to_string(),
"ghostlight.agent_state.v0".to_string(),
));
))?;
let mut origin = CultCache::new();
origin.register_entry_type::<GhostlightAgentStateFixture>()?;
origin.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&origin_store))?;
@ -2123,24 +2177,24 @@ fn raw_snapshot_replication_hydrates_same_schema_rust_aliases() -> Result<()> {
origin.put("epiphany.persona", &canonical)?;
let raw_snapshot = origin_registry.create_raw_snapshot_response(
&origin,
"raw-snapshot-alias",
"raw-snapshot-receiver-type",
Some(&["ghostlight.agent_state.v0".to_string()]),
Some(&["epiphany.persona".to_string()]),
)?;
let mut alias_registry = CultNetDocumentRegistry::new();
alias_registry.register(CultNetDocumentBinding::for_entry_with_schema_id::<
let mut target_registry = CultNetDocumentRegistry::new();
target_registry.register(CultNetDocumentBinding::for_entry_with_schema_id::<
GhostlightAgentStateUiFixture,
>(
"ghostlight.agent_state.v0".to_string(),
"ghostlight.agent_state.v0".to_string(),
));
))?;
let mut target = CultCache::new();
target.register_entry_type::<GhostlightAgentStateUiFixture>()?;
target.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&target_store))?;
target.pull_all_backing_stores()?;
let applied = alias_registry
let applied = target_registry
.sync_raw_document_from_snapshot_response::<GhostlightAgentStateUiFixture>(
&mut target,
&raw_snapshot,
@ -2163,16 +2217,16 @@ fn raw_snapshot_replication_hydrates_same_schema_rust_aliases() -> Result<()> {
}
#[test]
fn reactive_document_coalesces_direct_same_schema_alias_member_writes() -> Result<()> {
fn reactive_document_coalesces_direct_member_writes() -> Result<()> {
let temp = tempfile::tempdir()?;
let target_store = temp.path().join("target-reactive-alias.msgpack");
let target_store = temp.path().join("target-reactive.msgpack");
let mut registry = CultNetDocumentRegistry::new();
registry.register(CultNetDocumentBinding::for_entry_with_schema_id::<
GhostlightReactiveNoteUiFixture,
>(
"ghostlight.reactive_note.v0".to_string(),
"ghostlight.reactive_note.v0".to_string(),
));
))?;
let cache = Arc::new(Mutex::new(CultCache::new()));
{
let mut cache = cache.lock().expect("cache mutex");
@ -2237,7 +2291,7 @@ fn reactive_document_tracks_canonical_reconciliation_delta() -> Result<()> {
>(
"ghostlight.reactive_note.v0".to_string(),
"ghostlight.reactive_note.v0".to_string(),
));
))?;
let cache = Arc::new(Mutex::new(CultCache::new()));
{
let mut cache = cache.lock().expect("cache mutex");

8
packages/cultnet-ts/CHANGELOG.md
View file

@ -3,6 +3,14 @@
All notable changes to `cultnet-ts` are recorded here, following
[Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
## Unreleased
### Breaking
- `CultMeshCdnChunkRef` carries `id` (slot 4), the reference's element id. `encodeManifest` writes five-slot chunk references and `packContent` sets the decimal offset as the id, as the reference does; a manifest written before ids reads `""` and is rewritten with the slot. Content vectors regenerated from the reference.
- Needs the cultcache-ts release that ships with this change (its manifest range moves with that release): the Idunn runtime authority reader calls `readSingleFileStore`, which 0.14.x does not export. An absent authority file is now a plain `Error` without a code.
## [0.3.0]
### Added

4
packages/cultnet-ts/README.md
View file

@ -143,7 +143,9 @@ imports no `node:` module, so a browser bundle can take it.
- The decoder reads what MessagePack-CSharp reads and refuses what it refuses:
nil in a string, tag or metadata slot is `null` and is written back as nil,
a missing trailing slot takes the reference's default, extra slots and trailing
bytes are skipped, a float-encoded integer is refused, and invalid UTF-8
bytes are skipped (a chunk reference's fifth slot is its element id, a string
or nil, `""` when absent; `packContent` writes the decimal offset as the id, as the
reference does), a float-encoded integer is refused, and invalid UTF-8
decodes to U+FFFD. A nil `chunks` is refused, as the reference's
`ValidateManifest` refuses it. `content-acceptance-vectors.json` is the
reference's own verdict on each of those shapes.

9
packages/cultnet-ts/src/content.ts
View file

@ -26,6 +26,8 @@ export interface CultMeshCdnChunkRef {
readonly offset: number;
readonly sizeBytes: number;
readonly recordKey: string | null;
// Element id, slot 4: the reference derives it from the offset when packing; "" when a manifest written before ids omits it.
readonly id: string | null;
}
// Metadata is ordered entries, never an object: a JS object reorders integer-like keys, and the entry order is wire bytes.
@ -142,13 +144,13 @@ const mapHeader = (count: number): Uint8Array =>
/** A string slot is written as C# writes it: a lone surrogate has no UTF-8 encoding and becomes U+FFFD at every length. */
const text = <T>(field: T): T => (typeof field === "string" ? wellFormed(field) : field) as T;
/** The positional ten-array of the C# `[MessagePackObject]`: chunk references are four-arrays, metadata a map in entry order. */
/** The positional ten-array of the C# `[MessagePackObject]`: chunk references are five-arrays, metadata a map in entry order. */
export function encodeManifest(manifest: CultMeshCdnArtifactManifest): Uint8Array {
const { artifactId, kind, version, contentHash, sizeBytes, mimeType, createdAtUtc, chunks, tags, metadata } = manifest;
return concat([
Uint8Array.of(0x9a),
...[artifactId, kind, version, contentHash, sizeBytes, mimeType, createdAtUtc].map(field => encode(text(field))),
encode(chunks.map(c => [text(c.chunkHash), c.offset, c.sizeBytes, text(c.recordKey)])),
encode(chunks.map(c => [text(c.chunkHash), c.offset, c.sizeBytes, text(c.recordKey), text(c.id)])),
encode(tags === null ? null : tags.map(text)),
...(metadata === null ? [encode(null)] : [mapHeader(metadata.length), ...metadata.flatMap(([key, value]) => [encode(text(key)), encode(text(value))])]),
]);
@ -279,6 +281,7 @@ function readChunkRef(reader: Reader): CultMeshCdnChunkRef {
// The reference's `int`.
sizeBytes: slot(() => { const size = reader.integer("chunks.sizeBytes"); return size >= -(2 ** 31) && size < 2 ** 31 ? size : wrongType("chunks.sizeBytes"); }, 0),
recordKey: slot(() => reader.string("chunks.recordKey"), ""),
id: slot(() => reader.string("chunks.id"), ""),
};
for (; index < count; index += 1) reader.skip();
return chunk;
@ -308,7 +311,7 @@ export async function packContent(
sizeBytes: bytes.length,
mimeType: isNullOrWhiteSpaceCSharp(options.mimeType) ? "application/octet-stream" : options.mimeType!,
createdAtUtc: options.createdAtUtc ?? `${new Date().toISOString().slice(0, 23)}0000+00:00`,
chunks: chunks.map(({ chunkHash, payload }, index) => ({ chunkHash, offset: offsets[index]!, sizeBytes: payload.length, recordKey: chunkRecordKey(chunkHash) })),
chunks: chunks.map(({ chunkHash, payload }, index) => ({ chunkHash, offset: offsets[index]!, sizeBytes: payload.length, recordKey: chunkRecordKey(chunkHash), id: String(offsets[index]) })),
tags: options.tags ?? [],
metadata: options.metadata ?? [],
};

30
packages/cultnet-ts/src/idunn-runtime-authority.ts
View file

@ -1,7 +1,8 @@
import crypto from "node:crypto";
import fs from "node:fs";
import path from "node:path";
import { documentPayloadFromBytes } from "@gamecult/cultcache-ts/document";
import { readSingleFileStore } from "@gamecult/cultcache-ts";
import { documentPayloadBytes, documentPayloadFromBytes } from "@gamecult/cultcache-ts/document";
import { decode, encode } from "@msgpack/msgpack";
import {
@ -28,7 +29,6 @@ export const IDUNN_RUNTIME_CANDIDATE_BIND_ENVIRONMENT = "GAMECULT_IDUNN_CANDIDAT
const PROVIDER_ID_DOMAIN = Buffer.from("gamecult.provider-health.identity.v1\0", "utf8");
const PROVIDER_PROTECTOR_CONTEXT = "gamecult-provider-health-identity-v1";
const ACTIVATION_ID_DOMAIN = Buffer.from("idunn.runtime-activation.id.v1\0", "utf8");
const CULTCACHE_STORE_FORMAT = "cultcache.store.v1";
const ED25519_PKCS8_PREFIX = Buffer.from("302e020100300506032b657004220420", "hex");
// Limit: more than this many warmings signed between Idunn's observation and
// assertWriteLease would push the leased one out and refuse a valid lease.
@ -319,25 +319,13 @@ function deepFreeze<T>(value: T): T {
}
function readAuthorityRecord(filePath: string, type: string, schemaId: string): { payload: Uint8Array } {
const decoded = decode(fs.readFileSync(filePath));
if (!Array.isArray(decoded) || decoded[0] !== CULTCACHE_STORE_FORMAT || !Array.isArray(decoded[1]) || !Array.isArray(decoded[2])) {
throw new Error(`Idunn authority file ${filePath} is not a CultCache v1 store.`);
}
if (decoded[2].length !== 1) throw new Error(`Idunn authority file ${filePath} must contain exactly one record.`);
const matchingRecords = (decoded[2] as unknown[]).filter((raw) => {
const row = array(raw, 4, "CultCache record");
return row[1] === schemaId;
});
if (matchingRecords.length !== 1) throw new Error(`Idunn authority file must contain exactly one ${schemaId} record.`);
const row = array(matchingRecords[0], 4, "CultCache record");
const catalog = (decoded[1] as unknown[]).find((raw) => {
const entry = Array.isArray(raw) ? raw : [];
return entry[0] === schemaId;
});
const catalogEntry = array(catalog, 7, "CultCache schema catalog entry");
if (catalogEntry[1] !== type) throw new Error(`Idunn authority record has unexpected CultCache type ${String(catalogEntry[1])}.`);
if (!(row[3] instanceof Uint8Array)) throw new Error("Idunn authority payload is not MessagePack bytes.");
return { payload: row[3] };
const store = readSingleFileStore(filePath);
if (store.absent) throw new Error(`Idunn authority file ${filePath} does not exist.`);
if (store.envelopes.length !== 1) throw new Error(`Idunn authority file ${filePath} must contain exactly one record.`);
const record = store.envelopes[0]!;
if (record.schemaId !== schemaId) throw new Error(`Idunn authority file must contain exactly one ${schemaId} record.`);
if (record.type !== type) throw new Error(`Idunn authority record has unexpected CultCache type ${record.type}.`);
return { payload: documentPayloadBytes(record.payload) };
}
function decodeExpected(payload: Uint8Array): ExpectedIncarnation {

131
packages/cultnet-ts/src/replication.ts
View file

@ -1,15 +1,16 @@
import type {
AnyCultCacheDocumentDefinition,
CultCache,
CultCacheDocumentDefinition,
CultCacheDocumentValue,
CultCacheEnvelope,
} from "@gamecult/cultcache-ts";
import {
SchemaConflictError,
decodeDocumentPayload,
documentPayloadBytes,
documentPayloadFromBytes,
encodeDocumentPayload,
schemaIdentityOf,
} from "@gamecult/cultcache-ts";
import {
@ -42,7 +43,9 @@ export function defineCultNetDocumentBinding<
export class CultNetDocumentRegistry {
readonly #bindings = new Map<string, CultNetDocumentBinding>();
readonly #schemaBindings = new Map<string, CultNetDocumentBinding>();
// The binding that owns each schema id, and every binding listing an id as compatible, in registration order.
readonly #owners = new Map<string, CultNetDocumentBinding>();
readonly #listers = new Map<string, CultNetDocumentBinding[]>();
constructor(bindings: Iterable<CultNetDocumentBinding> = []) {
for (const binding of bindings) {
@ -51,8 +54,32 @@ export class CultNetDocumentRegistry {
}
register(binding: CultNetDocumentBinding): this {
// One definition owns a type, a schema id and a schema name, derived as CultCache derives them. Another definition
// claiming any of them is refused, so a schema never resolves to two types.
const identity = schemaIdentityOf(binding.definition);
for (const holder of this.#bindings.values()) {
if (holder.definition === binding.definition) {
continue;
}
const held = schemaIdentityOf(holder.definition);
const claimed =
holder.definition.type === binding.definition.type ? `type "${binding.definition.type}"`
: held.schemaId === identity.schemaId ? `schema id "${identity.schemaId}"`
: held.schemaName === identity.schemaName ? `schema name "${identity.schemaName}"`
: undefined;
if (claimed) {
throw new SchemaConflictError(identity.schemaId, [held.schemaName, identity.schemaName], "",
`CultNet ${claimed} is already bound to type "${holder.definition.type}"; type "${binding.definition.type}" (schema id "${identity.schemaId}") cannot also be bound.`);
}
}
this.#bindings.set(binding.definition.type, binding);
this.#schemaBindings.set(schemaIdForBinding(binding), binding);
this.#owners.set(identity.schemaId, binding);
for (const schemaId of identity.compatibleSchemaIds) {
const listers = this.#listers.get(schemaId) ?? [];
if (schemaId !== identity.schemaId && !listers.some(lister => lister.definition === binding.definition)) {
this.#listers.set(schemaId, [...listers, binding]);
}
}
return this;
}
@ -60,16 +87,22 @@ export class CultNetDocumentRegistry {
return this.#bindings.get(documentType);
}
// The binding a record under this schema id belongs to: the id's owner, else the one binding that lists it as
// compatible. An id nothing declares resolves to nothing. An id that several bindings list and none owns is refused,
// as the cache refuses it, whatever order the bindings were registered in.
getBySchemaId(schemaId: string): CultNetDocumentBinding | undefined {
const exact = this.#schemaBindings.get(schemaId);
if (exact) {
return exact;
const owner = this.#owners.get(schemaId);
if (owner) {
return owner;
}
const requestedSchemaIds = new Set([schemaId]);
return this.#uniqueBindings().find(binding =>
schemaMatchesBinding(schemaIdForBinding(binding), binding, requestedSchemaIds),
);
const listers = this.#listers.get(schemaId) ?? [];
if (listers.length > 1) {
const types = listers.map(lister => lister.definition.type).sort();
throw new SchemaConflictError(schemaId, listers.map(lister => schemaIdentityOf(lister.definition).schemaName).sort(), "",
`CultNet schema id "${schemaId}" is owned by no bound type and declared compatible by several (${types.join(", ")}), ` +
"so it resolves to none. Bind the type that owns the id, or declare it on one type.");
}
return listers[0];
}
createDocumentPutMessage<TDefinition extends AnyCultCacheDocumentDefinition>(
@ -169,7 +202,7 @@ export class CultNetDocumentRegistry {
const binding = this.#requireBinding(envelope.type);
const schemaId = schemaIdForEnvelope(envelope, binding);
if (requestedSchemaIds && !schemaMatchesBinding(schemaId, binding, requestedSchemaIds)) {
if (requestedSchemaIds && !answersTo(schemaId, binding, requestedSchemaIds)) {
continue;
}
@ -206,7 +239,7 @@ export class CultNetDocumentRegistry {
const binding = this.#requireBinding(envelope.type);
const schemaId = schemaIdForEnvelope(envelope, binding);
if (requestedSchemaIds && !schemaMatchesBinding(schemaId, binding, requestedSchemaIds)) {
if (requestedSchemaIds && !answersTo(schemaId, binding, requestedSchemaIds)) {
continue;
}
@ -274,16 +307,13 @@ export class CultNetDocumentRegistry {
binding: CultNetDocumentBinding;
envelope: CultCacheEnvelope;
} {
const resolution = this.#resolveRawDocumentBinding(message.document);
const { binding } = resolution;
const binding = this.#requireSchemaBinding(message.document.schemaId);
return {
binding,
envelope: {
key: message.document.recordKey,
type: binding.definition.type,
schemaId: resolution.preserveIncomingSchemaId
? message.document.schemaId
: schemaIdForBinding(binding),
schemaId: schemaIdForBinding(binding),
payload: documentPayloadFromBytes(new Uint8Array(documentPayloadBytes(message.document.payload))),
storedAt: message.document.storedAt,
},
@ -334,35 +364,6 @@ export class CultNetDocumentRegistry {
return binding;
}
#resolveRawDocumentBinding(document: CultNetRawDocumentRecord): {
binding: CultNetDocumentBinding;
preserveIncomingSchemaId: boolean;
} {
const exact = this.getBySchemaId(document.schemaId);
if (exact) {
return { binding: exact, preserveIncomingSchemaId: true };
}
const payload = documentPayloadFromBytes(new Uint8Array(documentPayloadBytes(document.payload)));
for (const binding of this.#uniqueBindings()) {
try {
decodeDocumentPayload(binding.definition, payload);
return { binding, preserveIncomingSchemaId: false };
} catch {
// Keep looking; a foreign schema id is acceptable only when the payload validates locally.
}
}
throw new Error(`No CultNet document binding is registered for schema "${document.schemaId}".`);
}
#uniqueBindings(): CultNetDocumentBinding[] {
return [...new Set([
...this.#schemaBindings.values(),
...this.#bindings.values(),
])];
}
#createRawDocumentRecord(envelope: CultCacheEnvelope): CultNetRawDocumentRecord {
const binding = this.#requireBinding(envelope.type);
return {
@ -376,43 +377,17 @@ export class CultNetDocumentRegistry {
}
function schemaIdForBinding(binding: CultNetDocumentBinding): string {
return binding.definition.schemaId ?? binding.definition.type;
return schemaIdentityOf(binding.definition).schemaId;
}
function schemaMatchesBinding(
// A held record matches a snapshot filter when the filter names the id it carries or any id its binding answers to.
function answersTo(
schemaId: string,
binding: CultNetDocumentBinding,
requestedSchemaIds: Set<string>,
): boolean {
if (requestedSchemaIds.has(schemaId)) {
return true;
}
const definition = binding.definition;
if (definition.schemaId && requestedSchemaIds.has(definition.schemaId)) {
return true;
}
if (definition.schemaName && requestedSchemaIds.has(definition.schemaName)) {
return true;
}
if (definition.schemaVersion && requestedSchemaIds.has(definition.schemaVersion)) {
return true;
}
if (definition.compatibleSchemaIds?.some(candidate => requestedSchemaIds.has(candidate))) {
return true;
}
return Array.from(requestedSchemaIds)
.some(candidate => inferSchemaName(candidate) === definition.schemaName);
}
function inferSchemaName(schemaId: string): string | undefined {
const marker = schemaId.lastIndexOf(".v");
if (marker <= 0 || marker + 2 >= schemaId.length) {
return undefined;
}
const version = schemaId.slice(marker + 2);
return /^\d+$/u.test(version) ? schemaId.slice(0, marker) : undefined;
return requestedSchemaIds.has(schemaId)
|| schemaIdentityOf(binding.definition).compatibleSchemaIds.some(candidate => requestedSchemaIds.has(candidate));
}
function schemaIdForEnvelope(

29
packages/cultnet-ts/test/content.test.ts
View file

@ -49,14 +49,14 @@ const HASH = "a".repeat(64);
const manifestOf = (over: Partial<CultMeshCdnArtifactManifest> = {}): CultMeshCdnArtifactManifest => ({
artifactId: "id", kind: "asset", version: "1", contentHash: HASH, sizeBytes: 4, mimeType: "text/plain", createdAtUtc: "t",
chunks: [{ chunkHash: HASH, offset: 0, sizeBytes: 4, recordKey: `mesh:cdn:chunk:${HASH}` }], tags: ["t"], metadata: [["k", "v"]],
chunks: [{ chunkHash: HASH, offset: 0, sizeBytes: 4, recordKey: `mesh:cdn:chunk:${HASH}`, id: "0" }], tags: ["t"], metadata: [["k", "v"]],
...over,
});
// Wire bytes of a ten-array with any field replaced, metadata as a plain object (no integer-like keys).
const fieldsOf = (over: Record<number, unknown> = {}) => {
const m = manifestOf();
const fields: unknown[] = [m.artifactId, m.kind, m.version, m.contentHash, m.sizeBytes, m.mimeType, m.createdAtUtc,
m.chunks.map(c => [c.chunkHash, c.offset, c.sizeBytes, c.recordKey]), m.tags, { k: "v" }];
m.chunks.map(c => [c.chunkHash, c.offset, c.sizeBytes, c.recordKey, c.id]), m.tags, { k: "v" }];
for (const [index, value] of Object.entries(over)) fields[Number(index)] = value;
return fields;
};
@ -95,7 +95,7 @@ test("the packed chunks are copies that rejoin to the body, each named by the ha
assert.deepEqual(Array.from(chunks.flatMap(c => Array.from(c.payload))), [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]);
for (const [i, c] of chunks.entries()) {
assert.equal(c.chunkHash, await sha256Hex(c.payload));
assert.deepEqual(manifest.chunks[i], { chunkHash: c.chunkHash, offset: i * 4, sizeBytes: c.payload.length, recordKey: chunkRecordKey(c.chunkHash) });
assert.deepEqual(manifest.chunks[i], { chunkHash: c.chunkHash, offset: i * 4, sizeBytes: c.payload.length, recordKey: chunkRecordKey(c.chunkHash), id: String(i * 4) });
}
const twins = await packContent("x", new Uint8Array(8), { chunkSizeBytes: 4 });
assert.equal(twins.manifest.chunks[0]!.chunkHash, twins.manifest.chunks[1]!.chunkHash, "equal chunks share a hash");
@ -216,7 +216,7 @@ test("a metadata key that is not a string names the field", () => {
});
test("validateManifest refuses negative size, a gap, an overlap and a size-sum mismatch with the C# messages", () => {
const chunk = (offset: number, sizeBytes: number) => ({ chunkHash: HASH, offset, sizeBytes, recordKey: "k" });
const chunk = (offset: number, sizeBytes: number) => ({ chunkHash: HASH, offset, sizeBytes, recordKey: "k", id: String(offset) });
const refuses = (over: Partial<CultMeshCdnArtifactManifest>, message: string) =>
assert.throws(() => validateManifest(manifestOf(over)), new Error(message));
refuses({ sizeBytes: -1, chunks: [] }, "CDN artifact manifest has a negative size.");
@ -232,7 +232,7 @@ test("validateManifest refuses negative size, a gap, an overlap and a size-sum m
});
test("validateManifest refuses a size that is not a whole number below 2^53 and a chunk size an int cannot hold, whether decoded or hand-built", () => {
const chunk = (offset: number, sizeBytes: number) => ({ chunkHash: HASH, offset, sizeBytes, recordKey: "k" });
const chunk = (offset: number, sizeBytes: number) => ({ chunkHash: HASH, offset, sizeBytes, recordKey: "k", id: String(offset) });
const refuses = (over: Partial<CultMeshCdnArtifactManifest>, message: string) =>
assert.throws(() => validateManifest(manifestOf(over)), new Error(message));
const manifestMessage = "CDN artifact manifest size is not a whole number below 2^53.";
@ -405,6 +405,23 @@ test("nil in a reference slot reads as null and writes back as nil; a missing tr
assert.deepEqual(short, { artifactId: "id", kind: "asset", version: "", contentHash: "", sizeBytes: 0, mimeType: "application/octet-stream", createdAtUtc: "", chunks: [], tags: [], metadata: [] });
});
test("a chunk reference's id is slot 4: nil reads null and writes nil, an absent slot reads \"\" and is written, packContent writes the decimal offset", async () => {
const chunk = (id: unknown[]) => [HASH, 0, 4, "k", ...id];
const read = (id: unknown[]) => decodeManifest(encode(fieldsOf({ 7: [chunk(id)] }) as unknown[], { useBigInt64: true, maxDepth: 1000 })).chunks[0]!;
const nilled = encode(fieldsOf({ 7: [chunk([null])] }), { useBigInt64: true, maxDepth: 1000 });
const nilManifest = decodeManifest(nilled);
assert.equal(nilManifest.chunks[0]!.id, null);
assert.deepEqual(encodeManifest(nilManifest), nilled);
const absent = decodeManifest(encode(fieldsOf({ 7: [[HASH, 0, 4, "k"]] }), { useBigInt64: true, maxDepth: 1000 }));
assert.equal(absent.chunks[0]!.id, "");
assert.deepEqual(decodeManifest(encodeManifest(absent)).chunks[0], { chunkHash: HASH, offset: 0, sizeBytes: 4, recordKey: "k", id: "" });
assert.equal(read(["7"]).id, "7");
assert.throws(() => read([0]), /chunks.id/);
const packed = await packContent("x", new Uint8Array(9), { chunkSizeBytes: 4 });
assert.deepEqual(packed.manifest.chunks.map(c => c.id), ["0", "4", "8"]);
assert.deepEqual((await packContent("x", new Uint8Array(0))).manifest.chunks.map(c => c.id), ["0"]);
});
// Refusals that are the same refusal in the reference but would read differently if the wrong rule fired: the message
// names the true cause (the sign, the nil chunk, the reserved code, the early end), never a neighbouring field.
test("a refusal names its true cause", () => {
@ -433,7 +450,7 @@ test("a lone surrogate is written as U+FFFD at every string length, short and lo
const withText = (value: string): CultMeshCdnArtifactManifest => ({
...base,
artifactId: value, kind: value, version: value, mimeType: value, createdAtUtc: value,
chunks: [{ ...base.chunks[0]!, recordKey: value }],
chunks: [{ ...base.chunks[0]!, chunkHash: value, recordKey: value, id: value }],
tags: [value],
metadata: [[value, value]],
});

374
packages/cultnet-ts/test/cultnet.test.ts
View file

@ -11,8 +11,10 @@ import { z } from "zod";
import { decode as decodeMsgpack, encode } from "@msgpack/msgpack";
import {
CultCache,
SchemaConflictError,
SingleFileMessagePackBackingStore,
defineDocumentType,
schemaIdentityOf,
} from "@gamecult/cultcache-ts";
import {
@ -1346,6 +1348,56 @@ test("CultNet raw replication preserves CultCache payload bytes for bit-compatib
}
});
test("CultNet raw replication under a compatible schema id leaves a store that reopens and takes the next write", async () => {
const tempDir = mkdtempSync(join(tmpdir(), "cultnetts-raw-compatible-"));
try {
const definition = defineDocumentType({
type: "tests.raw-compatible",
schema: z.object({ name: z.string() }),
schemaId: "tests.raw-compatible.current",
schemaName: "tests.raw-compatible",
schemaVersion: "tests.raw_compatible.v1",
compatibleSchemaIds: ["tests.raw-compatible.current", "tests.raw-compatible.older"],
});
const registry = new CultNetDocumentRegistry([
defineCultNetDocumentBinding({ definition, payloadSchemaVersion: "tests.raw_compatible.v1" }),
]);
const open = async (): Promise<CultCache> => {
const cache = CultCache.builder()
.withDocumentType(definition)
.withGenericStore(new SingleFileMessagePackBackingStore(join(tempDir, "target.msgpack")))
.build();
await cache.pullAllBackingStores();
return cache;
};
const origin = CultCache.builder().withDocumentType(definition).build();
await origin.put(definition, "seed", { name: "seed" });
const payload = origin.getRequiredEnvelope(definition, "seed").payload;
const target = await open();
await registry.applyRawDocumentPutMessage(target, {
schemaVersion: "cultnet.document_put_raw.v0",
messageId: "raw-compatible-1",
document: {
schemaId: "tests.raw-compatible.older",
recordKey: "replicated",
storedAt: "2026-09-30T00:00:00.0000000Z",
payloadEncoding: "messagepack",
payload,
},
} as Parameters<CultNetDocumentRegistry["applyRawDocumentPutMessage"]>[1]);
const reopened = await open();
assert.deepEqual(reopened.getRequired(definition, "replicated"), { name: "seed" });
await reopened.put(definition, "next", { name: "next" });
assert.deepEqual((await open()).getRequired(definition, "next"), { name: "next" });
} finally {
rmSync(tempDir, { recursive: true, force: true });
}
});
test("operation envelopes preserve typed service routing and payload correlation", () => {
const request = parseCultNetMessage({
schemaVersion: "cultnet.operation_request.v0",
@ -1702,162 +1754,204 @@ test("rudp sequence-neutral acknowledgements interoperate with ordered receivers
assert.deepEqual(receiver.receive(request).delivered.map(frame => Buffer.from(frame.payload).toString("utf8")), ["snapshot"]);
});
test("CultNet raw replication applies compatible foreign-schema snapshots", async () => {
const tempDir = mkdtempSync(join(tmpdir(), "cultnetts-foreign-schema-raw-"));
// A replicated record whose schema id no binding declares is refused, whatever its payload parses as: the registry never
// guesses a type from a payload's shape and never relabels a record it does not understand.
test("CultNet raw replication refuses a record under a schema id no binding declares, whatever its shape", async () => {
const note = defineDocumentType({
type: "p5.note", schemaId: "p5.note.v1", schemaName: "p5.note", schema: z.object({ body: z.string() }),
});
// Registered first, and accepts every payload: shape routing would store the record under it.
const anything = defineDocumentType({ type: "p5.anything", schema: z.any() });
const registry = new CultNetDocumentRegistry([
defineCultNetDocumentBinding({ definition: anything }),
defineCultNetDocumentBinding({ definition: note }),
]);
const origin = CultCache.builder().withDocumentType(note).build();
await origin.put(note, "k", { body: "from a peer" });
const snapshot = registry.createRawSnapshotResponse(origin, "p5");
assert.equal(snapshot.documents[0]?.schemaId, "p5.note.v1");
assert.equal(snapshot.documents[0]?.payloadEncoding, "messagepack");
try {
const documentDefinition = defineDocumentType({
type: "cultnet.note",
schemaId: "cultnet.note.v1",
schemaName: "cultnet.note",
schemaVersion: "cultnet.note.v1",
schema: z.object({
schema_version: z.string(),
noteId: z.string(),
body: z.string(),
for (const schemaId of ["runtime.generated.p5.note.ui.99", "p5.note.v7", "p5.note"]) {
const target = CultCache.builder().withDocumentType(note).withDocumentType(anything).build();
await assert.rejects(
registry.applyRawSnapshotResponse(target, {
...snapshot,
documents: snapshot.documents.map(document => ({ ...document, schemaId })),
}),
name: "noteId",
});
const registry = new CultNetDocumentRegistry([
defineCultNetDocumentBinding({ definition: documentDefinition }),
]);
const originCache = CultCache.builder()
.withDocumentType(documentDefinition)
.withGenericStore(new SingleFileMessagePackBackingStore(join(tempDir, "origin.msgpack")))
.build();
const targetCache = CultCache.builder()
.withDocumentType(documentDefinition)
.withGenericStore(new SingleFileMessagePackBackingStore(join(tempDir, "target.msgpack")))
.build();
await originCache.put(documentDefinition, "note:foreign-schema", {
schema_version: "cultnet.note.v1",
noteId: "note:foreign-schema",
body: "accepted by payload shape",
});
const rawSnapshot = registry.createRawSnapshotResponse(originCache, "foreign-schema-raw");
rawSnapshot.documents[0]!.schemaId = "runtime.generated.cultnet.note.ui.99";
await registry.applyRawSnapshotResponse(targetCache, rawSnapshot);
assert.equal(
targetCache.getRequired(documentDefinition, "note:foreign-schema").body,
"accepted by payload shape",
(error: unknown) => error instanceof Error
&& error.message === `No CultNet document binding is registered for schema "${schemaId}".`,
);
assert.equal(
targetCache.getRequiredEnvelope(documentDefinition, "note:foreign-schema").schemaId,
"cultnet.note.v1",
);
} finally {
rmSync(tempDir, { recursive: true, force: true });
assert.deepEqual(target.snapshot(), [], `nothing was stored for ${schemaId}`);
}
});
test("CultNet document registry filters snapshots by schema aliases", async () => {
const tempDir = mkdtempSync(join(tmpdir(), "cultnetts-alias-snapshot-"));
// The registry stamps and resolves a definition's schema id exactly as CultCache does, so a record a cache wrote is read
// back as the type that wrote it on every peer.
test("CultNet registry derives a definition's schema id as the cache does, and refuses what the cache refuses", async () => {
const shape = z.object({ body: z.string() });
const a = defineDocumentType({ type: "p1.a", schemaName: "p1.shared", schema: shape });
const b = defineDocumentType({ type: "p1.b", schemaId: "p1.shared", schemaName: "p1.b", schema: shape });
const fillers = [0, 1].map(index => defineCultNetDocumentBinding({
definition: defineDocumentType({ type: `p1.filler${index}`, schema: shape }),
}));
try {
const documentDefinition = defineDocumentType({
type: "cultnet.alias_note",
schemaId: "sha256:cultnet-alias-note",
schemaName: "cultnet.alias_note",
schemaVersion: "cultnet.alias_note.v1",
schema: z.object({
schema_version: z.string(),
noteId: z.string(),
body: z.string(),
}),
name: "noteId",
assert.deepEqual(schemaIdentityOf(a), {
schemaId: "p1.shared", schemaName: "p1.shared", schemaVersion: "p1.shared.v1", compatibleSchemaIds: ["p1.shared"],
});
// The cache refuses B beside A: A's records carry "p1.shared".
await assert.rejects(CultCache.builder().withDocumentType(a).build().registerDocumentType(b), SchemaConflictError);
for (const [first, second] of [[a, b], [b, a]]) {
// Two bindings stand ahead of the holder, so a check that looks only at the first holder misses it.
const registry = new CultNetDocumentRegistry([...fillers, defineCultNetDocumentBinding({ definition: first })]);
assert.throws(() => registry.register(defineCultNetDocumentBinding({ definition: second })), (error: unknown) => {
assert.ok(error instanceof SchemaConflictError, String(error));
assert.equal(error.schemaId, "p1.shared");
return true;
});
const registry = new CultNetDocumentRegistry([
defineCultNetDocumentBinding({ definition: documentDefinition }),
]);
const cache = CultCache.builder()
.withDocumentType(documentDefinition)
.withGenericStore(new SingleFileMessagePackBackingStore(join(tempDir, "alias.msgpack")))
.build();
await cache.put(documentDefinition, "note:alias", {
schema_version: "cultnet.alias_note.v1",
noteId: "note:alias",
body: "filtered by alias",
});
const typedSnapshot = registry.createSnapshotResponse(cache, "typed-alias", {
schemaVersion: "cultnet.snapshot_request.v0",
messageId: "typed-alias",
schemaIds: ["cultnet.alias_note.v1"],
});
const rawSnapshot = registry.createRawSnapshotResponse(cache, "raw-alias", {
schemaVersion: "cultnet.snapshot_request.v0",
messageId: "raw-alias",
schemaIds: ["cultnet.alias_note.v1"],
});
assert.equal(typedSnapshot.documents[0]?.schemaId, "sha256:cultnet-alias-note");
assert.equal(typedSnapshot.documents[0]?.recordKey, "note:alias");
assert.equal(rawSnapshot.documents[0]?.schemaId, "sha256:cultnet-alias-note");
assert.equal(rawSnapshot.documents[0]?.recordKey, "note:alias");
} finally {
rmSync(tempDir, { recursive: true, force: true });
assert.equal(registry.get(second.type), undefined);
}
const registry = new CultNetDocumentRegistry([...fillers, defineCultNetDocumentBinding({ definition: a })]);
const origin = CultCache.builder().withDocumentType(a).build();
await origin.put(a, "k", { body: "written by a" });
const stamped = origin.getRequiredEnvelope(a, "k").schemaId;
assert.equal(stamped, "p1.shared");
assert.equal(registry.createDocumentPutMessage(registry.get("p1.a")!, "m", "k", { body: "x" }).document.schemaId, stamped);
assert.equal(registry.createRawDocumentPutMessage(registry.get("p1.a")!, "m", "k", { body: "x" }).document.schemaId, stamped);
assert.equal(registry.getBySchemaId(stamped!)?.definition, a);
const peer = CultCache.builder().withDocumentType(a).build();
await registry.applyRawSnapshotResponse(peer, registry.createRawSnapshotResponse(origin, "p1"));
assert.deepEqual(peer.getRequired(a, "k"), { body: "written by a" });
assert.equal(peer.getRequiredEnvelope(a, "k").schemaId, stamped);
});
test("CultNet document registry applies typed put and delete messages by schema alias", async () => {
const tempDir = mkdtempSync(join(tmpdir(), "cultnetts-alias-typed-"));
// An id nobody owns resolves to the one binding that lists it as compatible. Several listers and no owner resolve to
// none, and the refusal does not depend on the order they were registered in. An owner beats every lister.
test("CultNet registry refuses an unowned schema id that several bindings list, in any order", async () => {
const shape = z.object({ body: z.string() });
const first = defineDocumentType({ type: "p2.first", schemaId: "p2.first", compatibleSchemaIds: ["p2.old"], schema: shape });
const second = defineDocumentType({ type: "p2.second", schemaId: "p2.second", compatibleSchemaIds: ["p2.old"], schema: shape });
const owner = defineDocumentType({ type: "p2.owner", schemaId: "p2.old", schema: shape });
const bind = (definition: typeof first) => defineCultNetDocumentBinding({ definition });
try {
const documentDefinition = defineDocumentType({
type: "cultnet.typed_alias_note",
schemaId: "sha256:cultnet-typed-alias-note",
schemaName: "cultnet.typed_alias_note",
schemaVersion: "cultnet.typed_alias_note.v1",
schema: z.object({
schema_version: z.string(),
noteId: z.string(),
body: z.string(),
}),
name: "noteId",
for (const order of [[first, second], [second, first]]) {
const registry = new CultNetDocumentRegistry(order.map(bind));
assert.throws(() => registry.getBySchemaId("p2.old"), (error: unknown) => {
assert.ok(error instanceof SchemaConflictError, String(error));
assert.equal(error.schemaId, "p2.old");
assert.deepEqual(error.schemaNames, ["p2.first", "p2.second"]);
assert.equal(error.recordKey, "");
assert.ok(error.message.includes("(p2.first, p2.second)"), error.message);
assert.ok(error.message.includes("so it resolves to none. Bind the type that owns the id"), error.message);
return true;
});
const registry = new CultNetDocumentRegistry([
defineCultNetDocumentBinding({ definition: documentDefinition }),
]);
const cache = CultCache.builder()
.withDocumentType(documentDefinition)
.withGenericStore(new SingleFileMessagePackBackingStore(join(tempDir, "typed-alias.msgpack")))
.build();
const cache = CultCache.builder().withDocumentType(first).withDocumentType(second).build();
const origin = CultCache.builder().withDocumentType(first).build();
await origin.put(first, "k", { body: "old" });
const document = { ...registry.createRawSnapshotResponse(origin, "p2").documents[0]!, schemaId: "p2.old" };
await assert.rejects(
registry.applyRawDocumentPutMessage(cache, { schemaVersion: "cultnet.document_put_raw.v0", messageId: "p2", document }),
SchemaConflictError,
);
assert.deepEqual(cache.snapshot(), []);
await registry.applyDocumentPutMessage(cache, {
for (const withOwner of [[...order, owner], [owner, ...order]]) {
assert.equal(new CultNetDocumentRegistry(withOwner.map(bind)).getBySchemaId("p2.old")?.definition, owner);
}
}
// One lister, bound twice, is still one lister.
const single = new CultNetDocumentRegistry([bind(first), bind(first)]);
assert.equal(single.getBySchemaId("p2.old")?.definition, first);
// A snapshot filtered by an id a binding lists returns that binding's records under their own id.
const origin = CultCache.builder().withDocumentType(first).build();
await origin.put(first, "k", { body: "listed" });
const filter = { schemaVersion: "cultnet.snapshot_request.v0" as const, messageId: "f", schemaIds: ["p2.old"] };
assert.deepEqual(single.createRawSnapshotResponse(origin, "f", filter).documents.map(document => document.schemaId), ["p2.first"]);
assert.deepEqual(single.createSnapshotResponse(origin, "f", filter).documents.map(document => document.schemaId), ["p2.first"]);
});
// A schema id is only ever an id a definition declares. A version string, a schema name, or an id that differs from a
// declared one by its version suffix names nothing.
test("CultNet registry resolves only declared schema ids, never a version, a name or an inferred version", async () => {
const note = defineDocumentType({
type: "p3.note",
schemaId: "p3.note.v1",
schemaName: "p3.note",
schemaVersion: "p4.note.v3",
schema: z.object({ body: z.string() }),
});
const registry = new CultNetDocumentRegistry([defineCultNetDocumentBinding({ definition: note })]);
const cache = CultCache.builder().withDocumentType(note).build();
await cache.put(note, "k", { body: "held" });
assert.equal(registry.getBySchemaId("p3.note.v1")?.definition, note);
for (const undeclared of ["p3.note.v7", "p4.note.v3", "p3.note"]) {
assert.equal(registry.getBySchemaId(undeclared), undefined, undeclared);
const filter = { schemaVersion: "cultnet.snapshot_request.v0" as const, messageId: "f", schemaIds: [undeclared] };
assert.deepEqual(registry.createSnapshotResponse(cache, "f", filter).documents, [], undeclared);
assert.deepEqual(registry.createRawSnapshotResponse(cache, "f", filter).documents, [], undeclared);
await assert.rejects(registry.applyDocumentPutMessage(cache, {
schemaVersion: "cultnet.document_put.v0",
messageId: "typed-alias-put",
document: {
schemaId: "cultnet.typed_alias_note.v1",
recordKey: "note:typed-alias",
storedAt: "2026-06-27T00:00:00.000Z",
payload: {
schema_version: "cultnet.typed_alias_note.v1",
noteId: "note:typed-alias",
body: "typed alias applied",
},
},
});
assert.equal(
cache.getRequired(documentDefinition, "note:typed-alias").body,
"typed alias applied",
);
assert.equal(registry.getBySchemaId("cultnet.typed_alias_note.v1")?.definition.type, documentDefinition.type);
assert.equal(await registry.applyDocumentDeleteMessage(cache, {
schemaVersion: "cultnet.document_delete.v0",
messageId: "typed-alias-delete",
schemaId: "cultnet.typed_alias_note.v1",
recordKey: "note:typed-alias",
}), true);
assert.equal(cache.get(documentDefinition, "note:typed-alias"), undefined);
} finally {
rmSync(tempDir, { recursive: true, force: true });
messageId: "put",
document: { schemaId: undeclared, recordKey: "k", storedAt: "2026-09-30T00:00:00.000Z", payload: { body: "relabelled" } },
}), /No CultNet document binding/);
await assert.rejects(registry.applyDocumentDeleteMessage(cache, {
schemaVersion: "cultnet.document_delete.v0", messageId: "delete", schemaId: undeclared, recordKey: "k",
}), /No CultNet document binding/);
assert.deepEqual(cache.getRequired(note, "k"), { body: "held" });
}
const filter = { schemaVersion: "cultnet.snapshot_request.v0" as const, messageId: "f", schemaIds: ["p3.note.v1"] };
assert.equal(registry.createSnapshotResponse(cache, "f", filter).documents.length, 1);
assert.equal(registry.createRawSnapshotResponse(cache, "f", filter).documents.length, 1);
});
test("CultNet document registry binds one type per schema and refuses a second, typed", () => {
const noteShape = z.object({ noteId: z.string() });
const owner = defineDocumentType({ type: "tests.note", schemaId: "id.note", schemaName: "tests.note", schema: noteShape });
const sameId = defineDocumentType({ type: "tests.note-ui", schemaId: "id.note", schemaName: "tests.note-ui", schema: noteShape });
const sameName = defineDocumentType({ type: "tests.note-named", schemaId: "id.other", schemaName: "tests.note", schema: noteShape });
const sameType = defineDocumentType({ type: "tests.note", schemaId: "id.third", schemaName: "tests.third", schema: noteShape });
const lister = defineDocumentType({
type: "tests.note-lister", schemaId: "id.lister", schemaName: "tests.note-lister", compatibleSchemaIds: ["id.note"], schema: noteShape,
});
for (const [claimant, schemaId, names] of [
[sameId, "id.note", ["tests.note", "tests.note-ui"]],
[sameName, "id.other", ["tests.note", "tests.note"]],
[sameType, "id.third", ["tests.note", "tests.third"]],
] as const) {
for (const order of [[owner, claimant], [claimant, owner]]) {
// Two bindings are registered ahead of the holder, so a conflict loop that checks only the first holder fails.
const registry = new CultNetDocumentRegistry([
...["tests.filler-a", "tests.filler-b"].map(type => defineCultNetDocumentBinding({
definition: defineDocumentType({ type, schema: noteShape }),
})),
defineCultNetDocumentBinding({ definition: order[0] }),
]);
assert.throws(() => registry.register(defineCultNetDocumentBinding({ definition: order[1] })), (error: unknown) => {
assert.ok(error instanceof SchemaConflictError, String(error));
const ownerFirst = order[0] === owner;
assert.equal(error.schemaId, ownerFirst ? schemaId : "id.note");
assert.deepEqual(error.schemaNames, ownerFirst ? names : [...names].reverse());
assert.equal(error.recordKey, "");
assert.ok(error.message.includes(`schema id "${error.schemaId}"`), error.message);
return true;
});
// The refused binding left nothing behind: the schema still resolves to the first type.
assert.equal(registry.getBySchemaId(order[0].schemaId!)?.definition, order[0]);
assert.equal(registry.get(order[1].type)?.definition, order[1].type === order[0].type ? order[0] : undefined);
}
}
// The same definition may be bound again, and a definition listing the id as compatible is not a second owner.
const registry = new CultNetDocumentRegistry([defineCultNetDocumentBinding({ definition: owner })]);
registry.register(defineCultNetDocumentBinding({ definition: owner, payloadSchemaVersion: "tests.note.v2" }));
registry.register(defineCultNetDocumentBinding({ definition: lister }));
assert.equal(registry.get("tests.note")?.payloadSchemaVersion, "tests.note.v2");
assert.equal(registry.getBySchemaId("id.note")?.definition, owner);
assert.equal(registry.getBySchemaId("id.lister")?.definition, lister);
});
test("CultNet interop slot compatibility defaults missing trailing fields and rejects mismatched slots", () => {

87
packages/cultnet-ts/test/idunn-runtime-authority.test.ts
View file

@ -3,8 +3,9 @@ import assert from "node:assert/strict";
import { spawnSync } from "node:child_process";
import dgram from "node:dgram";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import { documentPayloadBytes, documentPayloadFromBytes, type DocumentPayload } from "@gamecult/cultcache-ts";
import { StoreUnreadableError, documentPayloadBytes, documentPayloadFromBytes, type DocumentPayload } from "@gamecult/cultcache-ts";
import { decode, encode } from "@msgpack/msgpack";
import {
@ -55,7 +56,7 @@ function expectedFacts(set: FixtureSet) {
function openAuthority(
context: TestContext,
set: FixtureSet,
options: { machineId?: "etc" | "dbus"; lease?: string } = {},
options: { machineId?: "etc" | "dbus"; lease?: string; bundle?: string } = {},
): IdunnRuntimeAuthority {
const facts = expectedFacts(set);
const originalEnvironment = { ...process.env };
@ -63,7 +64,7 @@ function openAuthority(
for (const key of Object.keys(process.env)) if (!(key in originalEnvironment)) delete process.env[key];
Object.assign(process.env, originalEnvironment);
});
process.env.GAMECULT_IDUNN_RUNTIME_BUNDLE = path.join(FIXTURES, set);
process.env.GAMECULT_IDUNN_RUNTIME_BUNDLE = options.bundle ?? path.join(FIXTURES, set);
process.env.GAMECULT_IDUNN_CANDIDATE_BIND = facts.candidate;
if (options.lease) process.env.GAMECULT_IDUNN_PROCESS_WRITE_LEASE = path.join(FIXTURES, set, options.lease);
else delete process.env.GAMECULT_IDUNN_PROCESS_WRITE_LEASE;
@ -85,6 +86,22 @@ function openAuthority(
return loadIdunnRuntimeAuthorityFromEnvironment(facts.target, facts.healthContract);
}
/** A copy of one Rust-written bundle in a scratch directory, with `change` applied to the bytes of the named file. */
function bundleWith(context: TestContext, set: FixtureSet, file: string, change: (bytes: Buffer) => Buffer | undefined): string {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "idunn-authority-"));
context.after(() => fs.rmSync(dir, { recursive: true, force: true }));
for (const name of ["expected.cc", "activation.cc"]) {
const bytes = fixtureFile(set, name);
const changed = name === file ? change(bytes) : bytes;
if (changed !== undefined) fs.writeFileSync(path.join(dir, name), changed);
}
return dir;
}
function withHeader(header: string): (bytes: Buffer) => Buffer {
return (bytes) => Buffer.from(encode([header, ...(decode(bytes) as unknown[]).slice(1)]));
}
function signerFor(context: TestContext, set: FixtureSet, capacityDelta = 0, lease?: string) {
const authority = openAuthority(context, set, { lease });
const capabilities = expectedFacts(set).capabilities.map((capability) => ({ ...capability, capacity: capability.capacity + capacityDelta }));
@ -134,6 +151,70 @@ test("rejects a credential file holding more than one envelope", (context) => {
);
});
// The authority files are read by cultcache-ts's one store reader, so a store it reads is read here and a file it
// refuses is refused here with the same typed error and the same text.
for (const file of ["expected.cc", "activation.cc"]) {
test(`${file}: a store marked for element ids opens, as the one reader reads it`, (context) => {
const bundle = bundleWith(context, "web", file, withHeader("cultcache.store.v3"));
assert.equal(openAuthority(context, "web", { bundle }).expected.target, expectedFacts("web").target);
});
test(`${file}: a zero-byte file is refused as unreadable, not read as an empty store or a missing one`, (context) => {
const bundle = bundleWith(context, "web", file, () => Buffer.alloc(0));
assert.throws(() => openAuthority(context, "web", { bundle }), StoreUnreadableError);
});
test(`${file}: a file that is not there is refused`, (context) => {
const bundle = bundleWith(context, "web", file, () => undefined);
assert.throws(() => openAuthority(context, "web", { bundle }), /does not exist/);
});
test(`${file}: a refused header is never echoed, whether or not it starts at the store prefix`, (context) => {
for (const header of ["cultcache.store.SECRET", "cultcache.store.SECRETcultcache.store.v9", "SECRETcultcache.store.v9", "cultcache.store.v9"]) {
const bundle = bundleWith(context, "web", file, withHeader(header));
assert.throws(
() => openAuthority(context, "web", { bundle }),
(error: Error) => error instanceof StoreUnreadableError && (header === "cultcache.store.v9" ? error.message.includes(header) : !error.message.includes("SECRET")),
header,
);
}
});
}
// What an authority file may hold once the one reader has accepted it: exactly one record, under the schema
// id and the type the loader asked for. Each case edits the Rust-written store and changes one thing.
function withStore(change: (store: unknown[][]) => void): (bytes: Buffer) => Buffer {
return (bytes) => {
const store = decode(bytes) as unknown[][];
change(store);
return Buffer.from(encode(store));
};
}
test("an authority file holding two records is refused", (context) => {
const bundle = bundleWith(context, "web", "expected.cc", withStore((store) => {
const second = [...(store[2]![0] as unknown[])];
second[0] = "second";
store[2]!.push(second);
}));
assert.throws(() => openAuthority(context, "web", { bundle }), /must contain exactly one record/);
});
test("an authority file whose one record is under another schema id is refused", (context) => {
const bundle = bundleWith(context, "web", "expected.cc", () => fixtureFile("web", "activation.cc"));
assert.throws(
() => openAuthority(context, "web", { bundle }),
(error: Error) => /must contain exactly one .+ record/.test(error.message) && !error.message.includes("unexpected CultCache type"),
);
});
test("an authority record whose catalog names another type is refused, naming that type", (context) => {
const bundle = bundleWith(context, "web", "expected.cc", withStore((store) => {
(store[1]![0] as unknown[])[1] = "idunn.other_type";
}));
assert.throws(() => openAuthority(context, "web", { bundle }), /unexpected CultCache type idunn.other_type/);
});
function presenceSlot(document: { payload: DocumentPayload }, slot: number): unknown {
return (decode(documentPayloadBytes(document.payload)) as unknown[])[slot];
}

7
packages/cultnet-ts/test/interop/cultnet-interop.test.ts
View file

@ -1286,9 +1286,9 @@ test("CultNet TS/Rust/C#/Python peers discover each other and exchange raw state
schemaVersion: INTEROP_SCHEMA_VERSION,
schemaId: interopNoteSchemaId,
},
[{ role: "payload", schemaId: interopNoteSchemaId, schemaVersion: INTEROP_SCHEMA_VERSION }],
[{ role: "log", uri: "cultcache://bundle:ts-python-witness/log", mediaType: "text/plain" }],
[{ stage: "roundtrip", startedAt: "2026-06-13T00:00:03Z", completedAt: "2026-06-13T00:00:04Z", latencyMs: 1 }],
[{ id: "c0e1a2b3c4d5", role: "payload", schemaId: interopNoteSchemaId, schemaVersion: INTEROP_SCHEMA_VERSION }],
[{ id: "a0e1a2b3c4d5", role: "log", uri: "cultcache://bundle:ts-python-witness/log", mediaType: "text/plain" }],
[{ id: "e0e1a2b3c4d5", stage: "roundtrip", startedAt: "2026-06-13T00:00:03Z", completedAt: "2026-06-13T00:00:04Z", latencyMs: 1 }],
{ pipelineId: "interop", runId: "ts-python-witness", runtimeId: "ts-witness", agentId: "ts-agent" },
]),
sourceRuntimeId: "ts-witness",
@ -1306,6 +1306,7 @@ test("CultNet TS/Rust/C#/Python peers discover each other and exchange raw state
assert.equal(witnessSlots[1], "interop-proof");
assert.equal(witnessSlots[3].subjectId, "note:python-peer");
assert.equal(witnessSlots[7].runtimeId, "ts-witness");
assert.deepEqual([witnessSlots[4][0].id, witnessSlots[5][0].id, witnessSlots[6][0].id], ["c0e1a2b3c4d5", "a0e1a2b3c4d5", "e0e1a2b3c4d5"], "element ids cross TS to Python and back");
const expectedPeers = ["csharp-peer", "python-peer", "rust-peer", "ts-peer"];

155
scripts/test-declared-dependency-ranges.mjs Normal file
View file

@ -0,0 +1,155 @@
// Observes the dependency ranges the TypeScript workspace packages declare, not the workspace symlinks that satisfy them.
//
// Every workspace test resolves a sibling package through node_modules/<name> -> packages/<dir>, so a package that starts
// importing a new export from a sibling passes whatever range it declares. A consumer outside the workspace installs the
// newest published version the range admits. This check builds every tagged release (`<dir>-v<version>`) that a declared
// range admits and requires each runtime value the dependent imports from that package to be an export of it.
//
// node scripts/test-declared-dependency-ranges.mjs
//
// It needs the workspace installed (typescript at the repo root) and git with the release tags. A package whose range
// admits no tagged release other than the workspace's own passes, because nothing a consumer could install lacks the export.
// Namespace imports, default imports, type-only imports and subpath imports are not checked.
import assert from "node:assert/strict";
import { execFileSync } from "node:child_process";
import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, symlinkSync } from "node:fs";
import { createRequire } from "node:module";
import { tmpdir } from "node:os";
import { dirname, join, resolve } from "node:path";
import { fileURLToPath } from "node:url";
const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), "..");
const rootRequire = createRequire(join(repoRoot, "package.json"));
const ts = rootRequire("typescript");
const semver = createRequire(join(npmRoot(), "package.json"))("semver");
// npm ships semver, so the check needs no dependency of its own. npm's root is beside the node binary on Windows and under lib/ on Unix.
function npmRoot() {
const candidates = [
...(process.env.npm_execpath ? [dirname(dirname(process.env.npm_execpath))] : []),
join(dirname(process.execPath), "node_modules", "npm"),
join(dirname(process.execPath), "..", "lib", "node_modules", "npm"),
];
const root = candidates.find((candidate) => existsSync(join(candidate, "package.json")));
assert.ok(root, `npm's package root was not found in ${candidates.join(", ")}`);
return root;
}
function readJson(path) {
return JSON.parse(readFileSync(path, "utf8"));
}
function git(...args) {
return execFileSync("git", args, { cwd: repoRoot, encoding: "utf8" });
}
// The runtime names `sourceText` imports from, or re-exports from, exactly `packageName`.
function importedNames(sourceText, packageName) {
const file = ts.createSourceFile("source.ts", sourceText, ts.ScriptTarget.Latest, true);
const names = new Set();
for (const statement of file.statements) {
const from = statement.moduleSpecifier;
if (!from || !ts.isStringLiteral(from) || from.text !== packageName) {
continue;
}
let clause;
if (ts.isImportDeclaration(statement) && statement.importClause && !statement.importClause.isTypeOnly) {
clause = statement.importClause.namedBindings;
} else if (ts.isExportDeclaration(statement) && !statement.isTypeOnly) {
clause = statement.exportClause;
}
if (clause && (ts.isNamedImports(clause) || ts.isNamedExports(clause))) {
for (const element of clause.elements) {
if (!element.isTypeOnly) {
names.add((element.propertyName ?? element.name).text);
}
}
}
}
return [...names].sort();
}
function sourceFiles(directory) {
return readdirSync(directory, { withFileTypes: true }).flatMap((entry) => {
const path = join(directory, entry.name);
if (entry.isDirectory()) {
return sourceFiles(path);
}
return entry.name.endsWith(".ts") && !entry.name.endsWith(".d.ts") ? [path] : [];
});
}
// Builds the package as the tag released it, in a scratch directory that borrows the workspace's node_modules.
function buildRelease(directory, tag, scratch) {
const archive = join(scratch, `${tag}.tar`);
git("archive", "--format=tar", "-o", archive, tag, `packages/${directory}`);
const extracted = join(scratch, tag);
mkdirSync(extracted);
execFileSync("tar", ["-xf", archive, "-C", extracted]);
symlinkSync(join(repoRoot, "node_modules"), join(extracted, "node_modules"), "junction");
const packageRoot = join(extracted, "packages", directory);
execFileSync(process.execPath, [rootRequire.resolve("typescript/bin/tsc"), "-p", "tsconfig.json"], { cwd: packageRoot, stdio: "inherit" });
return rootRequire(join(packageRoot, "dist", "index.js"));
}
const workspaces = readJson(join(repoRoot, "package.json")).workspaces.map((directory) => {
const manifest = readJson(join(repoRoot, directory, "package.json"));
return { directory: directory.replace(/^packages\//u, ""), root: join(repoRoot, directory), manifest };
});
const byName = new Map(workspaces.map((workspace) => [workspace.manifest.name, workspace]));
const tags = git("tag", "-l").split(/\r?\n/u).filter(Boolean);
assert.ok(tags.length > 0, "this clone has no release tags; fetch them (git fetch --tags) so the admitted releases can be built");
const scratch = mkdtempSync(join(tmpdir(), "cultlib-declared-ranges-"));
const built = new Map();
let checked = 0;
try {
for (const dependent of workspaces) {
for (const [name, range] of Object.entries(dependent.manifest.dependencies ?? {})) {
const dependency = byName.get(name);
if (!dependency) {
continue;
}
assert.ok(
semver.satisfies(dependency.manifest.version, range),
`${dependent.manifest.name} declares ${name} ${range}, which excludes the workspace's own ${dependency.manifest.version}`,
);
const src = join(dependent.root, "src");
const imported = existsSync(src)
? [...new Set(sourceFiles(src).flatMap((file) => importedNames(readFileSync(file, "utf8"), name)))].sort()
: [];
const prefix = `${dependency.directory}-v`;
const admitted = tags.filter((tag) => {
const version = tag.startsWith(prefix) ? tag.slice(prefix.length) : undefined;
return version !== undefined && semver.valid(version) !== null && semver.satisfies(version, range) && version !== dependency.manifest.version;
});
for (const tag of admitted) {
if (!built.has(tag)) {
built.set(tag, buildRelease(dependency.directory, tag, scratch));
}
const exports = built.get(tag);
const missing = imported.filter((symbol) => !(symbol in exports));
assert.deepEqual(
missing,
[],
`${dependent.manifest.name} declares ${name} ${range}, which admits ${tag}, and ${tag} does not export ${missing.join(", ")}`,
);
}
checked += 1;
console.log(`${dependent.manifest.name} -> ${name} ${range}: ${imported.length} imported names, ${admitted.length} admitted tagged releases checked`);
}
}
assert.ok(checked > 0, "no workspace package declares a dependency on another workspace package");
console.log(`Declared dependency ranges: ${checked} checked`);
} finally {
rmSync(scratch, { recursive: true, force: true });
}

4
src/GameCult.Caching.MessagePack/CultCacheMessagePack.cs
View file

@ -23,10 +23,10 @@ namespace GameCult.Caching.MessagePack
// options, resolvers included, so dictionary keys compare by the bytes the store writes; a document type
// deserializes as a registered document, a member type as itself.
public static CultCodec CreateCodec(CultDocumentRegistry registry) => new CultCodec(
(value, type, document) => global::MessagePack.MessagePackSerializer.Serialize(type, value, CultDocumentMessagePackSerialization.OptionsFor(document.Assembly)),
(value, type, document) => global::MessagePack.MessagePackSerializer.Serialize(type, value, CultDocumentMessagePackSerialization.OptionsForDocument(document)),
(type, document, bytes) => type == document
? CultDocumentMessagePackSerialization.DeserializeUntyped(type, bytes, registry)
: global::MessagePack.MessagePackSerializer.Deserialize(type, bytes, CultDocumentMessagePackSerialization.OptionsFor(document.Assembly)),
: global::MessagePack.MessagePackSerializer.Deserialize(type, bytes, CultDocumentMessagePackSerialization.OptionsForDocument(document)),
CultDocumentMessagePackSerialization.OverlaySlots);
// The inspection model over that codec.

369
src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs
View file

@ -1,4 +1,5 @@
using System;
using System.IO;
using System.Buffers;
using System.Collections.Concurrent;
using System.Collections.Generic;
@ -37,11 +38,23 @@ public static class CultDocumentMessagePackSerialization
private const int SchemaCatalogMemberFieldCount = 8;
private const int StoreSnapshotFieldCount = 3;
public static readonly MessagePackSerializerOptions Options = Compose(Array.Empty<IFormatterResolver>());
private static readonly (MessagePackSerializerOptions Options, IFormatterResolver[] Ahead) Plain = Compose(Array.Empty<IFormatterResolver>());
private static readonly ConcurrentDictionary<Assembly, MessagePackSerializerOptions> AssemblyOptions = new();
public static readonly MessagePackSerializerOptions Options = Plain.Options;
public static MessagePackSerializerOptions OptionsFor(Assembly documentAssembly)
// Per assembly: the options a document type's bytes are written with, and the resolvers composed ahead of
// StandardResolver, the ones that can claim a type before its [Key] layout does.
private static readonly ConcurrentDictionary<Assembly, (MessagePackSerializerOptions Options, IFormatterResolver[] Ahead)> AssemblyOptions = new();
// Per document type: why its element ids cannot reach disk under the composed resolvers, or null. Types and resolvers
// are fixed for the process, so this is a cache of a pure answer.
private static readonly ConcurrentDictionary<Type, string?> DocumentProblems = new();
private static readonly MethodInfo GetFormatterMethod = typeof(IFormatterResolver).GetMethod(nameof(IFormatterResolver.GetFormatter))!;
public static MessagePackSerializerOptions OptionsFor(Assembly documentAssembly) => ComposedFor(documentAssembly).Options;
private static (MessagePackSerializerOptions Options, IFormatterResolver[] Ahead) ComposedFor(Assembly documentAssembly)
{
if (documentAssembly == null) throw new ArgumentNullException(nameof(documentAssembly));
return AssemblyOptions.GetOrAdd(documentAssembly, static assembly =>
@ -49,19 +62,118 @@ public static class CultDocumentMessagePackSerialization
var resolvers = assembly.GetCustomAttributes<CultCacheFormatterResolverAttribute>()
.Select(attribute => CreateResolver(attribute.ResolverType))
.ToArray();
return resolvers.Length == 0 ? Options : Compose(resolvers);
return resolvers.Length == 0 ? Plain : Compose(resolvers);
});
}
private static MessagePackSerializerOptions Compose(IFormatterResolver[] consumerResolvers)
// The options a document's bytes are written and read with. A document's list elements carry their ids by their [Key]
// layout, so a resolver ahead of StandardResolver that formats a type on the way to those lists would write them
// without the id: refused here, once per document type, on every encode and decode path.
internal static MessagePackSerializerOptions OptionsForDocument(Type documentType)
{
if (documentType == null) throw new ArgumentNullException(nameof(documentType));
var (options, ahead) = ComposedFor(documentType.Assembly);
var problem = DocumentProblems.GetOrAdd(documentType, type => ResolverClaim(type, ahead));
return problem == null ? options : throw new InvalidOperationException(problem);
}
private static string? ResolverClaim(Type documentType, IFormatterResolver[] ahead)
{
foreach (var type in CultElementIds.Inspect(documentType).IdPath)
{
var generic = GetFormatterMethod.MakeGenericMethod(type);
// A resolver that hands back MessagePack's own formatter for the type writes its [Key] layout: that is no claim, and the
// composite stops there, so no later resolver is asked. StandardResolverAllowPrivate's formatter writes the same layout
// plus non-public [Key] members; those extra members are whole values the walk does not own, so it claims nothing
// either.
object? standard;
try
{
standard = generic.Invoke(StandardResolver.Instance, null);
}
catch (Exception exception)
{
return CannotBuild(documentType, type, (exception.InnerException ?? exception).Message);
}
if (standard == null)
return CannotBuild(documentType, type, "StandardResolver has no formatter for it");
if (WrittenAsCollection(type, generic, standard))
return $"Cult document {documentType.Name}: MessagePack writes {type.Name} as a collection, but the element-id walk reads it by its [Key] layout, so element ids would be lost or invented. A [MessagePackObject] type that is also a collection (ICollection<T>, IList, IDictionary) is written as a collection; make it one or the other. Or declare [MessagePackFormatter] on the type or member, which makes it one whole value with no element ids.";
var allowPrivateAsked = false;
object? allowPrivate = null;
foreach (var resolver in ahead)
{
object? formatter;
try
{
formatter = generic.Invoke(resolver, null);
}
catch (Exception exception)
{
return $"Cult document {documentType.Name}: formatter resolver {resolver.GetType().FullName} failed to build a formatter for {type.Name}, which is on the path to a list whose elements carry ids ({(exception.InnerException ?? exception).Message}); a resolver may not throw for such a type.";
}
if (formatter == null)
continue;
if (ReferenceEquals(formatter, standard))
break;
if (!allowPrivateAsked)
{
allowPrivateAsked = true;
try
{
allowPrivate = generic.Invoke(StandardResolverAllowPrivate.Instance, null);
}
catch (Exception)
{
allowPrivate = null;
}
}
if (allowPrivate != null && ReferenceEquals(formatter, allowPrivate))
break;
return $"Cult document {documentType.Name}: formatter resolver {resolver.GetType().FullName} formats {type.Name}, which is on the path to a list whose elements carry ids, so its element ids would never reach disk. A resolver may not claim such a type; write it by its [Key] layout, keep it out of the document's lists, or declare [MessagePackFormatter] on the type or member, which makes it one whole value with no element ids.";
}
}
return null;
}
private static string CannotBuild(Type documentType, Type type, string reason) =>
$"Cult document {documentType.Name}: MessagePack cannot build a formatter for {type.Name}, which is on the path to a list whose elements carry ids ({reason}).";
// The door verifies the walk and never decides ownership. It refuses a proven mismatch: a type the walk reads by key
// (an owner) whose StandardResolver formatter is the very instance the generic resolver's singleton hands out (a keyed
// ICollection<T> or dictionary). Union types walk as owners and the immutable and builtin resolvers claim only framework
// types, so neither can contradict. Any other formatter, a source-generated one among them, passes.
private static bool WrittenAsCollection(Type type, MethodInfo generic, object? standard) =>
CultElementIds.WalksAsOwner(type) && ReferenceEquals(Formatter(DynamicGenericResolver.Instance, generic), standard);
private static object? Formatter(IFormatterResolver resolver, MethodInfo generic)
{
try
{
return generic.Invoke(resolver, null);
}
catch (Exception)
{
return null;
}
}
private static (MessagePackSerializerOptions Options, IFormatterResolver[] Ahead) Compose(IFormatterResolver[] consumerResolvers)
{
// The guard goes first and wraps whatever formatter the rest of the composite would hand out, so a consumer
// resolver still encodes an arm while the key check runs.
var inner = CompositeResolver.Create(
consumerResolvers.Append(CultDocumentResolver.Instance).Append(CultMathResolver.Instance).Append(CultMathExprResolver.Instance).Append(StandardResolver.Instance).ToArray());
return MessagePackSerializerOptions.Standard
var ahead = consumerResolvers.Append(CultDocumentResolver.Instance).Append(CultMathResolver.Instance).Append(CultMathExprResolver.Instance).ToArray();
var inner = CompositeResolver.Create(ahead.Append(StandardResolver.Instance).ToArray());
var options = MessagePackSerializerOptions.Standard
.WithResolver(CompositeResolver.Create(new CultUnionGuard(inner), inner))
.WithSecurity(CultMessagePackSecurity.Instance);
return (options, ahead);
}
private static IFormatterResolver CreateResolver(Type resolverType)
@ -83,7 +195,7 @@ public static class CultDocumentMessagePackSerialization
{
if (registry == null) throw new ArgumentNullException(nameof(registry));
if (value != null) registry.GetRequired(type);
var payload = MessagePackSerializer.Serialize(type, value, OptionsFor(type.Assembly));
var payload = MessagePackSerializer.Serialize(type, value, OptionsForDocument(type));
CultPayloadDepth.Check(payload, type, PayloadDirection.Write);
return payload;
}
@ -98,7 +210,7 @@ public static class CultDocumentMessagePackSerialization
if (registry == null) throw new ArgumentNullException(nameof(registry));
registry.GetRequired(type);
CultPayloadDepth.Check(payload, type, PayloadDirection.Read);
return MessagePackSerializer.Deserialize(type, payload, OptionsFor(type.Assembly))
return MessagePackSerializer.Deserialize(type, payload, OptionsForDocument(type))
?? throw new InvalidOperationException($"MessagePack returned null for Cult document type {type.FullName}.");
}
@ -133,7 +245,7 @@ public static class CultDocumentMessagePackSerialization
elements[slot] = value;
}
var buffer = new ArrayBufferWriter<byte>();
var buffer = new global::System.Buffers.ArrayBufferWriter<byte>();
var writer = new MessagePackWriter(buffer);
writer.WriteArrayHeader(elements.Count);
foreach (var element in elements)
@ -160,22 +272,35 @@ public static class CultDocumentMessagePackSerialization
return ReadPersistedRecord(ref reader);
}
public static byte[] SerializeSnapshot(CultPersistedStoreSnapshot snapshot)
/// <summary>
/// Encodes the snapshot: every catalog entry and record from its fields, so what a caller edited is what is written.
/// </summary>
public static byte[] SerializeSnapshot(CultPersistedStoreSnapshot snapshot) =>
SerializeStore(
snapshot.FormatVersion,
snapshot.SchemaCatalog.Select(SerializeCatalogEntry).ToArray(),
snapshot.Records.Select(SerializePersistedRecord).ToArray());
// A store file from its parts, each already encoded: what a store carries forward is the bytes its file held, what it writes is
// an encoding of the object.
internal static byte[] SerializeStore(string formatVersion, IReadOnlyList<byte[]> catalogEntries, IReadOnlyList<byte[]> records)
{
if (formatVersion == null)
throw new ArgumentException("A store is written under a format version; the snapshot carries none.", nameof(formatVersion));
var buffer = new global::System.Buffers.ArrayBufferWriter<byte>();
var writer = new MessagePackWriter(buffer);
writer.WriteArrayHeader(StoreSnapshotFieldCount);
writer.Write(snapshot.FormatVersion);
writer.WriteArrayHeader(snapshot.SchemaCatalog.Length);
foreach (var entry in snapshot.SchemaCatalog)
writer.Write(formatVersion);
writer.WriteArrayHeader(catalogEntries.Count);
foreach (var entry in catalogEntries)
{
WriteSchemaCatalogEntry(ref writer, entry);
writer.WriteRaw(new ReadOnlySpan<byte>(entry));
}
writer.WriteArrayHeader(snapshot.Records.Length);
foreach (var record in snapshot.Records)
writer.WriteArrayHeader(records.Count);
foreach (var record in records)
{
WritePersistedRecord(ref writer, record);
writer.WriteRaw(new ReadOnlySpan<byte>(record));
}
writer.Flush();
@ -183,57 +308,136 @@ public static class CultDocumentMessagePackSerialization
}
/// <summary>
/// Parses a store snapshot whose format stamp is a string, of any version. It refuses a nil or binary stamp
/// (1.0.61 read a nil stamp as v1). A store reading its own file checks the stamp against the formats it reads,
/// before the body is parsed.
/// Reads a store file: exactly one complete MessagePack array of three slots (header, schema catalog, records), or an empty
/// array, which is an empty store. A truncated file, bytes after the array, a value that is not an array, a first slot that
/// is not a header string, a store with fewer or more slots, a record whose schema the catalog does not publish, or a body
/// that does not decode are refused with
/// <see cref="CultStoreUnreadableException"/>, so a reader never takes part of a file for the whole and never skips a slot
/// it does not understand. The rewriting paths use this same reader as their verdict on whether a file may be replaced.
/// </summary>
public static CultPersistedStoreSnapshot DeserializeSnapshot(byte[] payload) =>
DeserializeSnapshot(payload, _ => { });
public static CultPersistedStoreSnapshot DeserializeSnapshot(byte[] payload) => ReadStore(payload).Snapshot;
// The stamp decides readability and the body never does: requireReadable runs on slot 0 before anything else is parsed.
internal static CultPersistedStoreSnapshot DeserializeSnapshot(byte[] payload, Action<string> requireReadable)
// The reader behind DeserializeSnapshot, and beside the snapshot the exact bytes the file held for each catalog entry and each
// record (parallel to the snapshot's arrays). The snapshot's objects hold no bytes: a store that carries entries forward keeps
// these slices itself. The stamp decides readability and the body never does: requireReadable, when a store passes its rule,
// runs on slot 0 before the catalog or any record is parsed; the public parser passes none and reads any stamp.
internal static (CultPersistedStoreSnapshot Snapshot, byte[][] CatalogBytes, byte[][] RecordBytes) ReadStore(
byte[] payload,
Action<string>? requireReadable = null)
{
var reader = new MessagePackReader(payload);
var fieldCount = reader.ReadArrayHeader();
var snapshot = new CultPersistedStoreSnapshot();
if (fieldCount > 0)
try
{
// A stamp encoded as binary is not one this runtime wrote or reads; it is refused here, not parsed as damage.
if (reader.NextMessagePackType == MessagePackType.Binary)
throw new NotSupportedException("Store snapshot declares its format version as binary; this runtime reads " + CultPersistedStoreSnapshot.FormatV1 + " and " + CultPersistedStoreSnapshot.FormatV2 + " as strings.");
snapshot.FormatVersion = reader.ReadString()
?? throw new NotSupportedException("Store snapshot declares no format version; this runtime reads " + CultPersistedStoreSnapshot.FormatV1 + " and " + CultPersistedStoreSnapshot.FormatV2 + ".");
requireReadable(snapshot.FormatVersion);
}
// The stamp is judged from the leading bytes alone, so a body that will not parse can never make a newer store look
// like damage.
JudgeStamp(payload, requireReadable);
RequireOneArray(payload);
var reader = new MessagePackReader(payload);
var fieldCount = reader.ReadArrayHeader();
if (fieldCount != 0 && fieldCount != StoreSnapshotFieldCount)
{
throw new CultStoreUnreadableException(
$"The store has {fieldCount} top-level slots; this runtime reads {StoreSnapshotFieldCount} (header, schema catalog, records) or none.");
}
var snapshot = new CultPersistedStoreSnapshot();
if (fieldCount == 0)
return (snapshot, Array.Empty<byte[]>(), Array.Empty<byte[]>());
snapshot.FormatVersion = reader.ReadString()!;
if (fieldCount > 1)
{
var catalogCount = reader.ReadArrayHeader();
snapshot.SchemaCatalog = new CultSchemaCatalogEntry[catalogCount];
var catalogBytes = new byte[catalogCount][];
for (var index = 0; index < catalogCount; index++)
{
var start = reader.Position;
snapshot.SchemaCatalog[index] = ReadSchemaCatalogEntry(ref reader);
catalogBytes[index] = reader.Sequence.Slice(start, reader.Position).ToArray();
}
}
if (fieldCount > 2)
{
var recordCount = reader.ReadArrayHeader();
snapshot.Records = new CultPersistedRecord[recordCount];
var recordBytes = new byte[recordCount][];
for (var index = 0; index < recordCount; index++)
{
var start = reader.Position;
snapshot.Records[index] = ReadPersistedRecord(ref reader);
recordBytes[index] = reader.Sequence.Slice(start, reader.Position).ToArray();
}
}
for (var index = StoreSnapshotFieldCount; index < fieldCount; index++)
RequirePublishedSchemas(snapshot);
return (snapshot, catalogBytes, recordBytes);
}
catch (CultStoreUnreadableException)
{
reader.Skip();
throw;
}
catch (Exception ex) when (ex is not OutOfMemoryException)
{
throw new CultStoreUnreadableException(ex.Message, innerException: ex);
}
}
// A record whose schema the store's catalog does not publish cannot be read by any runtime that has only the file.
private static void RequirePublishedSchemas(CultPersistedStoreSnapshot snapshot)
{
var published = new HashSet<string>(StringComparer.Ordinal);
foreach (var entry in snapshot.SchemaCatalog)
{
published.Add(entry.SchemaId);
foreach (var compatible in entry.CompatibleSchemaIds)
published.Add(compatible);
}
return snapshot;
var unpublished = snapshot.Records.FirstOrDefault(record => !published.Contains(record.SchemaId));
if (unpublished != null)
{
throw new CultStoreUnreadableException(
$"Record '{unpublished.Key}' names schema '{unpublished.SchemaId}', which the store's schema catalog does not publish.");
}
}
// Slot 0 of a non-empty store, read before anything after it: a binary or nil stamp is refused as such, and a string stamp is
// judged by the caller's rule when it passes one. Bytes that do not open with an array and a string are left to the full read.
private static void JudgeStamp(byte[] payload, Action<string>? requireReadable)
{
string? stamp;
try
{
var reader = new MessagePackReader(payload);
if (reader.NextMessagePackType != MessagePackType.Array || reader.ReadArrayHeader() == 0)
return;
if (reader.NextMessagePackType == MessagePackType.Binary)
throw new CultStoreUnreadableException("Store snapshot declares its format version as binary; this runtime reads " + CultPersistedStoreSnapshot.FormatV1 + ", " + CultPersistedStoreSnapshot.FormatV2 + " and " + CultPersistedStoreSnapshot.FormatV3 + " as strings.");
if (reader.TryReadNil())
throw new CultStoreUnreadableException("Store snapshot declares no format version; this runtime reads " + CultPersistedStoreSnapshot.FormatV1 + ", " + CultPersistedStoreSnapshot.FormatV2 + " and " + CultPersistedStoreSnapshot.FormatV3 + ".");
stamp = reader.NextMessagePackType == MessagePackType.String ? reader.ReadString() : null;
}
catch (Exception ex) when (ex is EndOfStreamException or MessagePackSerializationException or InvalidOperationException)
{
return;
}
if (stamp != null)
requireReadable?.Invoke(stamp);
}
private static void RequireOneArray(byte[] bytes)
{
var whole = new MessagePackReader(bytes);
try
{
if (whole.NextMessagePackType != MessagePackType.Array)
throw new CultStoreUnreadableException($"The store is a MessagePack {whole.NextMessagePackType}, not an array.");
whole.Skip();
}
catch (Exception ex) when (ex is EndOfStreamException or MessagePackSerializationException or InvalidOperationException)
{
throw new CultStoreUnreadableException("The store is not one complete MessagePack array.", innerException: ex);
}
if (!whole.End)
throw new CultStoreUnreadableException("The store has bytes after its MessagePack array.");
}
/// <summary>Refuses a single-file snapshot this runtime cannot read, naming the format or record it found.</summary>
@ -242,37 +446,49 @@ public static class CultDocumentMessagePackSerialization
var version = snapshot.FormatVersion;
RequireReadableSingleFileStamp(version);
// Only a v2 store may hold a variant: a v1 header over a variant record is a store no reader could trust.
// Only a v2 or v3 store may hold a variant: a v1 header over a variant record is a store no reader could trust.
if (string.Equals(version, CultPersistedStoreSnapshot.FormatV1, StringComparison.Ordinal))
{
var variant = snapshot.Records.FirstOrDefault(record => record.Variant != null);
if (variant != null)
{
throw new NotSupportedException(
$"Record '{variant.Key}' (schema '{variant.SchemaId}') is a variant but the store declares {version}; variants need {CultPersistedStoreSnapshot.FormatV2}.");
throw new CultStoreUnreadableException(
$"Record '{variant.Key}' (schema '{variant.SchemaId}') is a variant but the store declares {version}; variants need {CultPersistedStoreSnapshot.FormatV2} or {CultPersistedStoreSnapshot.FormatV3}.");
}
}
}
// The one verdict on a single-file store's stamp, judged on slot 0 before the body is parsed: this runtime reads v1, v2 and v3.
internal static void RequireReadableSingleFileStamp(string version)
{
if (!string.Equals(version, CultPersistedStoreSnapshot.FormatV1, StringComparison.Ordinal) &&
!string.Equals(version, CultPersistedStoreSnapshot.FormatV2, StringComparison.Ordinal))
!string.Equals(version, CultPersistedStoreSnapshot.FormatV2, StringComparison.Ordinal) &&
!string.Equals(version, CultPersistedStoreSnapshot.FormatV3, StringComparison.Ordinal))
{
throw new NotSupportedException(
$"Store format {DescribeHeader(version)} is not readable; this runtime reads {CultPersistedStoreSnapshot.FormatV1} and {CultPersistedStoreSnapshot.FormatV2}.");
throw new CultStoreUnreadableException(
$"Store format {DescribeHeader(version)} is not readable; this runtime reads {CultPersistedStoreSnapshot.FormatV1}, {CultPersistedStoreSnapshot.FormatV2} and {CultPersistedStoreSnapshot.FormatV3}.");
}
}
// A store header as a refusal may show it: echoed in the shape cultcache.store.v<digits>, and otherwise described
// only by its length, since the bytes are the store's.
private static string DescribeHeader(string header)
internal static string DescribeHeader(string header)
{
const string prefix = "cultcache.store.v";
var version = header.StartsWith(prefix, StringComparison.Ordinal) ? header.Substring(prefix.Length) : "";
return version.Length > 0 && version.All(character => character >= '0' && character <= '9')
? header
: $"an unrecognised cultcache.store.* header of {System.Text.Encoding.UTF8.GetByteCount(header)} bytes";
if (header.StartsWith(prefix, StringComparison.Ordinal))
{
var end = prefix.Length;
while (end < header.Length && header[end] >= '0' && header[end] <= '9')
end++;
var digits = end - prefix.Length;
if (digits is > 0 and <= 6 && (end == header.Length || header[end] == '.'))
{
var named = header.Substring(0, end);
return end == header.Length ? named : $"{named} with a suffix of {System.Text.Encoding.UTF8.GetByteCount(header) - end} bytes";
}
}
return $"an unrecognised cultcache.store.* header of {System.Text.Encoding.UTF8.GetByteCount(header)} bytes";
}
private static void WritePersistedRecord(ref MessagePackWriter writer, CultPersistedRecord record)
@ -403,6 +619,15 @@ public static class CultDocumentMessagePackSerialization
return record;
}
internal static byte[] SerializeCatalogEntry(CultSchemaCatalogEntry entry)
{
var buffer = new global::System.Buffers.ArrayBufferWriter<byte>();
var writer = new MessagePackWriter(buffer);
WriteSchemaCatalogEntry(ref writer, entry);
writer.Flush();
return buffer.WrittenSpan.ToArray();
}
private static void WriteSchemaCatalogEntry(ref MessagePackWriter writer, CultSchemaCatalogEntry entry)
{
writer.WriteArrayHeader(SchemaCatalogEntryFieldCount);
@ -555,21 +780,31 @@ public class SingleFileMessagePackBackingStore : SingleFileBackingStore
{
}
protected override byte[] SerializeSnapshot(CultPersistedStoreSnapshot snapshot)
protected override byte[] SerializeStore(string formatVersion, IReadOnlyList<byte[]> catalogEntries, IReadOnlyList<byte[]> records)
{
return CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot);
return CultDocumentMessagePackSerialization.SerializeStore(formatVersion, catalogEntries, records);
}
protected override CultPersistedStoreSnapshot DeserializeSnapshot(byte[] data)
protected override byte[] SerializeRecord(CultPersistedRecord record)
{
var snapshot = CultDocumentMessagePackSerialization.DeserializeSnapshot(data, CultDocumentMessagePackSerialization.RequireReadableSingleFileStamp);
CultDocumentMessagePackSerialization.RequireSingleFileFormat(snapshot);
return snapshot;
return CultDocumentMessagePackSerialization.SerializePersistedRecord(record);
}
protected override byte[] SerializePayload(object document)
protected override byte[] SerializeCatalogEntry(CultSchemaCatalogEntry entry)
{
return CultDocumentMessagePackSerialization.SerializeUntyped(document, document.GetType(), Registry);
return CultDocumentMessagePackSerialization.SerializeCatalogEntry(entry);
}
protected override (CultPersistedStoreSnapshot Snapshot, byte[][] CatalogBytes, byte[][] RecordBytes) ReadStore(byte[] data)
{
var store = CultDocumentMessagePackSerialization.ReadStore(data, CultDocumentMessagePackSerialization.RequireReadableSingleFileStamp);
CultDocumentMessagePackSerialization.RequireSingleFileFormat(store.Snapshot);
return store;
}
protected override byte[] SerializePayload(object document, Type documentType)
{
return CultDocumentMessagePackSerialization.SerializeUntyped(document, documentType, Registry);
}
protected override object DeserializePayload(Type documentType, byte[] payload)

263
src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs
View file

@ -15,7 +15,9 @@ namespace GameCult.Caching.MessagePack;
// A small hot manifest indexes the store; each record lives in one cold content-addressed page.
public sealed class DirectoryMessagePackBackingStore : CacheBackingStore
{
private const string IndexedFormatVersion = "cultcache.store.v4.directory-content-addressed-pages";
// The two manifest headers; the shared header decision names them.
private static readonly string IndexedFormatVersion = HeaderFor(holdsIds: false, existingHeader: null, wholeStore: true, directoryStore: true);
private static readonly string IndexedFormatVersionWithIds = HeaderFor(holdsIds: true, existingHeader: null, wholeStore: true, directoryStore: true);
// An unleased load that has not settled after this many attempts throws.
private const int UnleasedLoadAttempts = 5;
private readonly FileInfo _manifestFile;
@ -23,7 +25,6 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore
private readonly ConcurrentDictionary<string, bool> _dirtyKeys = new(StringComparer.Ordinal);
private readonly ConcurrentDictionary<string, bool> _deletedKeys = new(StringComparer.Ordinal);
private readonly HashSet<string> _hydratedKeys = new(StringComparer.Ordinal);
private CultSchemaCatalogEntry[] _durableCatalog = Array.Empty<CultSchemaCatalogEntry>();
public DirectoryMessagePackBackingStore(string manifestPath, string? recordDirectoryPath = null, bool readOnly = false)
: base(readOnly)
@ -62,18 +63,19 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore
var reports = new List<CultSchemaMigrationReport>();
var loaded = new Dictionary<string, CultStoredDocument>(StringComparer.Ordinal);
CultSchemaCatalogEntry[] catalog;
var foreign = new List<CultForeignRecord>();
for (var attempt = 1; ; attempt++)
{
loaded.Clear();
reports.Clear();
foreign.Clear();
// Only pages named by the manifest read under the lease are loaded; orphaned pages are never loaded. With no
// lock to lease (none created yet, or a store copied without it) a writer may commit mid-load, so an unleased
// load stands only if its manifest is still current; if the manifest moved it reloads, under the lease the
// writer created. A page that fails under an unmoved manifest is corruption, not a race.
using var lease = AcquireCommitLease(wait: true, create: false);
var manifestBytes = ReadManifestBytes();
var manifest = ParseManifest(manifestBytes);
var (manifest, _) = ParseManifest(manifestBytes);
Trace($"manifest records={manifest.Records.Length}");
try
{
@ -83,10 +85,10 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore
manifest.Records.OrderBy(record => record.Key, StringComparer.Ordinal).ToArray(),
manifest.SchemaCatalog,
loaded,
reports);
reports,
foreign);
if (lease != null || !ManifestMoved(manifestBytes))
{
catalog = manifest.SchemaCatalog;
Trace($"indexed-pages loaded={loaded.Count}");
break;
}
@ -109,8 +111,7 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore
var arrived = loaded
.Where(pair => !Staged(pair.Key) &&
(!Entries.TryGetValue(pair.Key, out var existing) ||
existing.StoredAt != pair.Value.StoredAt ||
existing.PersistedSchemaId != pair.Value.PersistedSchemaId))
!SameStoredIdentity(existing.StoredVersion, pair.Value.StoredVersion)))
.Select(pair => pair.Value)
.ToArray();
var departed = _hydratedKeys
@ -121,7 +122,6 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore
Loaded?.Invoke(arrived, departed);
// Adopted only once the cache admitted the load; a refused load leaves the store's durable view untouched.
_durableCatalog = catalog;
foreach (var stored in departed)
Entries.TryRemove(stored.Key.Value, out _);
_hydratedKeys.RemoveWhere(key => !loaded.ContainsKey(key) && !Staged(key));
@ -131,6 +131,7 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore
_hydratedKeys.Add(key);
SetLastSchemaMigrationReports(reports);
SetForeignRecords(foreign);
IsDirty = !_dirtyKeys.IsEmpty || !_deletedKeys.IsEmpty;
Trace("publish");
}
@ -179,10 +180,19 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore
using var commitLease = AcquireCommitLease(wait, create: true);
if (commitLease == null)
return CultCommitOutcome.Contended;
var manifest = ReadManifest();
if (!request.ConditionsHold(manifest.Records, Entries.Values))
var (manifest, catalogBytes) = ReadManifest();
if (request.HasConditions && !request.ConditionsHold(
manifest.Records.ToDictionary(record => record.Key, record => VersionOf(record.Payload), StringComparer.Ordinal), Entries.Values, ForeignRecords))
return CultCommitOutcome.Mismatch;
RefuseUndeclaredOnDisk(manifest, request.Upserts.Concat(request.Deletes).Select(entry => entry.Key.Value).Concat(request.Checks).Concat(StagedKeys));
if (request.Upserts.Count == 0 && request.Deletes.Count == 0 && !IsDirty)
{
// Nothing to write: the batch only judged keys it does not hold.
MarkFlushSucceeded();
return CultCommitOutcome.Committed;
}
var previousEntries = Entries.ToDictionary(pair => pair.Key, pair => pair.Value, StringComparer.Ordinal);
var previousDirtyKeys = _dirtyKeys.Keys.ToArray();
var previousDeletedKeys = _deletedKeys.Keys.ToArray();
@ -202,7 +212,7 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore
_deletedKeys.TryRemove(entry.Key.Value, out _);
}
IsDirty = true;
WriteGeneration(manifest);
WriteGeneration(manifest, catalogBytes);
return CultCommitOutcome.Committed;
}
catch
@ -227,20 +237,81 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore
ThrowIfReadOnly();
Held(() =>
{
if (!IsDirty)
if (!IsDirty && StagedChecks.Count == 0)
return;
Directory.CreateDirectory(_manifestFile.DirectoryName!);
using var commitLease = AcquireCommitLease(wait: true, create: true);
WriteGeneration(ReadManifest());
var (manifest, catalogBytes) = ReadManifest();
RefuseUndeclaredOnDisk(manifest, StagedKeys);
if (IsDirty)
WriteGeneration(manifest, catalogBytes);
else
MarkFlushSucceeded();
});
}
// Every key the next flush or commit judges: the staged writes and removals, and the staged checks.
private IEnumerable<string> StagedKeys => _deletedKeys.Keys.Concat(_dirtyKeys.Keys).Concat(StagedChecks);
// A writable store judges a removal of a key its cache does not hold at its next flush or commit, under the commit lease
// (StageChecks); a read-only store has neither, so it answers from its view and never creates the lease.
protected internal override void RefuseRemovalOfUndeclared(IReadOnlyCollection<string> keys)
{
if (IsReadOnly)
base.RefuseRemovalOfUndeclared(keys);
else
StageChecks(keys);
}
// A read-only store answers a batch's checks from its view and never creates the lease; a writable one reads the manifest
// under the commit lease.
protected internal override CultCommitOutcome JudgeChecks(IReadOnlyCollection<string> keys, bool wait)
{
if (IsReadOnly)
return base.JudgeChecks(keys, wait);
return Held(() =>
{
using var commitLease = AcquireCommitLease(wait, create: true);
if (commitLease == null)
return CultCommitOutcome.Contended;
RefuseUndeclaredOnDisk(ReadManifest().Snapshot, keys);
return CultCommitOutcome.Committed;
});
}
// A manifest record no registered type claims, listed at the version its index record names.
private CultForeignRecord? ForeignAt(CultPersistedRecord record, IReadOnlyCollection<CultSchemaCatalogEntry> catalog) =>
Foreign(record, catalog, VersionOf(record.Payload));
// The manifest's other records stay as they are, foreign and read-only ones included; a write that would replace or remove a
// record the manifest holds that the writer does not own is refused before any page is written, by the one rule every
// store shares (CacheBackingStore.RefuseUndeclaredOnDisk). The refused records reload as a pull loads them.
private void RefuseUndeclaredOnDisk(CultPersistedStoreSnapshot currentManifest, IEnumerable<string> written)
{
CultPersistedRecord? HeldAt(string key) =>
currentManifest.Records.FirstOrDefault(record => string.Equals(record.Key, key, StringComparison.Ordinal));
RefuseUndeclaredOnDisk(
written,
key => HeldAt(key) is { } held ? Undeclared(held, currentManifest.SchemaCatalog, Registry.Owns) : null,
key => HeldAt(key) is { } held && Foreign(held, currentManifest.SchemaCatalog, VersionOf(held.Payload)) == null
? LoadPage(held, currentManifest.SchemaCatalog, out _, out _)
: null,
key => HeldAt(key) is { } held ? ForeignAt(held, currentManifest.SchemaCatalog) : null,
key =>
{
_dirtyKeys.TryRemove(key, out _);
_deletedKeys.TryRemove(key, out _);
IsDirty = !_dirtyKeys.IsEmpty || !_deletedKeys.IsEmpty;
});
}
// Runs under the commit lease: pages first, then the manifest that names them.
private void WriteGeneration(CultPersistedStoreSnapshot currentManifest)
private void WriteGeneration(CultPersistedStoreSnapshot currentManifest, IReadOnlyDictionary<CultSchemaCatalogEntry, byte[]> catalogBytes)
{
Directory.CreateDirectory(_recordDirectory.FullName);
var currentIndex = currentManifest.Records.ToDictionary(record => record.Key, record => record, StringComparer.Ordinal);
var replaced = currentManifest.Records.ToDictionary(record => record.Key, record => record.StoredAt, StringComparer.Ordinal);
foreach (var key in _deletedKeys.Keys)
currentIndex.Remove(key);
foreach (var key in _dirtyKeys.Keys)
@ -249,21 +320,9 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore
currentIndex[key] = ToIndexRecord(stored);
}
var catalogCandidates = currentManifest.SchemaCatalog
.Concat(_durableCatalog)
.Concat(Entries.Values.Select(entry => entry.Descriptor.ToCatalogEntry()))
.GroupBy(entry => entry.SchemaId, StringComparer.Ordinal)
.Select(group => group.Last())
.ToArray();
var usedSchemaIds = currentIndex.Values.Select(record => record.SchemaId).ToHashSet(StringComparer.Ordinal);
var targetCatalog = catalogCandidates
.Where(entry => usedSchemaIds.Contains(entry.SchemaId))
.OrderBy(entry => entry.SchemaName, StringComparer.Ordinal)
.ToArray();
var keysToWrite = _dirtyKeys.Keys
.OrderBy(value => value, StringComparer.Ordinal)
.ToArray();
var written = new List<CultStoredDocument>(keysToWrite.Length);
foreach (var key in keysToWrite)
{
if (!Entries.TryGetValue(key, out var stored))
@ -271,6 +330,9 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore
continue;
}
// A record this write replaces is stored later than it: the one re-mint every store kind shares (StoredAtOver).
if (replaced.TryGetValue(key, out var replacedAt))
stored.StoredAt = StoredAtOver(stored.StoredAt, replacedAt);
var record = ToPersistedRecord(stored, document =>
CultDocumentMessagePackSerialization.SerializeUntyped(
document,
@ -278,45 +340,76 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore
Registry));
var pagePayload = CultDocumentMessagePackSerialization.SerializePersistedRecord(record);
var indexRecord = ToContentAddressedIndexRecord(record, pagePayload);
stored.StoredVersion = VersionOf(indexRecord.Payload);
stored.StoredSchemaId = stored.Descriptor.SchemaId;
currentIndex[key] = indexRecord;
WriteFileAtomically(
ContentAddressedRecordPath(indexRecord),
pagePayload);
written.Add(stored);
}
WriteManifest(targetCatalog, currentIndex.Values
// The catalog and the header follow the shared write rules: the registered entry publishes only the ids this write put a record
// under, every other record and entry stays as the manifest holds it.
var written = keysToWrite.Where(key => Entries.ContainsKey(key)).ToHashSet(StringComparer.Ordinal);
var targetCatalog = CultSchemaCatalogEntry.Derive(
currentIndex.Values,
written.Select(key => Entries[key].Descriptor).Distinct().Select(descriptor => descriptor.ToCatalogEntry()).ToArray(),
currentManifest.SchemaCatalog)
.OrderBy(entry => entry.SchemaName, StringComparer.Ordinal)
.ThenBy(entry => entry.SchemaId, StringComparer.Ordinal)
.ToArray();
var holdsIds = written.ToDictionary(key => key, key => Entries[key].HoldsIds, StringComparer.Ordinal);
var header = HeaderForWrite(
currentManifest.FormatVersion,
holdsIds.Values.Select(holds => (bool?)holds).ToArray(),
copiesAny: currentIndex.Values.Any(record => !written.Contains(record.Key)),
copiedHoldIds: false, // only C# writes a directory store, and it marks by content
directoryStore: true,
holdsVariants: false);
WriteManifest(targetCatalog, catalogBytes, currentIndex.Values
.OrderBy(record => record.Key, StringComparer.Ordinal)
.ToArray());
// The manifest names the new pages: each was written under its own descriptor.
foreach (var stored in written)
MarkWritten(stored);
.ToArray(), header);
DeleteUnreferencedRecordPages(currentIndex.Values);
_durableCatalog = targetCatalog;
_dirtyKeys.Clear();
_deletedKeys.Clear();
MarkFlushSucceeded();
}
private void WriteManifest(CultSchemaCatalogEntry[] catalog, CultPersistedRecord[] index)
{
var manifest = new CultPersistedStoreSnapshot
{
FormatVersion = IndexedFormatVersion,
SchemaCatalog = catalog,
Records = index
};
WriteFileAtomically(_manifestFile.FullName, CultDocumentMessagePackSerialization.SerializeSnapshot(manifest));
}
// The index records are this store's own and are encoded from their fields; a catalog entry the manifest already held is written
// as the slice it held, one the writer built is encoded.
private void WriteManifest(
CultSchemaCatalogEntry[] catalog,
IReadOnlyDictionary<CultSchemaCatalogEntry, byte[]> catalogBytes,
CultPersistedRecord[] index,
string header) =>
WriteFileAtomically(
_manifestFile.FullName,
CultDocumentMessagePackSerialization.SerializeStore(
header,
catalog.Select(entry => catalogBytes.TryGetValue(entry, out var held) ? held : CultDocumentMessagePackSerialization.SerializeCatalogEntry(entry)).ToArray(),
index.Select(CultDocumentMessagePackSerialization.SerializePersistedRecord).ToArray()));
private void LoadRecordPages(
CultPersistedRecord[] records,
IReadOnlyCollection<CultSchemaCatalogEntry> catalogEntries,
Dictionary<string, CultStoredDocument> loaded,
List<CultSchemaMigrationReport> reports)
List<CultSchemaMigrationReport> reports,
List<CultForeignRecord> foreign)
{
// A record no registered type claims stays as the manifest names it: its page is never read, and a commit keeps it.
var claimed = new List<CultPersistedRecord>(records.Length);
foreach (var metadata in records)
{
if (ForeignAt(metadata, catalogEntries) is { } carried)
foreign.Add(carried);
else
claimed.Add(metadata);
}
records = claimed.ToArray();
var tracePages = string.Equals(
Environment.GetEnvironmentVariable("CULTCACHE_TRACE_STARTUP_PHASES"),
"1",
@ -342,16 +435,10 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore
index =>
{
var started = tracePages ? Stopwatch.GetTimestamp() : 0L;
var metadata = records[index];
var record = ReadPersistedRecordPage(metadata, out var pagePayload);
storedRecords[index] = ToStoredDocument(
record,
catalogEntries,
(type, payload) => CultDocumentMessagePackSerialization.DeserializeUntyped(type, payload, Registry),
out recordReports[index]);
storedRecords[index] = LoadPage(records[index], catalogEntries, out recordReports[index], out var pageLength);
if (tracePages)
{
pageBytes[index] = pagePayload.LongLength;
pageBytes[index] = pageLength;
pageElapsedTicks[index] = Stopwatch.GetTimestamp() - started;
}
});
@ -409,7 +496,7 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore
Payload = HashPayload(pagePayload)
};
private CultPersistedStoreSnapshot ReadManifest() => ParseManifest(ReadManifestBytes());
private (CultPersistedStoreSnapshot Snapshot, Dictionary<CultSchemaCatalogEntry, byte[]> CatalogBytes) ReadManifest() => ParseManifest(ReadManifestBytes());
private byte[]? ReadManifestBytes() => File.Exists(_manifestFile.FullName) ? ReadAllBytesShared(_manifestFile.FullName) : null;
@ -435,37 +522,51 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore
private InvalidOperationException Unsettled(Exception? last) => new(
$"Directory store {_manifestFile.FullName} changed under every one of {UnleasedLoadAttempts} unlocked loads; it did not settle.", last);
// A missing manifest is an empty store. An existing manifest in any other format is refused.
private CultPersistedStoreSnapshot ParseManifest(byte[]? manifestBytes)
// A missing manifest is an empty store. An existing manifest in any other format is refused. Beside the snapshot, the bytes the
// manifest held for each catalog entry, by the entry this read produced: the slices a write carries forward.
private (CultPersistedStoreSnapshot Snapshot, Dictionary<CultSchemaCatalogEntry, byte[]> CatalogBytes) ParseManifest(byte[]? manifestBytes)
{
if (manifestBytes == null)
{
return new CultPersistedStoreSnapshot
return (new CultPersistedStoreSnapshot
{
FormatVersion = IndexedFormatVersion,
SchemaCatalog = Array.Empty<CultSchemaCatalogEntry>(),
Records = Array.Empty<CultPersistedRecord>()
};
}, new Dictionary<CultSchemaCatalogEntry, byte[]>());
}
// The stamp is judged before the manifest body is parsed, so a store stamped newer is refused whatever its body.
return CultDocumentMessagePackSerialization.DeserializeSnapshot(manifestBytes, RequireReadableStamp);
// The stamp is judged before the manifest body is parsed, so a store stamped newer is refused whatever its body. Every
// write path parses the manifest under its commit lease before it writes a page or the manifest, so a store another
// process has since stamped newer is refused there too, untouched.
(CultPersistedStoreSnapshot Snapshot, byte[][] CatalogBytes, byte[][] RecordBytes) read;
try
{
read = CultDocumentMessagePackSerialization.ReadStore(manifestBytes, RequireReadableStamp);
}
catch (CultStoreUnreadableException ex) when (ex.Path == null)
{
throw new CultStoreUnreadableException(ex.Message, _manifestFile.FullName, ex.InnerException);
}
var snapshot = read.Snapshot;
// An array with no stamp claims no directory format: it parses to the default v1 without the callback above ever seeing a stamp.
RequireReadableStamp(snapshot.FormatVersion);
var catalogBytes = new Dictionary<CultSchemaCatalogEntry, byte[]>();
for (var index = 0; index < snapshot.SchemaCatalog.Length; index++)
catalogBytes[snapshot.SchemaCatalog[index]] = read.CatalogBytes[index];
return (snapshot, catalogBytes);
}
// The one verdict on a directory manifest's stamp: this runtime reads v4 and v5.
private void RequireReadableStamp(string stamp)
{
if (CultPersistedStoreSnapshot.FormatNumber(stamp) is int number && number > 4)
if (!string.Equals(stamp, IndexedFormatVersion, StringComparison.Ordinal) &&
!string.Equals(stamp, IndexedFormatVersionWithIds, StringComparison.Ordinal))
{
// Every write path parses the manifest under its commit lease before it writes a page or the manifest, so a
// store another process has since stamped newer is refused here, untouched.
throw new NotSupportedException(
$"Directory store {_manifestFile.FullName} is {stamp}, a newer format than this runtime reads ({IndexedFormatVersion}).");
}
if (!string.Equals(stamp, IndexedFormatVersion, StringComparison.Ordinal))
{
throw new InvalidOperationException(
$"Directory store {_manifestFile.FullName} is {stamp}; only {IndexedFormatVersion} is readable.");
throw new CultStoreUnreadableException(
$"Directory store {_manifestFile.FullName} is {CultDocumentMessagePackSerialization.DescribeHeader(stamp)}; only {IndexedFormatVersion} and {IndexedFormatVersionWithIds} are readable.",
_manifestFile.FullName);
}
}
@ -507,6 +608,26 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore
}
}
// One record as a load decodes it, from the page its manifest record names: a pull's pages and a refusal's reload.
private CultStoredDocument LoadPage(
CultPersistedRecord metadata,
IReadOnlyCollection<CultSchemaCatalogEntry> catalog,
out CultSchemaMigrationReport report,
out long pageLength)
{
var record = ReadPersistedRecordPage(metadata, out var pagePayload);
pageLength = pagePayload.LongLength;
var stored = ToStoredDocument(
record,
VersionOf(metadata.Payload),
catalog,
(type, payload) => CultDocumentMessagePackSerialization.DeserializeUntyped(type, payload, Registry),
out report);
// The manifest is this store's record of what is on disk: the stored id and the version a commit condition compares with.
stored.StoredSchemaId = metadata.SchemaId;
return stored;
}
private CultPersistedRecord ReadPersistedRecordPage(CultPersistedRecord metadata, out byte[] pagePayload)
{
var path = ContentAddressedRecordPath(metadata);

1
src/GameCult.Caching.MessagePack/GameCult.Caching.MessagePack.csproj
View file

@ -11,6 +11,5 @@
<PackageReference Include="System.Text.Json" Version="9.0.10" />
<PackageReference Include="MessagePackAnalyzer" Version="3.1.7" PrivateAssets="all" />
<InternalsVisibleTo Include="GameCult.Caching.Tests" />
<InternalsVisibleTo Include="GameCult.Mesh" />
</ItemGroup>
</Project>

263
src/GameCult.Caching/Contracts/cultcache-persistence-format.md
View file

@ -46,23 +46,71 @@ store. The store commits the batch as one durable step. Nothing in the batch is
visible, to the committing flow or to observers, until the store has accepted
it; a failed commit changes nothing on disk or in memory.
A batch may carry conditions: per-record `(schemaId, storedAt)` identity, or
the whole store unchanged since it was last loaded. Conditions are evaluated
A batch may carry conditions: per-record version, or the whole store unchanged
since it was last loaded. A record's version is the SHA-256 of the exact bytes the
record is stored as: in a single file the record's own slice of the file, the array
`[key, schemaId, storedAt, payload(, variant)]` exactly as the file holds it, in
whatever legal MessagePack encoding its writer chose (never a re-encoding of the
decoded record); in a directory store the page the manifest indexes. A record being
written has the version of the bytes it is written as. It names the schema id and
storedAt as well as the payload. storedAt is metadata and decides no condition.
Conditions are evaluated
under the store's exclusive lock against what is durably on disk. A failed
condition is a lost race, not an error: the commit reports it and writes
nothing.
**Conditional commit is the only safe multi-process write.** A plain flush and
an unconditional commit are last-writer-wins and write the same bytes: a single
file is replaced by the writer's whole view, a directory manifest takes the
writer's staged keys over the current manifest. An unconditional commit also
persists writes staged earlier and leaves the store clean. Processes sharing a
store must all use conditional commit.
**A flush, a commit and a single-file document write apply a write set.** Under
the store's exclusive lock the writer reads what is durably in the store now,
applies only what it staged (writes and removals) and, for a commit, its batch,
and replaces the store with the result. Every other record, and every catalog
entry the result references, is copied forward as stored: a record's exact bytes
and a catalog entry's exact bytes, unknown fields included, so a copied record
keeps its version. No type interprets a copied record: a write may hash another
record's bytes to compare versions, and reads its header (key, schema id, variant
slot), but never decodes its payload.
A record is encoded from the object the writer hands in, and every other record is copied as the store holds it. The
store that read a file owns that file's bytes: for each stored record, its exact slice of the file and the SHA-256 of that
slice, which is the record's version; likewise each catalog entry's slice. A decoded record or snapshot object carries no
bytes, so an object a caller edited in place is never mistaken for what the file holds. Consequently a record the writer
staged is re-encoded canonically and has the version of its new encoding, even when its content is unchanged, and a
record nobody staged keeps its bytes and its version for as long as no writer stages it. The public snapshot and record
serializers encode every record from its fields.
A record written over one the store holds gets a storedAt later than the one it
replaces. A document changed in place and not staged is not written.
**In a store that holds no variant**, a plain flush and an unconditional commit
write the same bytes and are last-writer-wins only per staged key. An
unconditional commit also persists writes staged earlier and leaves the store
clean. Conditional commit is how such a writer learns that another writer moved
the keys it depends on.
**A store that holds a variant, or would after the write, is written only from
what its writer last read.** A variant resolves against other records (its base
chain, and the index values it shares the store with). So before writing, the
writer compares every record's version with the version it last read
or wrote. If any record moved, appeared or disappeared, the whole write is
refused with the typed write conflict, naming the keys it would have written and
the keys that moved, and nothing is written. A writer that has read nothing,
such as CultMesh's single-file document write, never writes into a store holding
a variant.
**A refused write reloads what it refused.** A refusal (a foreign record at a
key the write names, or a write conflict) reloads the durable view of every
refused and moved key into the cache, as a pull would, and only then forgets the
staged changes at those keys. The cache then serves what the store holds, and a
retry is judged against that. If the reload is itself refused (the store holds a
record this cache cannot load), nothing changes and the refusal carries that
cause. A writer refuses a file whose header it cannot read (truncated, or a
format it does not read) and leaves the file as it was. C# applies this to
single-file stores, directory stores and CultMesh's single-file document writes;
Rust, TypeScript and Python rewrite the store from their own copy until their
parity cuts land.
A store that fails to hydrate on open (corrupt bytes, unresolvable schema)
makes the open throw and is left byte-identical; it is never silently
overwritten. Within one process the lock order is the cache's gate, then the
store's file lock; a store's I/O blocks its cache's readers.
store's gate, then the file lock; nothing under a store's gate enters a cache's
gate. A store's I/O blocks its cache's readers.
C# implements batch and conditional commit (`CultCache.Commit`, `TryCommit`,
`CultCacheBatch`). Rust implements both. TypeScript and Python implement
@ -116,6 +164,87 @@ These are store-level bookkeeping fields, not per-record domain metadata.
The store must embed a catalog entry for every schema referenced by the records
it contains.
A reader never recovers a schema from a record's payload: a record whose schema id the
catalog does not publish (by id or as a compatible id) makes the store unreadable, whatever
the payload's first field says. The catalog is the store's own description of itself.
A writer keeps the same invariant: every record it writes carries a schema id that some catalog
entry it writes publishes, as that entry's id or as one of its compatible ids. A cache stamps a
record it puts with its registered schema's id, and a record it loads under an id its type owns or
declares compatible (see below for one it does not). The cache is the only owner of that stamp: a store writer never
changes the identity of a record it did not receive from its caller, so a record another writer
changed since this cache read it is written back as that writer left it.
A schema type declares the older ids its records may sit under: `CultDocumentAttribute.CompatibleSchemaIds`
(C#), `compatibleSchemaIds` (TypeScript, Python). The registered catalog entry lists them after its own id, and
the cache restamps a record loaded under one to the type's own id, so the next write of that record carries the
registered id. A record another writer left under a declared id, and that a commit does not rewrite, stays under it
and the registered entry publishes it. Without the declaration such a record is read-only to the cache (see
ownership by declaration, below). Rust has no such declaration.
A cache resolves a record to a local type by the record's schema id: the registered type that owns
the id, else one that lists it as compatible. The name the catalog entry carries is metadata. It
names a local type only when no local type has the id, and then only to read with: a record so
resolved is read-only (below). That serves a store written by a runtime whose schema ids this one
cannot know (a Rust type's schema id is its entry type). A schema renamed under a stable id
therefore opens in every runtime that holds the id.
**A cache owns a record when it declares the record's schema id, or when the type that reads it reads
it whole.** A type declares an id by registering it or listing it compatible. A type reads a record
whole when the catalog entry that publishes the id has the same content hash as the type's entry, or
compares compatible with it and leaves no persisted slot unread. An owned record is writable: a write,
relabel or removal is accepted and the write stores it under the writing type's own id (restamp on
write; only a record the cache writes is restamped, the rest is copied forward as stored). A record
stored under any other id that a type can read, through the catalog's schema name or an entry listing a
local id, only by dropping persisted slots is loaded read-only: Get and queries serve it, a write of
another record copies it as stored, and a staged write or removal of it (upsert, variant upsert,
flatten, remove, commit) is refused with the typed schema conflict before anything is staged.
Declaring its id compatible on any registered type makes it writable and is the consent to the loss:
the next write stores it under the writing type's own id and sheds the slots.
The rule is checked again where the write lands, under the store's lock: a write or removal at a key
whose durable record is not owned, because another writer rewrote it after this cache
loaded, is refused typed, and the record is reloaded as the store holds it, read-only when a type can
read it. This holds for every write through the store, the store's own `Push`, `Delete` and
`CommitBatch` included. A CultMesh single-file write (`WriteDocument`, `WriteRecord`, or `PushAll` on an unattached store) asks the same rule: it owns the ids its
written entries declare (their own ids and the ids they list compatible) and an id whose publishing entry its claimant reads whole (no persisted
slot unread). The claimant is picked by the one precedence over the written entries and the call's registry, a written entry being the only
description of its schema id (the registry's type for that id is never asked), and the record is owned only when the claimant is a written
entry. A registry adds claimants and declares nothing, so it only adds refusals; with none the written entries are the only claimants. Refused:
a record another registered type claims, under that type's current id or an old one it lists; a record whose stored entry lists another
registered type's id ahead of the written one's, whatever name the stored entry carries; a record two registered types list compatible or two
registered versions of the written schema could read; a record the picked written entry reads only by dropping persisted slots; and the
removal of any stored record by a call that writes no entry. A registry-less `WriteDocument` reads `CultDocumentRegistry.Shared`. It is
refused the same way otherwise.
A record that resolves to no local type is a foreign record, for example one of a type this build does not
have, or one renamed under a new schema name without declaring its old id. A store never destroys or relabels it: the store carries it
byte for byte under its own id and its own catalog entry, lists it as foreign (`CacheBackingStore.ForeignRecords`
in C#), and the cache never holds it. Writes of other records proceed and copy it forward as the file holds it. A write that would replace or remove it is refused with the typed schema conflict
naming its key and id, and nothing is written. A removal of a key the cache does not hold removes nothing and is free; the store judges the key at its next flush or commit, under its lock, with the keys that write writes, and refuses the write if it holds a record there under an undeclared id, also when that record arrived after the load. A commit judges such a key in every writable store its cache attaches, each under its own lock: the stores it does not write first, then the store it writes, and a refusal in any of them writes nothing. A judged key is never removed. A read-only store answers at the removal, from its last load. The refused keys are reloaded as the store holds them (a foreign record is not held, and is listed as foreign from then on, so its next write is refused before staging), and their staged changes are forgotten. A write that carries a record forward keeps a marked header marked. Declaring its id on a type claims it at the next load.
A reader resolves a record's schema id to the entry that owns it, that entry's own `schemaId`, and
only when no entry owns the id to an entry that lists it as a compatible id (the first such entry
in catalog order).
A writer derives the catalog from the records it writes, and from nothing else. For each distinct
schema id a record carries it chooses exactly one entry that publishes it. For an id the write stages
a record under, that is the entry the writer registered or passed, and a rename or a stale content hash
takes the descriptor's name, hash, members and compatible ids as they are. For every other id it is
the durable entry that owns the id, else one that lists it as compatible, carried as its exact bytes,
unknown trailing fields included. A registered entry never publishes an id the write stages nothing
under. Entries are written as chosen, deduplicated by own id, with
no union of compatible ids and no dependence on the order records arrive in; an entry no record needs
is not written. A write is refused, and the store left as it was, when two entries of one tier share
an own id and disagree on the schema name, when records of different types share a schema id, when a
write would change which type an existing record resolves to (the schema names of the records already
in the store and of the records written differ), or when a record's id is published by no chosen entry.
The retyping refusal has one exception, a rename: when the entries of both the existing records and
the written ones own the id, the registered descriptor wins. (A C# cache stamps every record it
writes with its descriptor's own id, so the retyping refusal cannot arise there.) Each runtime refuses with one typed error naming the id, the
schema names and a record key: `CultSchemaConflictException` (C#), `SchemaConflictError`
(Rust, TypeScript, Python).
This is not decorative paperwork. It is what allows another CultCache
implementation to inspect an old store, resolve the schema for each record, and
decide whether it can:
@ -212,10 +341,46 @@ Example:
The payload is the domain object only. It does not repeat the key, schema id,
or stored timestamp.
Under `cultcache.store.v1` a record has exactly these four slots. A reader refuses any
`cultcache.store.*` header it does not know and any record with more slots than its
header allows, naming the version or the record's key and schema id. It never skips a
slot it does not understand.
Under `cultcache.store.v1` and `cultcache.store.v3` a record has exactly these four slots.
A reader refuses any `cultcache.store.*` header it does not know and any record with more
slots than its header allows, naming the version or the record's key and schema id. It
never skips a slot it does not understand.
The headers are `v1` (no variant, no element ids), `v2` (holds a variant) and `v3` (holds
an element id, variants or not). A single-file store declares `v3` when any record it
holds carries an element id: a non-empty `[CultElementId]` value in a record's document,
or in a variant's override values. A store that cannot read a variant's override values
(no codec attached) counts the variant as holding one. An element id is an ordinary keyed
string in the payload, so only a reader with the record's type can tell; a C# write
decides for the records it encodes and, while the header it found is unmarked, for every
plain record it copies forward that its registry can read, decoding the bytes the file
holds under the store's lock. A copied record it cannot read, and a raw CultMesh payload,
decide nothing. A write that copies any record forward
therefore keeps a `v3` header it found. Only a write that encodes every record the store
will hold decides by content, and drops to `v1` (or `v2` with a variant) when none holds
an id. A directory store's manifest says
`cultcache.store.v5.directory-content-addressed-pages` under the same rule, and `v4`
otherwise.
A single-file store is exactly one MessagePack array of three slots: the header, the schema
catalog and the records. Every runtime refuses a file that is truncated, has bytes after the
array (a second store appended included), is not an array, has fewer or more than three slots
(a header alone included), whose first slot is not a header it reads, or whose catalog or
records it cannot decode (a record with more slots than its header allows, a variant record
it does not read). A runtime never skips a slot it does not understand. It refuses on open
and on rewrite alike, because the rewrite asks the same full reader as the open. Only nothing at the path is a missing
file, and a missing file and an empty array are an empty store; a zero-byte file is not a store and is refused as unreadable.
In C#, a link whose target is gone and a directory at the path are I/O errors, never an empty store. Within a runtime, every
reader and writer of a single-file store shares that one reader, lock and replace (in C#, CultMesh's single-file document
helpers included), and a catalog entry has one layout. A flush or a commit replaces a file only when the
runtime's own reader opens it, so one verdict covers open, flush and commit, and a file the
runtime cannot read is left as it is. Each runtime refuses with one typed error:
`CultStoreUnreadableException` (C#), `CultCacheStoreUnreadable{path, kind}` (Rust, its kind
telling an undecodable store from an unsupported format), `StoreUnreadableError` (TypeScript,
Python), with the underlying cause attached. Rust, TypeScript and Python also open the legacy envelope
array (an array of `key`/`type`/`payload`/`storedAt` maps) and replace it; C# has no legacy
reader and refuses it. The shared byte vectors and each runtime's verdict are
`tests/vectors/document-variants-c2a/readability/manifest.txt`.
A record's payload nests at most 256 MessagePack containers. Every array and map
counts, whether a reader decodes it or skips past it, and the outermost array is
@ -243,17 +408,73 @@ fifth slot:
`elementId` is `""` unless the step enters an object-list element. `value` is the
member's own MessagePack encoding, inline. A top-level Set has one step and `id` `""`;
a reader that resolves only that shape refuses the others, naming the variant.
- A store holding at least one variant declares `cultcache.store.v2`; a store holding
none declares `cultcache.store.v1` and is byte-identical to what a runtime without
variants writes. A `v1` header over a variant record is refused.
- A store holding at least one variant declares `cultcache.store.v2` (`v3` when it holds an
element id); a store holding none declares `cultcache.store.v1` (or `v3`) and is
byte-identical to what a runtime without variants writes. A `v1` header over a variant
record is refused.
- A variant's resolved document is derived in memory when the cache admits the record
or any record on its base chain, and is never written back. A plain write at a
variant's key is refused; `Flatten` replaces a variant with a plain record holding its
resolved document.
- The C# runtime resolves variants. Every other runtime refuses a `v2` store by name.
- The C# runtime resolves variants. Every other runtime refuses a `v2` store by name, and
refuses a variant record inside a `v3` store by its record key.
CultMesh single-file reads, the directory store and CultNet refuse a variant record by
name until each has its own cut.
## Element Ids
Every object that sits in a list, array or dictionary value of a shape the element-id walk
owns carries an id: one string member with an integer `[Key]` and `[CultElementId]`. The
owned shapes are arrays, `IEnumerable<T>` classes and interfaces, `IDictionary<,>` and
`IReadOnlyDictionary<,>` values, and `[MessagePackObject]` or `[Union]` types with integer
keys (a keyed struct included), read through the members MessagePack writes. Union subtypes
are included. The id identifies the element inside its list, so an override can name it (a
path step is `[slot, elementId]`) and a base edit that reorders or inserts elements
retargets nothing.
Any other shape is one whole value. Its elements carry no element ids: a `[CultElementId]`
member inside it is a plain string, never minted or checked and never counted for `v3`. A
variant replaces it whole (operator, 2026-10-16).
- A random id is 12 lowercase hex characters. A derived id
(`[CultElementId(nameof(Source))]`, for content-addressed elements) is the
invariant-culture text of its source member, which must be non-empty; it needs no
other format. An id is unique within its list.
- A write mints random ids for unset elements, including inside variant override
values, and refuses a duplicate, a malformed random id or an empty derived source. A
load refuses the same things, naming the record, the list and the id. A record
written before ids existed loads, and its ids are minted from SHA-256 of the record
key and the element's path, so every reader mints the same ids; they reach disk with
the record's next write, or `CultCache.MintElementIds()`.
- A store that holds an id declares the `v3` header (above), so a reader older than ids
refuses it instead of skipping the id slot and rewriting the elements without ids.
A C# write also marks the store when a record it copies forward holds an id, judged
from the file's bytes at the write (above). TypeScript,
Python and Rust read `v3` and keep a `v3` header they found when they rewrite a
store; they carry ids and do not decide when a document carries them.
- An object list's elements are written by their `[Key]` layout, because the id is a
keyed member of the element. Value lists are replaced whole: a list of a non-keyed
class or struct, or of primitives. So is a list of keyed structs, since a struct is a
value and never an element, but an object list inside a keyed struct is walked like any
other. The walk follows declared member types: a member declared `object` is untyped on
disk, and a subclass's extra members under a non-union base-typed member are not written,
so neither is walked. Whole values are `ImmutableArray<T>`, `ArraySegment<T>`,
`Memory<T>`, `KeyValuePair`, tuples, `Lazy<T>`, a class or struct without
`[MessagePackObject]`, a `[DataContract]` type, a string-keyed type, a type or member
declaring `[MessagePackFormatter]`, and a non-public member of a type without
`AllowPrivate`. The members read are an integer `[Key]` with no `[IgnoreMember]`: a
public field or a property whose getter is public, or any visibility up the base chain
under the most derived `[MessagePackObject(AllowPrivate = true)]` declaration. An id member
needs a setter MessagePack calls (public or `init`; any under `AllowPrivate`), so the id
MessagePack writes is read back. Still refused: a bad id member (missing, duplicate, not
a string, no integer `[Key]`, `[MessagePackFormatter]` on it, one MessagePack does not
write or cannot assign, a key shared with another member, a bad derived source), a
concrete union arm that is not keyed, a keyed type that is also a collection
(`ICollection<T>`, `IList`, `IDictionary`: MessagePack writes it as a collection, so C#
refuses it at the first encode or decode when it is on the path to an object list), a
type MessagePack has no formatter for, and a formatter resolver that claims a type on the
path. A formatter MessagePack's source generator emits is not refused.
## Canonical Semantic Schema Hash
`schemaId` should be derived from a canonical semantic schema hash.
@ -408,8 +629,12 @@ The v1 concurrent single-file policy is:
2. Writers take an exclusive sidecar lock derived from the `.cc` path.
3. Writers re-read the current snapshot after taking the lock.
4. Writers using conditional commit evaluate their conditions against the
latest snapshot and write nothing if one fails. A plain flush and an
unconditional commit compare nothing and are last-writer-wins.
latest snapshot and write nothing if one fails. In a store with no variant, a
plain flush and an unconditional commit compare nothing: they apply their
staged writes onto the current snapshot, last-writer-wins per staged key. A
write into a store that holds or would hold a variant compares every record's
version with what the writer last read, and is refused if any moved. One whose
header they cannot read is refused untouched.
5. Writers write a temp file, flush it, atomically replace the `.cc` file, and
release the lock.

13
src/GameCult.Caching/Contracts/cultcache-schema-compatibility.md
View file

@ -61,6 +61,12 @@ either way is refused: a get-only property, a readonly field without
`AllowPrivate`, or a property whose setter is non-public without
`AllowPrivate`, when the constructor does not fill it.
Never reuse a dropped slot; retire it. A write set copies a record the writer did not touch forward as stored bytes, so an
untouched record keeps a dropped member's value in its slot through every write. A type that later puts a new member in that slot
reads the dropped value as its own. The store does not remember retired slots, and a reuse that changes the slot's shape is refused
at load, but a reuse of the same shape reads silently. This is the rule protobuf, Thrift and MessagePack-CSharp give for numbered
fields: leave the slot empty, and take the next unused number for a new member.
## Which schema a record carries
A record is written under the schema of its runtime type. Writing a derived
@ -123,10 +129,15 @@ lie about the payload:
- target schema change
- name/index lookup semantics change
- missing embedded catalog entry
- no compatible local schema candidate
This is not negotiable. Better a loud refusal than quiet bit-rot.
## Foreign records
A record with no local schema candidate is not rejected: the store carries it untouched as a
foreign record and refuses any write that would replace or remove it. See "foreign record" in
`cultcache-persistence-format.md`.
## Variant overrides
A variant's override follows the same outcome as the same member on a plain record; there

154
src/GameCult.Caching/Contracts/cultcache-store-composition.md
View file

@ -81,9 +81,8 @@ which route wrote it.
after subscribing with a `Sequence` at or below the read is already in the
document and is dropped. CultMesh cache-record mirrors (`ObserveAsync`,
`ReactiveAsync`, and `RefreshAsync` on both) do this.
- Not stale-protected: schema-alias handles (`AsSchemaAlias`) get no
`Sequence`; removals do not advance a mirror's applied `Sequence`, and the
mirrors do not surface removals.
- Not stale-protected: removals do not advance a mirror's applied `Sequence`,
and the mirrors do not surface removals.
- `AddJournal` registers an ordered admission journal, for audit, replication or
a derived index. The cache calls each journal under its gate, right after an
admission is applied and before that admission is published, once per
@ -127,10 +126,39 @@ which route wrote it.
## Locking
- One lock order: the cache's gate, then the store's lock. A store attached to a
cache takes that cache's gate as its own lock, so a store's load callback into
the cache cannot take the two out of order. A store must not call `Loaded`
while holding its own lock outside the cache's hold.
- One lock order: the cache's gate, then the store's gate, then the file lock. A
store's gate may be taken inside a cache's gate (the attach; a journal writing
a cacheless file), never the reverse. What the code enforces, at run time and
by `InvalidOperationException`: a thread holds at most one store's gate;
while it holds a store's gate it enters no cache gate it does not already
hold, reads included; a cacheless write made while the thread already holds
that store's gate is refused; and a cache's hold (a write, a flush, an attach)
entered inside another cache's hold is refused. What it does not enforce: a cache
read made inside another cache's hold takes that cache's gate without a
check, so two threads each holding one cache and reading the other can
deadlock. That is open as `variants:follow_up:cross-cache-read-lock-order`;
until it lands, a journal or a getter must not read a second cache. User code
a cacheless call or an attach runs under the store's gate (a document's
getter, a constructor while a copied record is read) is the way into the
refused cases.
- One owner picks the gate a store call holds. A store attached to a cache takes
that cache's gate as its own lock, so a store's load callback into the cache
cannot take the two out of order; a cacheless store takes its own gate. The
store reads which one applies, takes it, and checks the answer again under it:
`Cache` is written only by the attach, holding both gates: it publishes `Cache`
before its pull and takes it back in the same hold if the pull fails. Another
thread's unlocked guess may read the tentative cache, but none runs a body
against it: the guess is checked under that cache's gate, which the attach
holds, and fails once the pull retracts it. Under the store's gate `Cache` is
settled, and a settled cache never changes, because nothing detaches. A call
takes at most three gates: the guessed cache's, the store's, the settled
cache's. The pull reads the registry the attach lent, not `Cache`; a variant it
loads carries no codec, and the cache resolves it with its own. During an
attach the store is attached, so a journal the pull runs that calls the store
is refused as it would be afterwards; a cacheless write that finds a cache
published, during an attach or after it, is refused with the same text. A
store must not call `Loaded` while holding its own lock outside the cache's
hold.
- A journal runs under the cache's gate, so a slow journal slows every writer of
that cache. It must not enter the cache, and must not wait for anything that
needs the cache or its gate: another thread's write, a lock a writer holds.
@ -140,8 +168,6 @@ which route wrote it.
CultNet journal, under the cache's gate: it must not call the cache or wait
for a thread that does. A store that writes its own cache throws through the
journal's reentry guard, and the admission stands.
- A thread holds one cache's gate at a time: entering a cache's hold inside
another cache's hold throws.
- A store doing I/O on behalf of its cache (pull, flush, commit) holds the gate,
so that cache's readers wait for the I/O. The file lock (`<path>.lock` or the
directory commit lease) is always taken inside the gate and released before a
@ -223,26 +249,12 @@ against what is durably on disk at that moment:
constrained; unrelated concurrent writes do not fail the commit. Passing any
instance other than the one the cache holds throws.
- `ExpectUnchanged()`: the store's persisted record set must equal what this
cache last loaded from it, compared as the ordered list of
`(key, schemaId, storedAt)`. Any insert, delete or replace fails it.
cache last loaded from it, compared as the ordered list of `(key, version)`.
Any insert, delete or replace fails it.
Record identity for conditions is `(schemaId, storedAt)`, where `schemaId` is
the id the record is persisted under (the internal `CultStoredDocument.PersistedSchemaId`),
not the one this cache decodes it as. A value change mints a `storedAt`
strictly later than the record it replaces; writers bump a minted timestamp by
one tick when it is not later. An entry loaded under compatible schema drift
keeps its persisted id until its store rewrites it; a store records each
record its completed write rewrote with the internal instance method
`CacheBackingStore.MarkWritten` (the shipped stores call it; it is not part of
the public API), which refuses an entry that is not the one
the store holds in its own `Entries` at that key. That guards against a store
marking by mistake an entry it never held; a store that writes another copy
into its own `Entries` and then marks it is trusted to have rewritten that
record, so a mark is the store's own claim. A store that does not mark leaves
the entry's `Expect` failing closed until the next pull. A whole-file flush or
unconditional commit rewrites every record under the writer's current schema
and keeps its `storedAt`, so another cache's `Expect` on a drifted record it
read before that rewrite fails until it re-reads the store.
Record identity for conditions is the record's version, the SHA-256 of its stored
bytes (cultcache-persistence-format.md). Any write that changes a record changes
its version, whatever storedAt it carries.
- A failed condition is a lost race, not an error: `Commit` returns false and
nothing is written, changed in memory, or published. `TryCommit` makes one
@ -268,17 +280,33 @@ read before that rewrite fails until it re-reads the store.
match its content address, at most five times; a pull that has not settled
by then throws.
**A plain flush and an unconditional commit are last-writer-wins: per file for
single-file stores, per key for directory stores.** Both run
under the same lock, so two writers never interleave bytes, but they compare
nothing. Only a conditional commit (`Expect` or `ExpectUnchanged`) protects
against another writer; processes sharing a store must all use it.
**In a store holding no variant, a plain flush and an unconditional commit are
last-writer-wins per staged key, in both store kinds.** Both run under the same lock, so two writers never
interleave bytes, but they compare nothing. Only a conditional commit
(`Expect` or `ExpectUnchanged`) tells a writer another writer moved the keys it
depends on. A writer wins only over a file it can read: a single-file store
refuses to replace a file whose header it cannot read, and leaves it as it was.
- A single-file store writes this cache's whole view of the file: a record
another writer added since this cache last pulled is gone.
- A directory store writes this cache's staged keys onto the current manifest:
another writer's unrelated keys survive, a key both wrote holds the last
write.
- A single-file store and a directory store apply this cache's staged writes and
removals onto what the store holds now: another writer's unrelated records
survive as stored, a key both wrote holds the last write.
- In both store kinds, a write or removal at a key whose durable record is under
an id the writer does not declare is refused with the typed schema conflict
before anything is written, and that record is reloaded as the store holds
it: dropped when foreign, held read-only when a type reads it
(cultcache-persistence-format.md, ownership by declaration).
- A removal of a key the cache does not hold travels as a check-only key: a
commit carries it in the request, a flush stages it, and either store kind
judges it under its lock with the keys it writes, never removing it. Every
writable store judges a batch's checks under its own lock: the stores the
batch does not commit through first (`JudgeChecks`), the home last. A batch
that only checks keys writes no file. `ExpectUnchanged` compares the foreign
records a store carries at the versions it read them; a store kind lists a
foreign record through `Foreign` with the version it read.
- A single-file store holding a variant (the directory store refuses variants)
is written only from what its writer last read. A record that moved refuses
the whole write with the typed write conflict, and the refused and moved keys
are reloaded (cultcache-persistence-format.md).
- An unconditional commit writes exactly what a flush of the same staged state
plus the batch would write. It also persists any single writes staged earlier
in that store, and leaves the store clean.
@ -301,6 +329,25 @@ and Rust's open fails rather than waits while a C# writer holds the lock
for `X` that `Doc<X>` needs is declared in `Doc`'s assembly.
- An assembly that declares nothing, and the store envelope, use the base
options.
- A resolver may not claim a type a document's element lists pass through: the list's
element types, the types and members on the way to them, and the collection types
the walk passes (`List<T>`, `T[]`, a collection class, a dictionary on the way).
Those are written by their `[Key]` layout so each element's id reaches disk, and the
codec refuses the document on first encode or decode, naming the resolver and the
type. The formatter `StandardResolver` or `StandardResolverAllowPrivate` hands out for a
type does not claim it: the extra non-public members the `AllowPrivate` formatter writes
are whole values the walk does not own. A type or member declaring
`[MessagePackFormatter]` is a whole value off the path, the supported way to give a type its
own formatter. A value list (`List<float2>`, a list of a non-keyed class) is not on the
path, so a resolver may claim it.
- The codec also checks, for every type on that path, that `StandardResolver` does not write it
against the walk. A type read by key may not be written by the generic collection resolver
(a `[MessagePackObject]` class that is also an `ICollection<T>` or dictionary); it is
refused on first encode or decode, naming the type. Any other formatter, such as one
MessagePack's source generator emits, passes: the walk decides which types carry ids and
the door only verifies. It also refuses, typed, a type MessagePack cannot build a formatter
for (it throws, or `StandardResolver` returns none), and a resolver that throws while asked
about one.
- Deserialization runs under untrusted-data security. `CultRecordRef<T>` is a
permitted dictionary key, compared by its key string.
@ -327,6 +374,37 @@ type name is the CLR full name, the same rule as `System.Int32`. A reader in
another runtime needs only the component count and primitive type. CultMath has
no serialization; a CultMath encoding, if added, uses this shape.
## Cacheless single-file access
A single-file store file can be read and written without a cache, through public members of `SingleFileBackingStore` (for
example `SingleFileMessagePackBackingStore`): `ReadStoreSnapshot()`, `WriteRecord(record, catalogEntry, holdsIds, registry)`,
and `WriteDocument<T>(key, document, storedAt, registry)`. Decoding a stored payload needs no file, so it is on the codec, not the store:
`CultCodec.DecodeDocument<T>(key, payload)` (`CultCacheMessagePack.CreateCodec(registry)` makes the codec).
- `ReadStoreSnapshot()` reads the file once, under the store's gate (or its cache's gate when attached, validated after it is taken), and returns the durable snapshot (records, schema catalog,
format stamp), or null when nothing is at the path. It throws `CultStoreUnreadableException` for a file the reader refuses.
- `WriteRecord` writes one record under the store's lock: the record replaces the one at its key, every other record and catalog
entry is copied forward as stored, and the file is replaced atomically. It takes a payload the caller already encoded, so a
payload it cannot read leaves the header as the file has it (`holdsIds` null).
- `WriteDocument<T>` writes one typed document as a cache write does: it plans element ids, encodes the document as `T` (the type
its schema id and ids are taken from; a derived instance is stored as `T`) with the registry, and stores it as `WriteRecord`
does. A failed write gives the minted ids back.
- `CultCodec.DecodeDocument<T>` decodes a payload read from the store as a cache load does: it mints the ids a load mints and
refuses what a cache load refuses. It reads no file and holds no state.
The `registry` argument of `WriteRecord` and `WriteDocument` serves that one call: the store attaches it under its gate for the call
and keeps none, so the same call decides the same header on a fresh instance and on one that has served other calls.
They share everything with a cache's store and nothing is re-implemented beside it: one reader (the stamp is judged before the
body, and a file the reader refuses is refused by a read and by a write alike), the store's lock, the atomic replace, and the
header rule (`holdsIds` true marks a v3 store; false leaves it as the rule decides; null, for a payload the caller cannot read,
keeps whatever the file already says). Ownership by declaration binds a cacheless write too: a record at the key under a schema id
the written catalog entry does not own or list as compatible is refused with `CultSchemaConflictException`, and the file is left as
it was. Element-id planning (`CultElementIds`) is internal to the library: a caller never plans or marks ids, the store does. A store
attached to a cache refuses `WriteRecord` and `WriteDocument` (write through the cache, whose staged writes land through the
same private write) with `InvalidOperationException`. CultMesh's single-file document helpers (`CultMesh.WriteSingleFileDocument`,
`ReadSingleFileDocument` and their payload forms) are built on these members and nothing else of the store.
## Runtime status
| Rule | C# | Rust | TypeScript | Python |

2012
src/GameCult.Caching/CultCache.cs
View file

File diff suppressed because it is too large Load diff

8
src/GameCult.Caching/CultDocumentContracts.cs
View file

@ -18,6 +18,14 @@ namespace GameCult.Caching
public string SchemaName { get; }
public string SchemaVersion { get; }
/// <summary>
/// Schema ids other than this type's own that a store may hold its records under, typically the ids an earlier
/// version of the schema had. The registered catalog entry lists them, and a cache resolves a record stored under
/// one of them to this type (its own id is tried first) and restamps it to the type's own id on load. TypeScript
/// and Python name the same declaration <c>compatibleSchemaIds</c>.
/// </summary>
public string[] CompatibleSchemaIds { get; set; } = Array.Empty<string>();
}
[AttributeUsage(AttributeTargets.Class, Inherited = false)]

684
src/GameCult.Caching/CultElementIds.cs Normal file
View file

@ -0,0 +1,684 @@
using System;
using System.Collections;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Reflection;
using System.Runtime.Serialization;
using System.Security.Cryptography;
using System.Text;
using MessagePack;
namespace GameCult.Caching
{
// The identity of one element of an object list. A string member, keyed like any other, on every object type that
// sits in a list of a registered document type (union bases included: every concrete subtype carries the member).
// [CultElementId(nameof(Offset))] declares the id derived, never random: an unset id becomes the invariant-culture text of
// that member (a keyed string or number on the same type, unique within its list). Content-addressed elements use it so the
// same content always encodes to the same bytes.
[AttributeUsage(AttributeTargets.Field | AttributeTargets.Property, Inherited = true, AllowMultiple = false)]
public sealed class CultElementIdAttribute : Attribute
{
public CultElementIdAttribute(string? derivedFrom = null)
{
DerivedFrom = derivedFrom;
}
public string? DerivedFrom { get; }
}
// A record the id rule refuses: a duplicate id in one list, a random id that is not 12 lowercase hex characters, or a derived
// id whose source member is null or empty. It names the record, the list (record key, member slot, dictionary key and
// element index) and, where there is one, the element id and the source member.
public sealed class CultElementIdException : InvalidOperationException
{
public CultElementIdException(string message, string recordKey, string listPath, string? elementId = null, string? member = null)
: base(message)
{
RecordKey = recordKey;
ListPath = listPath;
ElementId = elementId;
Member = member;
}
public string RecordKey { get; }
public string ListPath { get; }
public string? ElementId { get; }
public string? Member { get; }
}
// The one owner of element identity: which types need an id, and how unset ids are filled.
// A random id is 12 lowercase hex characters (48 bits); a derived id is the invariant text of its source and only has to be
// non-empty. Either need only be unique within one list. A write mints a random id; a load of a store written before ids
// existed mints from SHA-256(record key, element path), so every reader of the same store mints the same id and a reload is
// identical to what was read. A write and a load refuse the same things.
internal static class CultElementIds
{
private const int IdBytes = 6;
/// <summary>The element ids a pass would fill, decided before anything changes. <see cref="Apply"/> sets them; <see cref="Undo"/> gives back what Apply replaced.</summary>
internal sealed class IdPlan
{
private readonly List<(object Item, Shape Shape, string? Before, string Id)> _fills = new();
public int Count => _fills.Count;
internal void Add(object item, Shape shape, string? before, string id) => _fills.Add((item, shape, before, id));
public void Apply()
{
foreach (var (item, shape, _, id) in _fills)
shape.SetId!(item, id);
}
public void Undo()
{
foreach (var (item, shape, before, _) in _fills)
shape.SetId!(item, before!);
}
}
internal sealed class Shape
{
public Func<object, string?>? GetId;
public Action<object, string>? SetId;
public Func<object, string?>? Derive;
public string? DerivedFrom;
public (int Slot, Type Type, Func<object, object?> Get)[] Walk = Array.Empty<(int, Type, Func<object, object?>)>();
}
private static readonly ConcurrentDictionary<Type, Shape> Shapes = new();
// ---- registration: which element types must carry an id ----
internal sealed class Survey
{
public List<string> Problems { get; } = new();
// Every type whose bytes carry an element id: the types a walk descends through to reach an object list, the list's
// element types and every concrete type under them. A serialization codec must write all of these by their [Key]
// layout, so it checks them against the formatters it composed.
public List<Type> IdPath { get; } = new();
// The concrete element types already judged in this document, so each is refused at most once however many lists hold it.
internal HashSet<Type> Judged { get; } = new();
}
// What the walk saw, settled once it is complete: a type reaches a list only through types reached later, and a type
// can sit in a cycle, so the path is a fixed point over these edges and not a stack.
private sealed class Graph
{
public List<(Type Owner, MemberInfo? Member, Type Child, Type[] Through)> Edges { get; } = new();
public List<(Type Owner, MemberInfo Member, Type Element, List<Type> Concrete, Type[] Through)> Lists { get; } = new();
// Keyed types that also enumerate objects: the walk reads them by key, so the codec must confirm MessagePack does too.
public List<Type> Enumerating { get; } = new();
}
private static readonly ConcurrentDictionary<Type, bool> IdCapable = new();
// A document type whose walk reaches no object list can hold no element id, so a write never decodes one to ask.
internal static bool CanHoldIds(Type documentType) => IdCapable.GetOrAdd(documentType, type => Inspect(type).IdPath.Count > 0);
// Every reason this document type cannot be registered under the id rule, one per offending concrete element type.
internal static Survey Inspect(Type documentType)
{
var survey = new Survey();
var graph = new Graph();
VisitObject(documentType, documentType, new HashSet<Type>(), survey, graph);
Settle(documentType, survey, graph);
return survey;
}
private static void VisitObject(Type documentType, Type type, HashSet<Type> seen, Survey survey, Graph graph)
{
if (!seen.Add(type))
return;
if (EnumeratesObjects(type))
graph.Enumerating.Add(type);
foreach (var (member, memberType) in KeyedMembers(type))
VisitMemberType(documentType, type, member, memberType, Array.Empty<Type>(), seen, survey, graph);
foreach (var union in type.GetCustomAttributes<UnionAttribute>(true))
{
graph.Edges.Add((type, null, union.SubType, Array.Empty<Type>()));
VisitObject(documentType, union.SubType, seen, survey, graph);
}
}
private static void VisitMemberType(Type documentType, Type owner, MemberInfo member, Type memberType, Type[] through, HashSet<Type> seen, Survey survey, Graph graph)
{
memberType = Nullable.GetUnderlyingType(memberType) ?? memberType;
// A sequence or map whose element is itself (an unkeyed Node : IEnumerable<Node>) ends the type walk here; without
// this it would recurse until the stack overflows.
if (Array.IndexOf(through, memberType) >= 0)
return;
var (kind, inner) = Classify(memberType);
switch (kind)
{
case Kind.Leaf:
// A shape the walk does not own is one whole value, never refused (q3 amended).
return;
case Kind.Map:
VisitMemberType(documentType, owner, member, inner, through.Append(memberType).ToArray(), seen, survey, graph);
return;
case Kind.Sequence when IsElement(inner):
var concretes = ConcreteTypes(inner).ToList();
graph.Lists.Add((owner, member, inner, concretes, through.Append(memberType).ToArray()));
foreach (var concrete in concretes)
{
if (!survey.Judged.Add(concrete))
continue;
var problem = IsElement(concrete) ? IdProblem(concrete) : UnkeyedElementProblem;
if (problem != null)
survey.Problems.Add($"Cult document {documentType.Name}: list element type {concrete.Name} (in {owner.Name}.{member.Name}) {problem}");
}
VisitObject(documentType, inner, seen, survey, graph);
return;
case Kind.Sequence:
VisitMemberType(documentType, owner, member, inner, through.Append(memberType).ToArray(), seen, survey, graph);
return;
default:
graph.Edges.Add((owner, member, memberType, through));
VisitObject(documentType, memberType, seen, survey, graph);
return;
}
}
private const string UnkeyedElementProblem =
"is written by a formatter the cache does not own, so its element id would never reach disk; mark it [MessagePackObject] with [Key] members, or keep it out of lists.";
// The id path: every type that reaches an object list through members MessagePack writes by key, the lists' element and
// concrete types, and the keyed types that enumerate objects. The codec verifies that MessagePack writes each of them the
// way Classify reads it; it never decides ownership.
private static void Settle(Type documentType, Survey survey, Graph graph)
{
var lists = graph.Lists;
var reaching = new HashSet<Type>(lists.Select(list => list.Owner));
var through = new List<Type>(lists.SelectMany(list => list.Through));
for (var grew = true; grew;)
{
grew = false;
foreach (var (owner, _, child, via) in graph.Edges)
{
if (!reaching.Contains(child))
continue;
through.AddRange(via);
grew |= reaching.Add(owner);
}
}
var path = new List<Type>();
void Add(Type type)
{
if (!path.Contains(type))
path.Add(type);
}
foreach (var type in reaching)
Add(type);
foreach (var type in through)
Add(type);
foreach (var list in lists)
{
Add(list.Element);
foreach (var concrete in list.Concrete)
Add(concrete);
}
// A keyed type the walk reads by key but whose enumeration holds objects is on the path: the codec asks whether
// MessagePack writes it by key too.
foreach (var type in graph.Enumerating)
Add(type);
survey.IdPath.AddRange(path);
}
// The types an element can actually be: the element type itself when it can be instantiated, and every union subtype
// under it. Shapes are built from these, so registration judges exactly what minting will meet.
private static IEnumerable<Type> ConcreteTypes(Type element)
{
var seen = new HashSet<Type>();
return Expand(element).ToList();
IEnumerable<Type> Expand(Type type)
{
if (!seen.Add(type))
yield break;
if (!type.IsAbstract && !type.IsInterface)
yield return type;
foreach (var union in type.GetCustomAttributes<UnionAttribute>(true))
{
foreach (var subtype in Expand(union.SubType))
yield return subtype;
}
}
}
private static string? IdProblem(Type element)
{
// Candidates come from every instance member up the base chain, public or not, so an id MessagePack does not write
// is named as such instead of being reported missing. An override or a hiding member is one member, the most derived.
var ids = AllInstanceMembers(element).Where(member => Declares(member, typeof(CultElementIdAttribute)))
.GroupBy(member => member.Name).Select(group => group.First()).ToArray();
if (ids.Length == 0)
return "has no [CultElementId] member; add a public string member with [Key] and [CultElementId] so each element has an identity.";
if (ids.Length > 1)
return $"has two [CultElementId] members ({ids[0].Name}, {ids[1].Name}); an element has one identity.";
var id = ids[0];
if (Declares(id, typeof(MessagePackFormatterAttribute)))
return $"declares [MessagePackFormatter] on its [CultElementId] member {id.Name}, so the id would not be written by its [Key].";
if (TypeOf(id) != typeof(string))
return $"declares [CultElementId] on {id.Name}, which is not a string.";
if (KeyOf(id) is not { } idKey)
return $"declares [CultElementId] on {id.Name}, which has no integer [Key], so it is never persisted.";
if (MembersOf(element).FirstOrDefault(member => member.Name == id.Name && member.MemberType == id.MemberType) is not { } written)
return $"declares [CultElementId] on {id.Name}, which MessagePack does not write ([IgnoreMember], or non-public on a type without [MessagePackObject(AllowPrivate = true)]); the id would never reach disk.";
id = written;
var sharing = KeyedMembers(element).FirstOrDefault(entry => entry.Member != id && KeyOf(entry.Member) == idKey);
if (sharing.Member != null)
return $"declares [CultElementId] on {id.Name} with [Key({idKey})], which {sharing.Member.Name} also uses; an id needs a key of its own.";
if (id is PropertyInfo { SetMethod: null } or FieldInfo { IsInitOnly: true })
return $"declares [CultElementId] on {id.Name}, which cannot be assigned; the cache mints into it.";
if (id is PropertyInfo { SetMethod.IsPublic: false } && !AllowsPrivate(element))
return $"declares [CultElementId] on {id.Name}, whose setter is not public: MessagePack writes the id but does not read it back, so every load would mint a different one. Make the setter public (init is fine).";
var source = id.GetCustomAttribute<CultElementIdAttribute>(true)!.DerivedFrom;
if (source != null)
{
var from = MembersOf(element).FirstOrDefault(member => member.Name == source);
if (from == null || KeyOf(from) == null)
return $"derives its [CultElementId] from {source}, which is not a keyed member of the type.";
var kind = Nullable.GetUnderlyingType(TypeOf(from)) ?? TypeOf(from);
if (kind != typeof(string) && !kind.IsPrimitive)
return $"derives its [CultElementId] from {source}, which is not a string or a number.";
}
return null;
}
// ---- write: what the stored form holds ----
// True when some element under root holds a non-empty id: the one fact a store marks its header by. It reads the value
// as it is now, refuses nothing and changes nothing, so a writer asks it of exactly what it is about to write. The walk
// follows declared types, as registration does; a value's own type picks only a union arm.
/// <summary>True when some element under <paramref name="root"/> holds a non-empty id: the one fact a single-file store marks its header by. Reads the value as it is, refuses nothing and changes nothing.</summary>
/// <param name="declared">The declared type of the root; the walk follows declared types, as registration does.</param>
/// <param name="root">The document about to be written.</param>
public static bool Holds(Type declared, object? root)
{
if (root == null)
return false;
var (kind, inner) = Classify(declared);
switch (kind)
{
case Kind.Leaf:
return false;
case Kind.Map:
return MapEntries(declared)(root).Any(entry => Holds(inner, entry.Value));
case Kind.Sequence:
var elements = IsElement(inner);
if (!elements && Classify(inner).Kind == Kind.Leaf)
return false;
foreach (var item in (IEnumerable)root)
{
if (item == null)
continue;
if (elements && OwnerShape(inner, item).GetId is { } getId && !string.IsNullOrEmpty(getId(item)))
return true;
if (Holds(inner, item))
return true;
}
return false;
default:
return OwnerShape(declared, root).Walk.Any(member => Holds(member.Type, member.Get(root)));
}
}
// ---- write and load: mint and check ----
// Decides every unset element id under root, and refuses what the rule refuses, without changing anything: a random id
// that is not 12 lowercase hex characters, a duplicate id in one list, a derived id whose source is null or empty.
// rootPath (the record key) seeds deterministic minting and names the record in a refusal. An element object that sits
// twice in one list is refused too: planning would give it two ids and it would then be a duplicate on load.
/// <summary>Decides every unset element id under <paramref name="root"/> and refuses what a cache write refuses (a malformed or duplicate id, an unusable derived source), without changing anything until <see cref="IdPlan.Apply"/>. A cacheless single-file writer plans before it serializes, so the ids are in the bytes.</summary>
/// <param name="declared">The declared type of the root.</param>
/// <param name="root">The document being written or loaded.</param>
/// <param name="rootPath">The record key: it seeds deterministic minting and names the record in a refusal.</param>
/// <param name="deterministic">True on a load, which mints from the key and path so every reader of one store mints the same ids; false on a write, which mints random ones.</param>
public static IdPlan Plan(Type declared, object? root, string rootPath, bool deterministic)
{
var plan = new IdPlan();
Visit(declared, root, rootPath);
return plan;
void Visit(Type type, object? value, string path)
{
if (value == null)
return;
var (kind, inner) = Classify(type);
switch (kind)
{
case Kind.Map:
foreach (var entry in MapEntries(type)(value))
Visit(inner, entry.Value, path + "{" + Convert.ToString(entry.Key, CultureInfo.InvariantCulture) + "}");
break;
case Kind.Sequence when IsElement(inner):
Elements(inner, (IEnumerable)value, path);
break;
case Kind.Sequence:
if (Classify(inner).Kind == Kind.Leaf)
break;
var index = 0;
foreach (var item in (IEnumerable)value)
Visit(inner, item, path + "/" + (index++).ToString(CultureInfo.InvariantCulture));
break;
case Kind.Owner:
foreach (var (slot, memberType, get) in OwnerShape(type, value).Walk)
Visit(memberType, get(value), path + "." + slot.ToString(CultureInfo.InvariantCulture));
break;
}
}
void Elements(Type element, IEnumerable list, string path)
{
var items = list.Cast<object?>().ToArray();
var taken = new HashSet<string>(StringComparer.Ordinal);
var objects = new HashSet<object>(ReferenceComparer.Instance);
foreach (var item in items)
{
if (item == null || OwnerShape(element, item) is not { GetId: { } getId } shape)
continue;
if (!objects.Add(item))
throw new CultElementIdException(
$"Record {rootPath}: the same {item.GetType().Name} object appears twice in the list at {path}; an element is one object with one id.",
rootPath, path);
var id = getId(item);
if (string.IsNullOrEmpty(id))
continue;
if (shape.Derive == null && !IsRandomId(id!))
throw new CultElementIdException(
$"Record {rootPath}: element id '{id}' in the list at {path} is not 12 lowercase hex characters. An id you write by hand must be 12 lowercase hex; for readable ids, derive them from a member with [CultElementId(nameof(Member))].",
rootPath, path, id);
if (!taken.Add(id!))
throw Duplicate(id!, path);
}
for (var index = 0; index < items.Length; index++)
{
var item = items[index];
if (item == null)
continue;
var itemPath = path + "/" + index.ToString(CultureInfo.InvariantCulture);
if (OwnerShape(element, item) is { SetId: { } , GetId: { } get } itemShape &&
string.IsNullOrEmpty(get(item)))
{
string id;
if (itemShape.Derive != null)
{
id = itemShape.Derive(item) ?? string.Empty;
if (id.Length == 0)
throw new CultElementIdException(
$"Record {rootPath}: the element at {itemPath} derives its id from {itemShape.DerivedFrom}, which is null or empty.",
rootPath, itemPath, member: itemShape.DerivedFrom);
if (!taken.Add(id))
throw Duplicate(id, path);
}
else
id = Mint(itemPath, deterministic, rootPath, taken);
plan.Add(item, itemShape, get(item), id);
}
Visit(element, item, itemPath);
}
}
CultElementIdException Duplicate(string id, string path) => new(
$"Record {rootPath}: element id '{id}' appears twice in the list at {path}; an element id is unique within its list.",
rootPath, path, id);
}
private sealed class ReferenceComparer : IEqualityComparer<object>
{
public static readonly ReferenceComparer Instance = new();
bool IEqualityComparer<object>.Equals(object? left, object? right) => ReferenceEquals(left, right);
int IEqualityComparer<object>.GetHashCode(object value) => System.Runtime.CompilerServices.RuntimeHelpers.GetHashCode(value);
}
private static bool IsRandomId(string id)
{
if (id.Length != IdBytes * 2)
return false;
foreach (var character in id)
{
if (!(character is >= '0' and <= '9' or >= 'a' and <= 'f'))
return false;
}
return true;
}
private static string Mint(string itemPath, bool deterministic, string rootPath, HashSet<string> taken)
{
for (var attempt = 0; ; attempt++)
{
var bytes = new byte[IdBytes];
if (deterministic)
{
using var sha = SHA256.Create();
var hash = sha.ComputeHash(Encoding.UTF8.GetBytes(rootPath + "|" + itemPath + "|" + attempt.ToString(CultureInfo.InvariantCulture)));
Array.Copy(hash, bytes, IdBytes);
}
else
RandomBytes(bytes);
var id = Hex(bytes);
if (taken.Add(id))
return id;
}
}
private static void RandomBytes(byte[] bytes)
{
using var random = RandomNumberGenerator.Create();
random.GetBytes(bytes);
}
private static string Hex(byte[] bytes)
{
var text = new StringBuilder(bytes.Length * 2);
foreach (var value in bytes)
text.Append(value.ToString("x2", CultureInfo.InvariantCulture));
return text.ToString();
}
// ---- shapes ----
private static Shape ShapeOf(Type type) => Shapes.GetOrAdd(type, Build);
// The shape of an owner or element a walk holds under a declared type: a union base writes its arm, so the value's own
// type picks the arm; any other declared type is written by its own [Key] layout, whatever a subclass adds.
private static Shape OwnerShape(Type declared, object value)
{
declared = Nullable.GetUnderlyingType(declared) ?? declared;
return declared.IsDefined(typeof(UnionAttribute), false) ? ShapeOf(value.GetType()) : ShapeOf(declared);
}
private static Shape Build(Type type)
{
var shape = new Shape();
var idMember = MembersOf(type).FirstOrDefault(member => Declares(member, typeof(CultElementIdAttribute)));
if (idMember != null)
{
shape.GetId = idMember is FieldInfo field ? o => (string?)field.GetValue(o) : o => (string?)((PropertyInfo)idMember).GetValue(o);
shape.SetId = idMember is FieldInfo f ? (o, v) => f.SetValue(o, v) : (o, v) => ((PropertyInfo)idMember).SetValue(o, v);
if (idMember.GetCustomAttribute<CultElementIdAttribute>(true)!.DerivedFrom is { } source &&
MembersOf(type).FirstOrDefault(member => member.Name == source) is { } from)
{
shape.DerivedFrom = source;
shape.Derive = o => Convert.ToString(from is FieldInfo ff ? ff.GetValue(o) : ((PropertyInfo)from).GetValue(o), CultureInfo.InvariantCulture);
}
}
shape.Walk = KeyedMembers(type)
.Where(entry => Classify(entry.Type).Kind != Kind.Leaf)
.Select(entry => (Slot: KeyOf(entry.Member)!.Value, entry.Type, Get: entry.Member is FieldInfo fi
? (Func<object, object?>)(o => fi.GetValue(o))
: o => ((PropertyInfo)entry.Member).GetValue(o)))
.ToArray();
return shape;
}
// The members MessagePack's StandardResolver writes for an integer-keyed type, none of them [IgnoreMember]. Without
// AllowPrivate: public instance fields, and public instance properties with a public getter. With it, read from the most
// derived [MessagePackObject] declaration (a derived type without it is written without it, probe NoApDerived): every
// instance field and indexer-free property up the base chain at any visibility, the most derived of a name.
// TheWalkReadsTheMembersMessagePackWrites checks this set against MessagePack's own member discovery.
private static IEnumerable<MemberInfo> MembersOf(Type type) =>
AllowsPrivate(type)
? AllInstanceMembers(type).Where(member => !Declares(member, typeof(IgnoreMemberAttribute)))
.GroupBy(member => member.Name).Select(group => group.First())
: PublicMembersOf(type);
private static bool AllowsPrivate(Type type) => type.GetCustomAttribute<MessagePackObjectAttribute>(true)?.AllowPrivate == true;
private static IEnumerable<MemberInfo> PublicMembersOf(Type type) =>
type.GetFields(BindingFlags.Instance | BindingFlags.Public).Cast<MemberInfo>()
.Concat(type.GetProperties(BindingFlags.Instance | BindingFlags.Public)
.Where(property => property.GetIndexParameters().Length == 0 && property.GetMethod is { IsPublic: true }))
.Where(member => !Declares(member, typeof(IgnoreMemberAttribute)));
// Every instance field and indexer-free property declared on the type or any base, most derived first.
private static IEnumerable<MemberInfo> AllInstanceMembers(Type type)
{
const BindingFlags flags = BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.DeclaredOnly;
for (var level = type; level != null && level != typeof(object); level = level.BaseType)
{
foreach (var field in level.GetFields(flags))
yield return field;
foreach (var property in level.GetProperties(flags).Where(property => property.GetIndexParameters().Length == 0))
yield return property;
}
}
// The [Key] members MessagePack writes, for the oracle test that checks them against MessagePack.
internal static IEnumerable<MemberInfo> WrittenKeyedMembers(Type type) => MembersOf(type).Where(member => KeyOf(member) != null);
private static IEnumerable<(MemberInfo Member, Type Type)> KeyedMembers(Type type) =>
WrittenKeyedMembers(type).Where(member => !Declares(member, typeof(MessagePackFormatterAttribute)))
.Select(member => (member, TypeOf(member)));
// An attribute a member declares, or inherits through an override: MemberInfo.IsDefined ignores inherit for properties, so an
// override of a base virtual member would not see its base's attribute.
private static bool Declares(MemberInfo member, Type attribute) => Attribute.IsDefined(member, attribute, inherit: true);
// True when the walk reads the type by its [Key] members: the codec compares this with how MessagePack writes it.
internal static bool WalksAsOwner(Type type) => Classify(Nullable.GetUnderlyingType(type) ?? type).Kind == Kind.Owner;
private static bool EnumeratesObjects(Type type) =>
(MapTypes(type)?.Value ?? ElementTypeOf(type)) is { } inner && Classify(inner).Kind != Kind.Leaf;
private static int? KeyOf(MemberInfo member) => member.GetCustomAttribute<KeyAttribute>(true)?.IntKey;
private static Type TypeOf(MemberInfo member) => member is FieldInfo field ? field.FieldType : ((PropertyInfo)member).PropertyType;
// ---- type classification ----
private enum Kind { Leaf, Map, Sequence, Owner }
private static readonly ConcurrentDictionary<Type, (Kind Kind, Type Inner)> Classes = new();
// The walk's model of a declared type: registration, Plan, Holds and the codec's id path all go through it. A
// [MessagePackObject] or [Union] type is read by key even when it is also IEnumerable<T>; the codec checks, for every type
// on the id path, that MessagePack writes it the way this model reads it, and refuses the document when it does not.
// Inner is a map's value type, a sequence's element type, or the type itself for an owner.
// The owned shapes are a closed list (Owner, Sequence, Map); every other shape is Leaf: one whole value with no element
// ids, replaced whole by a variant (q3 amended 2026-10-16). When in doubt the answer is Leaf.
// Leaf strings (an IEnumerable<char> MessagePack writes whole), a type written whole (IsWrittenWhole: attribute-
// formatted, [DataContract], string-keyed), a struct or a class or interface with no [MessagePackObject] or
// [Union] and no collection shape (primitives, enums, float2, DateTime, ImmutableArray, ArraySegment, Memory,
// KeyValuePair, tuples, Lazy, unkeyed classes), and object, which is untyped on disk.
// Map a class or interface implementing IDictionary<,> or IReadOnlyDictionary<,>.
// Sequence an array or an IEnumerable<T> class or interface.
// Owner a class, interface or struct MessagePack serializes by integer key. A keyed struct is walked as an owner;
// it is never an element (IsElement), since a boxed copy cannot carry a minted id.
private static (Kind Kind, Type Inner) Classify(Type declared) => Classes.GetOrAdd(declared, static type =>
{
type = Nullable.GetUnderlyingType(type) ?? type;
if (type == typeof(string))
return (Kind.Leaf, type);
if (IsWrittenWhole(type))
return (Kind.Leaf, type);
if (IsKeyed(type))
return (Kind.Owner, type);
if (type.IsValueType)
return (Kind.Leaf, type);
if (MapTypes(type) is { } map)
return (Kind.Map, map.Value);
if (ElementTypeOf(type) is { } element)
return (Kind.Sequence, element);
return (Kind.Leaf, type);
});
// A type MessagePack writes as one value the walk cannot look into: its own formatter ([MessagePackFormatter] on the type,
// inherited), [DataContract] on a type that is not [MessagePackObject] (the object resolver writes its members), or string
// keys (KeyAsPropertyName, or a [Key] with a string key on a member MessagePack writes: a private or [IgnoreMember] one is not written). Such a type is Leaf, even when it is also enumerable.
private static bool IsWrittenWhole(Type type) =>
type.IsDefined(typeof(MessagePackFormatterAttribute), true) ||
(type.IsDefined(typeof(DataContractAttribute), true) && !type.IsDefined(typeof(MessagePackObjectAttribute), true)) ||
type.GetCustomAttribute<MessagePackObjectAttribute>(true)?.KeyAsPropertyName == true ||
MembersOf(type).Any(member => member.GetCustomAttribute<KeyAttribute>(true)?.StringKey != null);
// A type MessagePack serializes by key: [MessagePackObject] or [Union].
private static bool IsKeyed(Type type) =>
type.IsDefined(typeof(MessagePackObjectAttribute), true) || type.IsDefined(typeof(UnionAttribute), true);
// What an object list holds: a keyed class or interface. Never a struct.
private static bool IsElement(Type type) => !type.IsValueType && Classify(type).Kind == Kind.Owner;
private static Type? ElementTypeOf(Type type)
{
if (type.IsArray)
return type.GetElementType();
foreach (var candidate in type.GetInterfaces().Append(type))
{
if (candidate.IsGenericType && candidate.GetGenericTypeDefinition() == typeof(IEnumerable<>))
return candidate.GetGenericArguments()[0];
}
return null;
}
// The key and value types of a map MessagePack writes as a map: IDictionary<,> or IReadOnlyDictionary<,>.
private static (Type Key, Type Value)? MapTypes(Type type)
{
foreach (var candidate in type.GetInterfaces().Append(type))
{
if (candidate.IsGenericType &&
(candidate.GetGenericTypeDefinition() == typeof(IDictionary<,>) || candidate.GetGenericTypeDefinition() == typeof(IReadOnlyDictionary<,>)))
{
var arguments = candidate.GetGenericArguments();
return (arguments[0], arguments[1]);
}
}
return null;
}
private static readonly ConcurrentDictionary<Type, Func<object, IEnumerable<(object? Key, object? Value)>>> MapReaders = new();
// A map's entries read through the generic interface Classify found, so no map shape is recognised twice.
private static Func<object, IEnumerable<(object? Key, object? Value)>> MapEntries(Type map) => MapReaders.GetOrAdd(map, static type =>
{
var (key, value) = MapTypes(type)!.Value;
return (Func<object, IEnumerable<(object? Key, object? Value)>>)Delegate.CreateDelegate(
typeof(Func<object, IEnumerable<(object? Key, object? Value)>>),
typeof(CultElementIds).GetMethod(nameof(ReadMap), BindingFlags.NonPublic | BindingFlags.Static)!.MakeGenericMethod(key, value));
});
private static IEnumerable<(object? Key, object? Value)> ReadMap<TKey, TValue>(object map) =>
((IEnumerable<KeyValuePair<TKey, TValue>>)map).Select(entry => ((object?)entry.Key, (object?)entry.Value));
}
}

34
src/GameCult.Caching/CultSchemaConflictException.cs Normal file
View file

@ -0,0 +1,34 @@
using System;
namespace GameCult.Caching
{
/// <summary>
/// A write the catalog cannot describe: two schemas that share a schema id and disagree on the schema name (both arrived with the
/// records, neither registered), a record whose schema id no chosen entry publishes, or a write that would replace or remove a
/// <see cref="CultForeignRecord"/>. Nothing is written, so the store stays as it was. It names the id, the schema names involved
/// and a record key.
/// </summary>
public sealed class CultSchemaConflictException : InvalidOperationException
{
public CultSchemaConflictException(string message, string schemaId, string[] schemaNames, string recordKey)
: base(message)
{
SchemaId = schemaId;
SchemaNames = schemaNames;
RecordKey = recordKey;
}
/// <summary>The same refusal when the reload it owes (what the store holds now, loaded into the cache) itself failed: that failure is the inner exception.</summary>
public CultSchemaConflictException(string message, string schemaId, string[] schemaNames, string recordKey, Exception? innerException)
: base(message, innerException)
{
SchemaId = schemaId;
SchemaNames = schemaNames;
RecordKey = recordKey;
}
public string SchemaId { get; }
public string[] SchemaNames { get; }
public string RecordKey { get; }
}
}

22
src/GameCult.Caching/CultStoreUnreadableException.cs Normal file
View file

@ -0,0 +1,22 @@
using System;
namespace GameCult.Caching
{
/// <summary>
/// A store file this runtime cannot read: not exactly one complete store (truncated, bytes after it, a value that is not a
/// store, a missing or extra top-level slot), a format or record it does not know, or a body it cannot decode. Open, flush,
/// commit and CultMesh's single-file reads all refuse a file with this one exception, and a refused file is left as it was.
/// The underlying cause, where there is one, is the inner exception, and <see cref="Path"/> names the file.
/// </summary>
public sealed class CultStoreUnreadableException : NotSupportedException
{
public CultStoreUnreadableException(string message, string? path = null, Exception? innerException = null)
: base(message, innerException)
{
Path = path;
}
/// <summary>The store file that was refused, or null when the bytes were read without a file.</summary>
public string? Path { get; }
}
}

18
src/GameCult.Caching/CultVariants.cs
View file

@ -28,6 +28,24 @@ namespace GameCult.Caching
// (document payload, slot and encoded value pairs)
public Func<byte[], IReadOnlyList<KeyValuePair<int, byte[]>>, byte[]> Overlay { get; }
/// <summary>
/// Decodes a document payload read from a store file, as a cache load decodes it: the ids a load mints are minted from the key
/// and their path, so every reader of one store mints the same ids, and a payload a cache write would refuse is refused here too.
/// It reads no file and holds no state.
/// </summary>
/// <typeparam name="TDocument">The registered document type.</typeparam>
/// <param name="key">The record key the payload was stored under.</param>
/// <param name="payload">The record's payload.</param>
/// <exception cref="CultElementIdException">The payload holds an element id or list a cache load refuses.</exception>
public TDocument DecodeDocument<TDocument>(CultRecordKey key, byte[] payload)
where TDocument : class
{
if (payload == null) throw new ArgumentNullException(nameof(payload));
var document = (TDocument)Deserialize(typeof(TDocument), typeof(TDocument), payload)!;
CultElementIds.Plan(typeof(TDocument), document, key.Value, deterministic: true).Apply();
return document;
}
}
public enum CultOverrideOp

27
src/GameCult.Caching/CultWriteConflictException.cs Normal file
View file

@ -0,0 +1,27 @@
using System;
using System.Collections.Generic;
namespace GameCult.Caching
{
/// <summary>
/// A write refused because the store it would land in holds a variant and has changed since its writer last read it. A variant
/// resolves against other records, and a write never decodes a record it did not stage, so it cannot judge records another
/// writer changed: it writes only while every record header in the store equals what the writer last read or wrote. Nothing is
/// written. <see cref="RecordKeys"/> are the keys the write would have written or removed and <see cref="ChangedKeys"/> the keys
/// whose durable header differs from what the writer last read, both ordinal-sorted. The cache has reloaded all of them as the
/// store holds them and forgotten the staged changes at those keys, so a retry is judged against that. When reloading failed,
/// nothing changed and <see cref="Exception.InnerException"/> is the cause.
/// </summary>
public sealed class CultWriteConflictException : InvalidOperationException
{
public CultWriteConflictException(string message, IReadOnlyList<string> recordKeys, IReadOnlyList<string> changedKeys, Exception? inner = null)
: base(message, inner)
{
RecordKeys = recordKeys;
ChangedKeys = changedKeys;
}
public IReadOnlyList<string> RecordKeys { get; }
public IReadOnlyList<string> ChangedKeys { get; }
}
}

9
src/GameCult.Caching/GameCult.Caching.csproj
View file

@ -5,10 +5,13 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="MessagePack.Annotations" Version="3.1.7" />
<PackageReference Include="R3" Version="1.3.0" />
<PackageReference Include="ConcurrentHashSet " Version="1.3.0" />
<InternalsVisibleTo Include="GameCult.Caching.Tests" />
<InternalsVisibleTo Include="GameCult.Caching.MessagePack" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="MessagePack.Annotations" Version="3.1.7" />
<PackageReference Include="R3" Version="1.3.0" />
<PackageReference Include="ConcurrentHashSet " Version="1.3.0" />
</ItemGroup>
</Project>

34
src/GameCult.Caching/README.md
View file

@ -188,6 +188,40 @@ for (var row = 0; row < table.Count; row++)
cache-owned physical table for hot loops and diagnostics; it is not a second
domain model callers have to maintain.
## Element ids
A document type that holds a list of objects gives every object type in that list
(union arms included) a string member marked `[CultElementId]`, next to its
`[Key]`. The id is what lets a variant override one list element instead of the
whole list. Lists of strings, numbers and other non-object values need none.
The rule covers the shapes the id walk owns: arrays, `IEnumerable<T>` classes and
interfaces, dictionary values, and integer-keyed `[MessagePackObject]` and `[Union]`
types. Any other holder is one whole value: `ImmutableArray<T>`, `ArraySegment<T>`,
`Memory<T>`, `KeyValuePair`, tuples, `Lazy<T>`, a class without `[MessagePackObject]`,
a `[DataContract]` or string-keyed type, and a type or member declaring
`[MessagePackFormatter]`. Its elements carry no element ids: a `[CultElementId]`
member inside it is a plain string, never minted or checked, and a variant can only
replace the holder whole.
```csharp
[MessagePackObject]
public sealed class Line
{
[Key(0)] [CultElementId] public string Id = string.Empty;
[Key(1)] public string Text = string.Empty;
}
```
Leave `Id` empty and the cache mints a random id when the record is written. An
id you set yourself must be exactly 12 lowercase hex characters
(`0123456789ab`); anything else is refused with a `CultElementIdException`.
For readable ids, derive them: `[CultElementId(nameof(Name))]` makes an unset id
the text of that member (a string or number on the same type, unique within its
list), and a derived id may be any text. A readable name set by hand on a
non-derived list id is refused; the same member outside a list is unconstrained.
## Persistence discipline
Persistence stays manual by default. Mutations mark the cache and attached

12
src/GameCult.Geometry/CultGeometryDocuments.cs
View file

@ -60,6 +60,10 @@ namespace GameCult.Geometry
[MessagePackObject]
public sealed class CultGeometryDomainNode
{
[Key(7)]
[CultElementId]
public string Id { get; set; } = string.Empty;
[Key(0)]
public string Name { get; set; } = string.Empty;
@ -105,6 +109,10 @@ namespace GameCult.Geometry
[MessagePackObject]
public sealed class CultGeometryFeatureClaim
{
[Key(8)]
[CultElementId]
public string Id { get; set; } = string.Empty;
[Key(0)]
public string Name { get; set; } = string.Empty;
@ -261,6 +269,10 @@ namespace GameCult.Geometry
[MessagePackObject]
public sealed class CultGeometryContributionRow
{
[Key(13)]
[CultElementId]
public string Id { get; set; } = string.Empty;
[Key(0)]
public string DomainKey { get; set; } = string.Empty;

8
src/GameCult.Mesh/CultMesh.cs
View file

@ -2732,11 +2732,9 @@ namespace GameCult.Mesh
// runtime does not recognize by any alias) its payload's own embedded schema stamp,
// alias-matched against TDocument's descriptor. The parse step
// (CultNetDocumentRegistry.TryReadSchemaVersion) is the one decode rule shared with
// DecodeSnapshotDocuments; the match itself stays a direct alias-match against the caller's
// own descriptor rather than a global "which registered type owns this schema string" lookup,
// because that lookup is ambiguous when two document types alias the same schema id (as
// MeshNoteDocument/MeshNoteAliasDocument do in tests/GameCult.Mesh.Tests). No re-filter of the
// server's answer by schema alone, and no fallback that returns a record under the wrong key.
// DecodeSnapshotDocuments; the match itself is a direct alias-match against the caller's own
// descriptor. No re-filter of the server's answer by schema alone, and no fallback that
// returns a record under the wrong key.
private static TDocument ReadDocumentFromSnapshotResponse<TDocument>(
CultNetSnapshotResponseRawMessage response,
string schemaId,

5
src/GameCult.Mesh/CultMeshBodies.cs
View file

@ -1,3 +1,4 @@
using GameCult.Caching;
using System;
using System.IO;
using System.IO.MemoryMappedFiles;
@ -34,6 +35,10 @@ namespace GameCult.Mesh
[MessagePackObject]
public sealed class CultMeshBodyDescriptor
{
[Key(13)]
[CultElementId]
public string Id { get; set; } = string.Empty;
[Key(0)] public string BodyId { get; set; } = string.Empty;
[Key(1)] public string SchemaId { get; set; } = string.Empty;
[Key(2)] public int LayoutVersion { get; set; }

11
src/GameCult.Mesh/CultMeshCdn.cs
View file

@ -87,6 +87,11 @@ namespace GameCult.Mesh
/// <summary>CultCache record key for the referenced chunk document.</summary>
[Key(3)]
public string RecordKey { get; set; } = string.Empty;
/// <summary>Element id: the decimal text of <see cref="Offset"/>, which is unique within a manifest and never random.</summary>
[Key(4)]
[CultElementId(nameof(Offset))]
public string Id { get; set; } = string.Empty;
}
/// <summary>
@ -266,7 +271,8 @@ namespace GameCult.Mesh
ChunkHash = chunkHash,
Offset = offset,
SizeBytes = count,
RecordKey = CultMeshCdnArtifactChunk.CreateRecordKey(chunk).Value
RecordKey = CultMeshCdnArtifactChunk.CreateRecordKey(chunk).Value,
Id = offset.ToString(CultureInfo.InvariantCulture)
});
}
@ -285,7 +291,8 @@ namespace GameCult.Mesh
ChunkHash = chunkHash,
Offset = 0,
SizeBytes = 0,
RecordKey = CultMeshCdnArtifactChunk.CreateRecordKey(chunk).Value
RecordKey = CultMeshCdnArtifactChunk.CreateRecordKey(chunk).Value,
Id = "0"
});
}

4
src/GameCult.Mesh/CultMeshDiscoveryService.cs
View file

@ -417,6 +417,10 @@ namespace GameCult.Mesh
[MessagePackObject]
public sealed class CultMeshDiscoveryCandidateDocument
{
[Key(5)]
[CultElementId]
public string Id { get; set; } = string.Empty;
[Key(0)] public CultMeshVerseDescriptorMessage Descriptor { get; set; } = new CultMeshVerseDescriptorMessage();
[Key(1)] public string SourceId { get; set; } = "";
[Key(2)] public string ObservedAtUtc { get; set; } = "";

12
src/GameCult.Mesh/CultMeshEntityPrefabs.cs
View file

@ -80,6 +80,10 @@ namespace GameCult.Mesh
[MessagePackObject]
public sealed class CultMeshEntityPrefabComponent
{
[Key(2)]
[CultElementId]
public string Id { get; set; } = string.Empty;
/// <summary>Portable component kind or runtime adapter type.</summary>
[Key(0)]
public string Type { get; set; } = string.Empty;
@ -95,6 +99,10 @@ namespace GameCult.Mesh
[MessagePackObject]
public sealed class CultMeshEntityPrefabAssetRef
{
[Key(6)]
[CultElementId]
public string Id { get; set; } = string.Empty;
/// <summary>Stable source asset id inside the prefab package.</summary>
[Key(0)]
public string AssetId { get; set; } = string.Empty;
@ -126,6 +134,10 @@ namespace GameCult.Mesh
[MessagePackObject]
public sealed class CultMeshEntityPrefabNode
{
[Key(10)]
[CultElementId]
public string Id { get; set; } = string.Empty;
/// <summary>Stable node id inside the package.</summary>
[Key(0)]
public string NodeId { get; set; } = string.Empty;

164
src/GameCult.Mesh/CultMeshPrimitives.cs
View file

@ -910,13 +910,10 @@ namespace GameCult.Mesh
/// <summary>Gets whether this handle can submit a client prediction for the underlying document value.</summary>
bool CanSubmitPrediction { get; }
/// <summary>Creates a same-schema alias presentation for another CLR document type.</summary>
CultMeshDocumentHandle<TAlias> AsSchemaAlias<TAlias>() where TAlias : class;
}
/// <summary>
/// Typed reactive document handle with schema-aware alias conversion.
/// Typed reactive document handle.
/// </summary>
public sealed class CultMeshDocumentHandle<TDocument> : ICultMeshDocumentHandle
where TDocument : class
@ -1117,39 +1114,6 @@ namespace GameCult.Mesh
return ObserveAsync().ConfigureAwait(false).GetAwaiter().GetResult();
}
/// <summary>Creates a same-schema alias presentation for another CLR document type.</summary>
public CultMeshDocumentHandle<TAlias> AsSchemaAlias<TAlias>() where TAlias : class
{
var aliasDescriptor = CultDocumentRegistry.Shared.GetRequired<TAlias>();
if (!string.Equals(Descriptor.SchemaName, aliasDescriptor.SchemaName, StringComparison.Ordinal) ||
!string.Equals(Descriptor.SchemaVersion, aliasDescriptor.SchemaVersion, StringComparison.Ordinal))
{
throw new InvalidOperationException(
$"Document type {typeof(TAlias).FullName} uses schema '{aliasDescriptor.SchemaName}' " +
$"version '{aliasDescriptor.SchemaVersion}', but handle '{DocumentId}' exposes " +
$"schema '{SchemaName}' version '{SchemaVersion}'.");
}
var aliasFeed = new CultMeshLiveFeed<CultMeshDocumentQueryParameters, TAlias>(
DocumentId,
async (parameters, context) => ConvertDocument<TDocument, TAlias>(
await Feed.SnapshotAsync(parameters, context).ConfigureAwait(false)),
(parameters, context) => Feed
.Watch(parameters, context)
.Select(ConvertDocument<TDocument, TAlias>),
Sources,
RouteHint);
Func<TAlias, Task>? replace = _replace == null
? null
: value => _replace(ConvertDocument<TAlias, TDocument>(value));
Func<TAlias, Task>? submitPrediction = _submitPrediction == null
? null
: value => _submitPrediction(ConvertDocument<TAlias, TDocument>(value));
return new CultMeshDocumentHandle<TAlias>(aliasFeed.Bind(Context), replace, submitPrediction);
}
internal static TDocumentValue CloneDocument<TDocumentValue>(TDocumentValue document)
where TDocumentValue : class
{
@ -1216,19 +1180,6 @@ namespace GameCult.Mesh
"field");
}
internal static TTarget ConvertDocument<TSource, TTarget>(TSource document)
where TSource : class
where TTarget : class
{
if (document == null) throw new ArgumentNullException(nameof(document));
if (document is TTarget alreadyTyped)
{
return alreadyTyped;
}
var payload = CultDocumentMessagePackSerialization.SerializeUntyped(document, typeof(TSource));
return (TTarget)CultDocumentMessagePackSerialization.DeserializeUntyped(typeof(TTarget), payload);
}
}
/// <summary>
@ -1912,11 +1863,12 @@ namespace GameCult.Mesh
$"CultMesh document catalog does not expose a document for schema '{schema}'.");
}
/// <summary>Looks up a typed document handle by CLR type or same-schema alias.</summary>
/// <summary>Looks up a typed document handle by CLR type.</summary>
public bool TryGetDocument<TDocument>(
out CultMeshDocumentHandle<TDocument> document)
where TDocument : class
{
// One type owns a schema, so the handle for TDocument's schema is the handle for TDocument.
if (_documentsByType.TryGetValue(typeof(TDocument), out var untypedDocument) &&
untypedDocument is CultMeshDocumentHandle<TDocument> typedDocument)
{
@ -1924,25 +1876,11 @@ namespace GameCult.Mesh
return true;
}
var descriptor = CultDocumentRegistry.Shared.GetRequired<TDocument>();
if (_documentsBySchema.TryGetValue(descriptor.SchemaVersion, out var schemaDocument) ||
_documentsBySchema.TryGetValue(descriptor.SchemaName, out schemaDocument!))
{
if (schemaDocument is CultMeshDocumentHandle<TDocument> schemaTypedDocument)
{
document = schemaTypedDocument;
return true;
}
document = schemaDocument.AsSchemaAlias<TDocument>();
return true;
}
document = null!;
return false;
}
/// <summary>Looks up a typed document handle by CLR type or same-schema alias.</summary>
/// <summary>Looks up a typed document handle by CLR type.</summary>
public CultMeshDocumentHandle<TDocument> Document<TDocument>()
where TDocument : class
{
@ -1953,70 +1891,70 @@ namespace GameCult.Mesh
$"CultMesh document catalog does not expose a document for {typeof(TDocument).FullName}.");
}
/// <summary>Reads one typed document snapshot by CLR type or same-schema alias.</summary>
/// <summary>Reads one typed document snapshot by CLR type.</summary>
public Task<TDocument> LatestAsync<TDocument>()
where TDocument : class
{
return Document<TDocument>().LatestAsync();
}
/// <summary>Reads one typed document synchronously by CLR type or same-schema alias.</summary>
/// <summary>Reads one typed document synchronously by CLR type.</summary>
public TDocument Latest<TDocument>()
where TDocument : class
{
return Document<TDocument>().Latest();
}
/// <summary>Gets whether one typed document can be replaced by CLR type or same-schema alias.</summary>
/// <summary>Gets whether one typed document can be replaced by CLR type.</summary>
public bool CanReplace<TDocument>()
where TDocument : class
{
return TryGetDocument<TDocument>(out var document) && document.CanReplace;
}
/// <summary>Replaces one typed document by CLR type or same-schema alias.</summary>
/// <summary>Replaces one typed document by CLR type.</summary>
public Task ReplaceAsync<TDocument>(TDocument value)
where TDocument : class
{
return Document<TDocument>().ReplaceAsync(value);
}
/// <summary>Gets whether one typed document can submit client predictions by CLR type or same-schema alias.</summary>
/// <summary>Gets whether one typed document can submit client predictions by CLR type.</summary>
public bool CanSubmitPrediction<TDocument>()
where TDocument : class
{
return TryGetDocument<TDocument>(out var document) && document.CanSubmitPrediction;
}
/// <summary>Submits a client prediction for one typed document by CLR type or same-schema alias.</summary>
/// <summary>Submits a client prediction for one typed document by CLR type.</summary>
public Task SubmitPredictionAsync<TDocument>(TDocument value)
where TDocument : class
{
return Document<TDocument>().SubmitPredictionAsync(value);
}
/// <summary>Gets an explicit authoritative writer by CLR type or same-schema alias.</summary>
/// <summary>Gets an explicit authoritative writer by CLR type.</summary>
public CultMeshDocumentWriter<TDocument> AuthoritativeWriter<TDocument>()
where TDocument : class
{
return Document<TDocument>().AuthoritativeWriter();
}
/// <summary>Gets an explicit prediction writer by CLR type or same-schema alias.</summary>
/// <summary>Gets an explicit prediction writer by CLR type.</summary>
public CultMeshDocumentWriter<TDocument> PredictionWriter<TDocument>()
where TDocument : class
{
return Document<TDocument>().PredictionWriter();
}
/// <summary>Watches one typed document by CLR type or same-schema alias.</summary>
/// <summary>Watches one typed document by CLR type.</summary>
public Observable<TDocument> Watch<TDocument>()
where TDocument : class
{
return Document<TDocument>().Watch();
}
/// <summary>Subscribes to one typed document by CLR type or same-schema alias.</summary>
/// <summary>Subscribes to one typed document by CLR type.</summary>
public IDisposable Watch<TDocument>(Action<TDocument> onNext)
where TDocument : class
{
@ -2050,9 +1988,6 @@ namespace GameCult.Mesh
/// <summary>Gets typed state sources this collection handle depends on, when known.</summary>
IReadOnlyList<CultMeshProjectionSource> Sources { get; }
/// <summary>Creates a same-schema alias presentation for another CLR document type.</summary>
CultMeshCollectionHandle<TAlias> AsSchemaAlias<TAlias>() where TAlias : class;
}
/// <summary>
@ -2118,11 +2053,12 @@ namespace GameCult.Mesh
$"CultMesh collection catalog does not expose a collection for schema '{schema}'.");
}
/// <summary>Looks up a typed collection handle by CLR type or same-schema alias.</summary>
/// <summary>Looks up a typed collection handle by CLR type.</summary>
public bool TryGetCollection<TDocument>(
out CultMeshCollectionHandle<TDocument> collection)
where TDocument : class
{
// One type owns a schema, so the handle for TDocument's schema is the handle for TDocument.
if (_collectionsByType.TryGetValue(typeof(TDocument), out var untypedCollection) &&
untypedCollection is CultMeshCollectionHandle<TDocument> typedCollection)
{
@ -2130,26 +2066,11 @@ namespace GameCult.Mesh
return true;
}
var descriptor = CultDocumentRegistry.Shared.GetRequired<TDocument>();
if (_collectionsBySchema.TryGetValue(descriptor.SchemaId, out var schemaCollection) ||
_collectionsBySchema.TryGetValue(descriptor.SchemaVersion, out schemaCollection!) ||
_collectionsBySchema.TryGetValue(descriptor.SchemaName, out schemaCollection!))
{
if (schemaCollection is CultMeshCollectionHandle<TDocument> schemaTypedCollection)
{
collection = schemaTypedCollection;
return true;
}
collection = schemaCollection.AsSchemaAlias<TDocument>();
return true;
}
collection = null!;
return false;
}
/// <summary>Looks up a typed collection handle by CLR type or same-schema alias.</summary>
/// <summary>Looks up a typed collection handle by CLR type.</summary>
public CultMeshCollectionHandle<TDocument> Collection<TDocument>()
where TDocument : class
{
@ -2160,21 +2081,21 @@ namespace GameCult.Mesh
$"CultMesh collection catalog does not expose a collection for {typeof(TDocument).FullName}.");
}
/// <summary>Reads one typed collection snapshot by CLR type or same-schema alias.</summary>
/// <summary>Reads one typed collection snapshot by CLR type.</summary>
public Task<IReadOnlyList<TDocument>> LatestAsync<TDocument>()
where TDocument : class
{
return Collection<TDocument>().LatestAsync();
}
/// <summary>Watches one typed collection by CLR type or same-schema alias.</summary>
/// <summary>Watches one typed collection by CLR type.</summary>
public Observable<CultMeshCollectionChange<TDocument>> WatchChanges<TDocument>()
where TDocument : class
{
return Collection<TDocument>().WatchChanges();
}
/// <summary>Subscribes to one typed collection by CLR type or same-schema alias.</summary>
/// <summary>Subscribes to one typed collection by CLR type.</summary>
public IDisposable WatchChanges<TDocument>(Action<CultMeshCollectionChange<TDocument>> onNext)
where TDocument : class
{
@ -2246,7 +2167,7 @@ namespace GameCult.Mesh
}
/// <summary>
/// Typed reactive collection handle with schema-aware alias conversion.
/// Typed reactive collection handle.
/// </summary>
public sealed class CultMeshCollectionHandle<TDocument> : ICultMeshCollectionHandle
where TDocument : class
@ -2313,49 +2234,6 @@ namespace GameCult.Mesh
if (onNext == null) throw new ArgumentNullException(nameof(onNext));
return WatchChanges().Subscribe(onNext);
}
/// <summary>Creates a same-schema alias presentation for another CLR document type.</summary>
public CultMeshCollectionHandle<TAlias> AsSchemaAlias<TAlias>() where TAlias : class
{
var aliasDescriptor = CultDocumentRegistry.Shared.GetRequired<TAlias>();
if (!string.Equals(Descriptor.SchemaName, aliasDescriptor.SchemaName, StringComparison.Ordinal) ||
!string.Equals(Descriptor.SchemaVersion, aliasDescriptor.SchemaVersion, StringComparison.Ordinal))
{
throw new InvalidOperationException(
$"Document type {typeof(TAlias).FullName} uses schema '{aliasDescriptor.SchemaName}' " +
$"version '{aliasDescriptor.SchemaVersion}', but collection '{CollectionId}' exposes " +
$"schema '{SchemaName}' version '{SchemaVersion}'.");
}
return new CultMeshCollectionHandle<TAlias>(
CollectionId,
async () => (await LatestAsync().ConfigureAwait(false))
.Select(ConvertDocument<TDocument, TAlias>)
.ToArray(),
() => WatchChanges().Select(change => new CultMeshCollectionChange<TAlias>(
change.Kind,
change.Key,
change.SchemaId,
change.Document == null ? null : ConvertDocument<TDocument, TAlias>(change.Document),
change.PreviousDocument == null ? null : ConvertDocument<TDocument, TAlias>(change.PreviousDocument),
change.Message)),
Sources,
RouteHint);
}
private static TTarget ConvertDocument<TSource, TTarget>(TSource document)
where TSource : class
where TTarget : class
{
if (document == null) throw new ArgumentNullException(nameof(document));
if (document is TTarget alreadyTyped)
{
return alreadyTyped;
}
var payload = CultDocumentMessagePackSerialization.SerializeUntyped(document, typeof(TSource));
return (TTarget)CultDocumentMessagePackSerialization.DeserializeUntyped(typeof(TTarget), payload);
}
}
/// <summary>

182
src/GameCult.Mesh/CultMeshSingleFileDocuments.cs
View file

@ -1,11 +1,9 @@
using System;
using System.Buffers;
using System.IO;
using System.Linq;
using System.Security.Cryptography;
using GameCult.Caching;
using GameCult.Caching.MessagePack;
using MessagePack;
namespace GameCult.Mesh
{
@ -96,14 +94,9 @@ namespace GameCult.Mesh
{
if (document == null) throw new ArgumentNullException(nameof(document));
var resolvedRegistry = registry ?? CultDocumentRegistry.Shared;
var descriptor = resolvedRegistry.GetRequired<TDocument>();
WriteSingleFileDocumentPayload(
path,
key,
descriptor.ToCatalogEntry(),
storedAt,
CultDocumentMessagePackSerialization.SerializeUntyped(document, typeof(TDocument)));
if (string.IsNullOrWhiteSpace(path)) throw new ArgumentException("Value must be non-empty.", nameof(path));
// The store decides element ids, the header and the copy of every other record, as a cache write does.
StoreAt(path).WriteDocument(key, document, storedAt, registry);
}
/// <summary>
@ -118,7 +111,8 @@ namespace GameCult.Mesh
var resolvedRegistry = registry ?? CultDocumentRegistry.Shared;
var descriptor = resolvedRegistry.GetRequired<TDocument>();
var payload = ReadSingleFileDocumentPayload(path, key, descriptor.SchemaId);
return (TDocument)CultDocumentMessagePackSerialization.DeserializeUntyped(typeof(TDocument), payload);
// A load refuses what a write refuses, and mints the ids a cache load of the same file would mint.
return CultCacheMessagePack.CreateCodec(resolvedRegistry).DecodeDocument<TDocument>(key, payload);
}
/// <summary>
@ -132,7 +126,7 @@ namespace GameCult.Mesh
byte[] payload)
{
if (schema == null) throw new ArgumentNullException(nameof(schema));
WriteSingleFileDocumentPayload(path, key, schema.ToCatalogEntry(payload), storedAt, payload);
WriteSingleFileDocumentPayload(path, key, schema.ToCatalogEntry(payload), storedAt, payload, holdsIds: null, registry: null);
}
/// <summary>
@ -146,7 +140,7 @@ namespace GameCult.Mesh
if (string.IsNullOrWhiteSpace(path)) throw new ArgumentException("Value must be non-empty.", nameof(path));
if (string.IsNullOrWhiteSpace(expectedSchemaId)) throw new ArgumentException("Value must be non-empty.", nameof(expectedSchemaId));
var snapshot = ReadSingleFileSnapshot(path);
var snapshot = StoreAt(path).ReadStoreSnapshot() ?? throw new FileNotFoundException($"CultCache document '{path}' does not exist.", path);
var record = snapshot.Records.SingleOrDefault(candidate => string.Equals(candidate.Key, key.Value, StringComparison.Ordinal))
?? throw new InvalidDataException($"CultCache document '{path}' does not contain record '{key.Value}'.");
@ -177,7 +171,7 @@ namespace GameCult.Mesh
if (string.IsNullOrWhiteSpace(path)) throw new ArgumentException("Value must be non-empty.", nameof(path));
if (string.IsNullOrWhiteSpace(expectedSchemaId)) throw new ArgumentException("Value must be non-empty.", nameof(expectedSchemaId));
var snapshot = ReadSingleFileSnapshot(path);
var snapshot = StoreAt(path).ReadStoreSnapshot() ?? throw new FileNotFoundException($"CultCache document '{path}' does not exist.", path);
if (snapshot.Records.Length != 1)
{
throw new InvalidDataException(
@ -214,7 +208,9 @@ namespace GameCult.Mesh
CultRecordKey key,
CultSchemaCatalogEntry catalogEntry,
string? storedAt,
byte[] payload)
byte[] payload,
bool? holdsIds,
CultDocumentRegistry? registry)
{
if (string.IsNullOrWhiteSpace(path)) throw new ArgumentException("Value must be non-empty.", nameof(path));
if (catalogEntry == null) throw new ArgumentNullException(nameof(catalogEntry));
@ -222,140 +218,25 @@ namespace GameCult.Mesh
throw new ArgumentException("Catalog entry must include a schema id.", nameof(catalogEntry));
payload ??= Array.Empty<byte>();
var snapshot = new CultPersistedStoreSnapshot
{
SchemaCatalog = new[] { catalogEntry },
Records = new[]
// The write replaces this one key and copies every other record and catalog entry forward as the store holds them, under
// the store's lock. A typed write sees its document, so its ids decide the header; a raw payload is opaque (holdsIds null),
// so a file already marked stays marked. A typed write attaches the registry its payload was encoded with, so the write
// decides for copied records that registry can read; a raw payload attaches none and decides nothing. A store this runtime
// cannot read refuses the write instead of being overwritten.
StoreAt(path).WriteRecord(
new CultPersistedRecord
{
new CultPersistedRecord
{
Key = key.Value,
SchemaId = catalogEntry.SchemaId,
StoredAt = string.IsNullOrWhiteSpace(storedAt) ? DateTimeOffset.UtcNow.ToString("O") : storedAt!,
Payload = payload
}
}
};
var directory = Path.GetDirectoryName(path);
if (!string.IsNullOrWhiteSpace(directory))
Directory.CreateDirectory(directory);
// The write replaces the file, so it first reads what it would replace, under the lock a CultCache single-file
// store takes (<path>.lock): a store this runtime cannot read (a newer format, or damage) or one holding any
// record but this key is refused, never overwritten.
using (AcquireStoreLock(path))
{
RefuseToReplaceAStoreHoldingMore(path, key);
WriteFileAtomically(path, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot));
}
Key = key.Value,
SchemaId = catalogEntry.SchemaId,
StoredAt = string.IsNullOrWhiteSpace(storedAt) ? DateTimeOffset.UtcNow.ToString("O") : storedAt!,
Payload = payload
},
catalogEntry,
holdsIds,
registry);
}
private static void RefuseToReplaceAStoreHoldingMore(string path, CultRecordKey key)
{
if (!File.Exists(path) || new FileInfo(path).Length == 0)
return;
var existing = ReadSingleFileSnapshot(path);
var other = existing.Records.FirstOrDefault(record =>
record.Variant != null || !string.Equals(record.Key, key.Value, StringComparison.Ordinal));
if (other != null)
throw new InvalidOperationException(
$"CultCache document '{path}' holds record '{other.Key}' besides '{key.Value}'; a CultMesh single-file write " +
"replaces the whole file and would drop it. Write through a CultCache store instead.");
}
private static FileStream AcquireStoreLock(string path)
{
var lockPath = path + ".lock";
var started = System.Diagnostics.Stopwatch.StartNew();
while (true)
{
try
{
return new FileStream(lockPath, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None, bufferSize: 1);
}
catch (IOException) when (started.Elapsed < TimeSpan.FromSeconds(30))
{
System.Threading.Thread.Sleep(10);
}
}
}
private static CultPersistedStoreSnapshot ReadSingleFileSnapshot(string path)
{
var bytes = File.ReadAllBytes(path);
CultPersistedStoreSnapshot snapshot;
try
{
snapshot = CultDocumentMessagePackSerialization.DeserializeSnapshot(bytes, CultDocumentMessagePackSerialization.RequireReadableSingleFileStamp);
}
catch (Exception ex) when (ex is MessagePackSerializationException or InvalidOperationException)
{
snapshot = ReadLegacySingleFileSnapshot(bytes);
}
CultDocumentMessagePackSerialization.RequireSingleFileFormat(snapshot);
return snapshot;
}
private static CultPersistedStoreSnapshot ReadLegacySingleFileSnapshot(byte[] bytes)
{
var reader = new MessagePackReader(bytes);
var fieldCount = reader.ReadArrayHeader();
var snapshot = new CultPersistedStoreSnapshot();
if (fieldCount > 0)
snapshot.FormatVersion = reader.ReadString()
?? throw new NotSupportedException("Store snapshot declares no format version.");
if (fieldCount > 1)
{
var catalogCount = reader.ReadArrayHeader();
snapshot.SchemaCatalog = new CultSchemaCatalogEntry[catalogCount];
for (var index = 0; index < catalogCount; index++)
snapshot.SchemaCatalog[index] = ReadLegacySchemaCatalogEntry(ref reader);
}
if (fieldCount > 2)
{
var recordCount = reader.ReadArrayHeader();
snapshot.Records = new CultPersistedRecord[recordCount];
for (var index = 0; index < recordCount; index++)
snapshot.Records[index] = CultDocumentMessagePackSerialization.ReadPersistedRecord(ref reader);
}
for (var index = 3; index < fieldCount; index++)
reader.Skip();
return snapshot;
}
private static CultSchemaCatalogEntry ReadLegacySchemaCatalogEntry(ref MessagePackReader reader)
{
var fieldCount = reader.ReadArrayHeader();
var entry = new CultSchemaCatalogEntry();
if (fieldCount > 0) entry.SchemaId = reader.ReadString() ?? string.Empty;
if (fieldCount > 1) entry.SchemaName = reader.ReadString() ?? string.Empty;
if (fieldCount > 2) entry.SchemaVersion = reader.ReadString() ?? string.Empty;
if (fieldCount > 3) entry.CanonicalSchemaJson = reader.ReadString() ?? string.Empty;
if (fieldCount > 4) reader.Skip();
if (fieldCount > 5) entry.ContentHash = reader.ReadString() ?? string.Empty;
if (fieldCount > 6)
{
var memberCount = reader.ReadArrayHeader();
for (var index = 0; index < memberCount; index++)
reader.Skip();
}
for (var index = 7; index < fieldCount; index++)
reader.Skip();
entry.CompatibleSchemaIds = string.IsNullOrWhiteSpace(entry.SchemaId)
? Array.Empty<string>()
: new[] { entry.SchemaId };
entry.Members = Array.Empty<CultSchemaMemberCatalogEntry>();
return entry;
}
private static SingleFileMessagePackBackingStore StoreAt(string path) => new SingleFileMessagePackBackingStore(path);
private static bool PublishesSchema(CultSchemaCatalogEntry[] catalog, string schemaId)
{
@ -363,14 +244,5 @@ namespace GameCult.Mesh
string.Equals(entry.SchemaId, schemaId, StringComparison.Ordinal) ||
entry.CompatibleSchemaIds.Any(candidate => string.Equals(candidate, schemaId, StringComparison.Ordinal)));
}
private static void WriteFileAtomically(string path, byte[] bytes)
{
var tempPath = path + ".tmp";
File.WriteAllBytes(tempPath, bytes);
if (File.Exists(path))
File.Delete(path);
File.Move(tempPath, path);
}
}
}

4
src/GameCult.Mesh/CultMeshSnapshots.cs
View file

@ -979,11 +979,11 @@ namespace GameCult.Mesh
var canDeserializeWithBinding =
binding != null &&
typeof(TDocument).IsAssignableFrom(binding.DocumentType);
var canDeserializeAsSchemaAlias =
var canDeserializeByWireSchema =
CultNetSchemaAliasMatching.Matches(record.SchemaId, descriptor) ||
(binding != null && CultNetSchemaAliasMatching.Matches(binding.SchemaId, descriptor)) ||
CultNetDocumentRegistry.PayloadMatchesSchema(record.Payload, descriptor);
if (!canDeserializeWithBinding && !canDeserializeAsSchemaAlias)
if (!canDeserializeWithBinding && !canDeserializeByWireSchema)
continue;
if (!string.Equals(record.PayloadEncoding, "messagepack", StringComparison.Ordinal))

62
src/GameCult.Mesh/docs/public-api.md
View file

@ -173,28 +173,22 @@ portable machinery that makes it feel native.
the normal shard log. Callers can bind a projected live feed, a local
`CultCache` record, a distributed `CultNetDatabase` record, or a
`CultMeshNode` record through the same surface instead of walking transport
and projection layers themselves. Same-schema CLR aliases can be requested
directly from node/database/snapshot surfaces; CultMesh resolves the canonical
registered document by shared `[CultDocument(schemaName, schemaVersion)]`
identity and uses the shared CultCache MessagePack codec for
read/watch/replace/prediction conversion. `AsSchemaAlias<TAlias>()` remains
available when a caller already has a handle and wants to project it.
and projection layers themselves. One CLR type owns each
`[CultDocument(schemaName, schemaVersion)]` schema; the registry refuses a
second type for the same schema with `CultSchemaConflictException`.
- `CultMeshDocumentCatalog` is the schema-aware lookup edge for a set of
document handles. It indexes handles by CLR document type, schema name, and
schema version, and resolves same-schema CLR aliases by delegating to
`CultMeshDocumentHandle<TDocument>.AsSchemaAlias<TAlias>()`. Domain facades
can expose `Document<T>()`, `DocumentBySchema(...)`, `LatestAsync<T>()`, and
`Watch<T>()` without owning schema dictionaries or alias serializers. Mutable
facades can also call `ReplaceAsync<T>(...)` or
`SubmitPredictionAsync<T>(...)` directly on the catalog when they only need
the schema-aliased operation and not the intermediate handle.
document handles. It indexes handles by CLR document type, schema id, schema
name, and schema version. Domain facades can expose `Document<T>()`,
`DocumentBySchema(...)`, `LatestAsync<T>()`, and `Watch<T>()` without owning
schema dictionaries. Mutable facades can also call `ReplaceAsync<T>(...)` or
`SubmitPredictionAsync<T>(...)` directly on the catalog when they do not need
the intermediate handle.
- `CultMeshCollectionHandle<TDocument>` is the typed reactive edge for
multi-record state. It exposes one `LatestAsync()` collection snapshot and a
`WatchChanges()` stream while hiding whether the backing source is an
in-process `CultCache`, a distributed `CultNetDatabase`, or a `CultMeshNode`.
Collection handles can represent all records of a document type, a named
document view, or an indexed value view, and they support the same
same-schema CLR alias conversion as document handles.
document view, or an indexed value view.
- `CultMeshCollectionCatalog` is the collection sibling of
`CultMeshDocumentCatalog`. It indexes typed collection handles by CLR
document type, schema id, schema name, and schema version, then exposes
@ -258,19 +252,17 @@ var objects = await visibleObjects.ExecuteAsync(viewportRequest);
await movePilot.InvokeAsync(moveRequest, idempotencyKey);
```
A single typed document can be surfaced the same way. The caller asks for the
typed view it wants; if that type is a same-schema alias of the registered
canonical document, CultMesh still opens the correct backing record:
A single typed document can be surfaced the same way:
```csharp
var stockForUi = CultMesh.Document<StationStockUiDocument>(
var stockHandle = CultMesh.Document<StationStockDocument>(
node,
new CultRecordKey("station:starbridge:stock"),
verse);
var stock = await stockForUi.LatestAsync();
using var watch = stockForUi.Watch(RenderStock);
await stockForUi.SetAsync(updatedStockFromUi);
var stock = await stockHandle.LatestAsync();
using var watch = stockHandle.Watch(RenderStock);
await stockHandle.SetAsync(updatedStock);
```
Projected or derived documents can use the same handle shape with explicit
@ -291,8 +283,6 @@ var cockpit = CultMesh.Document(
var current = await cockpit.LatestAsync();
using var subscription = cockpit.Watch(next => RenderCockpit(next));
var daemonAlias = cockpit.AsSchemaAlias<DaemonCockpitState>();
```
Remote CultNet snapshots use the same document handle surface. The caller names
@ -301,20 +291,18 @@ id, response filtering, raw MessagePack payload decode, route metadata, and
polling watch fallback:
```csharp
var remoteHealth = CultMesh.DocumentFromPeerSnapshot<DaemonHealthUiDocument>(
var remoteHealth = CultMesh.DocumentFromPeerSnapshot<DaemonHealthDocument>(
"cultnet://daemon.local:3075",
"daemon:aetheria.health.v1",
verse);
var uiHealth = await remoteHealth.LatestAsync();
var health = await remoteHealth.LatestAsync();
```
Snapshot requests are record-key-first so same logical documents can still be
read when another runtime generated a different schema id for a compatible
schema alias. The decode path prefers exact schema id matches when present and
falls back to the requested record key. Ask for the runtime-facing alias type
directly; `AsSchemaAlias<TAlias>()` is for adapting handles that were already
constructed elsewhere.
Snapshot requests are record-key-first so the same logical document can still be
read when another runtime generated a different schema id for it. The decode
path prefers exact schema id matches when present and falls back to the
requested record key.
A domain facade can still collect existing handles into one schema-aware
catalog when it is composing pre-built surfaces:
@ -326,7 +314,7 @@ var documents = CultMesh.Documents(
zoneContacts);
var refit = await documents.LatestAsync<StationRefitDocument>();
var uiDocking = documents.Document<CurrentDockingUiDocument>();
var docking = documents.Document<CurrentDockingDocument>();
var bySchema = documents.DocumentBySchema("gamecult.aetheria.station_refit.v1");
```
@ -341,11 +329,9 @@ var allies = CultMesh.CollectionByIndex<PlayerShipDocument>(
var currentAllies = await allies.LatestAsync();
using var changes = allies.WatchChanges(change => UpdateRoster(change));
var uiAllies = allies.AsSchemaAlias<PlayerShipUiDocument>();
var collections = CultMesh.Collections(allies);
var uiRoster = await collections.LatestAsync<PlayerShipUiDocument>();
using var uiRosterChanges = collections.WatchChanges<PlayerShipUiDocument>(RenderRoster);
var roster = await collections.LatestAsync<PlayerShipDocument>();
using var rosterChanges = collections.WatchChanges<PlayerShipDocument>(RenderRoster);
```
```ts

23
src/GameCult.Networking/CultNetDocumentRegistry.cs
View file

@ -165,10 +165,31 @@ namespace GameCult.Networking
}
/// <summary>
/// Registers a document binding.
/// Registers a document binding. One binding owns a document type and a schema id: a binding claiming a type or a schema
/// id another binding already holds is refused with <see cref="CultSchemaConflictException"/>, naming the new binding's
/// schema id and both document types, and nothing is bound. Binding a type to the schema id it already holds replaces
/// that binding.
/// </summary>
public CultNetDocumentRegistry Register(CultNetDocumentBinding binding)
{
var holder =
_bindingsByType.TryGetValue(binding.DocumentType, out var sameType) &&
!string.Equals(sameType.SchemaId, binding.SchemaId, StringComparison.Ordinal)
? sameType
: _bindingsBySchemaId.TryGetValue(binding.SchemaId, out var sameSchema) &&
sameSchema.DocumentType != binding.DocumentType
? sameSchema
: null;
if (holder != null)
{
throw new CultSchemaConflictException(
$"CultNet schema id \"{holder.SchemaId}\" is already bound to type \"{holder.DocumentType.FullName}\"; " +
$"type \"{binding.DocumentType.FullName}\" (schema id \"{binding.SchemaId}\") cannot also be bound.",
binding.SchemaId,
new[] { holder.DocumentType.FullName ?? holder.DocumentType.Name, binding.DocumentType.FullName ?? binding.DocumentType.Name },
string.Empty);
}
_bindingsBySchemaId[binding.SchemaId] = binding;
_bindingsByType[binding.DocumentType] = binding;
return this;

12
src/GameCult.Networking/CultWitnessArtifactBundle.cs
View file

@ -104,6 +104,10 @@ namespace GameCult.Networking
[MessagePackObject]
public sealed class CultWitnessContractPin
{
[Key(4)]
[CultElementId]
public string Id { get; set; } = string.Empty;
/// <summary>
/// Gets or sets the contract role inside the bundle.
/// </summary>
@ -135,6 +139,10 @@ namespace GameCult.Networking
[MessagePackObject]
public sealed class CultWitnessArtifactEntry
{
[Key(6)]
[CultElementId]
public string Id { get; set; } = string.Empty;
/// <summary>
/// Gets or sets the artifact role inside the witness bundle.
/// </summary>
@ -178,6 +186,10 @@ namespace GameCult.Networking
[MessagePackObject]
public sealed class CultWitnessTimingEntry
{
[Key(5)]
[CultElementId]
public string Id { get; set; } = string.Empty;
/// <summary>
/// Gets or sets the pipeline stage name.
/// </summary>

59
tests/GameCult.Caching.ConsumerBuild/Program.cs
View file

@ -1,10 +1,69 @@
using System.Collections.Generic;
using System.Linq;
using GameCult.Caching;
using MessagePack;
// RefKeyedDictionaryCompilesInAConsumer: this project building is the test.
System.Console.WriteLine(new ConsumerFaction().Standing.Count);
// An outside store kind versions its foreign records: this project has no InternalsVisibleTo, so building and running it pins that
// Foreign(record, catalog, version) is all a store kind outside the library needs for ExpectUnchanged to hold over them.
using var outside = new CultCache(CultDocumentRegistry.ForTypes(new[] { typeof(ConsumerFaction) }));
outside.AddBackingStore(new OutsideStore(), typeof(ConsumerFaction));
var committed = outside.Commit(batch =>
{
batch.ExpectUnchanged();
batch.Upsert(new ConsumerFaction { Name = "a" }, new CultRecordHandle<ConsumerFaction>(new CultRecordKey("a")));
});
if (!committed)
return 1;
// A store file is written and read without a cache through the public members: this project has no InternalsVisibleTo, so
// building it pins that ReadStoreSnapshot, WriteRecord and WriteDocument, and CultCodec.DecodeDocument, are public API.
var storePath = System.IO.Path.Combine(System.IO.Path.GetTempPath(), "consumer-build-" + System.Guid.NewGuid().ToString("N") + ".cc");
try
{
var entry = new CultSchemaCatalogEntry { SchemaId = "outside.written", SchemaName = "outside.written" };
var store = new GameCult.Caching.MessagePack.SingleFileMessagePackBackingStore(storePath);
store.WriteRecord(new CultPersistedRecord { Key = "r", SchemaId = entry.SchemaId, StoredAt = "t", Payload = new byte[] { 0x90 } }, entry);
var snapshot = store.ReadStoreSnapshot();
if (snapshot == null || snapshot.Records.Length != 1 || snapshot.Records[0].Key != "r")
return 1;
var typedRegistry = CultDocumentRegistry.ForTypes(new[] { typeof(ConsumerFaction) });
store.WriteDocument(new CultRecordKey("f"), new ConsumerFaction { Name = "typed" }, registry: typedRegistry);
var typed = store.ReadStoreSnapshot()!.Records.Single(record => record.Key == "f");
if (GameCult.Caching.MessagePack.CultCacheMessagePack.CreateCodec(typedRegistry).DecodeDocument<ConsumerFaction>(new CultRecordKey("f"), typed.Payload).Name != "typed")
return 1;
}
finally
{
System.IO.File.Delete(storePath);
}
return 0;
// Lists one record under an id no type claims, at version "v1", and commits only when the conditions hold over it.
public sealed class OutsideStore : CacheBackingStore
{
private static readonly CultSchemaCatalogEntry Entry = new() { SchemaId = "outside.unclaimed", SchemaName = "outside.unclaimed" };
public override void PullAll()
{
var record = new CultPersistedRecord { Key = "w", SchemaId = Entry.SchemaId, StoredAt = "t" };
SetForeignRecords(new[] { Foreign(record, new[] { Entry }, "v1")! });
}
public override void Push(CultStoredDocument entry) { }
public override void Delete(CultStoredDocument entry) { }
public override void PushAll() { }
public override CultCommitOutcome CommitBatch(CultCommitRequest request, bool wait) =>
request.ConditionsHold(new Dictionary<string, string> { ["w"] = "v1" }, Entries.Values, ForeignRecords)
? CultCommitOutcome.Committed
: CultCommitOutcome.Mismatch;
}
[CultDocument("tests.consumer_faction", "tests.consumer_faction.v1")]
[MessagePackObject]
public sealed class ConsumerFaction

139
tests/GameCult.Caching.InteropPeer/Program.cs
View file

@ -1,6 +1,7 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Text.Json;
using System.Threading.Tasks;
@ -16,7 +17,7 @@ static async Task<int> ProgramMainAsync(string[] args)
{
if (args.Length == 0)
{
throw new InvalidOperationException("Expected mode: write | read | hold-lock | write-routed <catalog.cc> <run.cc>");
throw new InvalidOperationException("Expected mode: write | read | rewrite | hold-lock | write-routed <catalog.cc> <run.cc>");
}
var mode = args[0];
@ -38,6 +39,18 @@ static async Task<int> ProgramMainAsync(string[] args)
case "read":
await ReadAsync(file);
return 0;
case "rewrite":
await RewriteAsync(file);
return 0;
case "write-deck":
await WriteDeckAsync(file);
return 0;
case "read-deck":
await ReadDeckAsync(file);
return 0;
case "edit-in-place":
await EditInPlaceAsync(file, RequireArg(options, "kind"));
return 0;
case "hold-lock":
HoldLock(file, int.Parse(RequireArg(options, "ms"), CultureInfo.InvariantCulture));
return 0;
@ -94,6 +107,113 @@ static async Task ReadAsync(string file)
WriteJsonLine(note);
}
// Puts the stored note back: the cache stamps it with the registered schema id, whatever id it was stored under.
static async Task RewriteAsync(string file)
{
var cache = BuildCache(file);
await cache.PullAllBackingStoresAsync();
var note = cache.AllEntries.OfType<CultCacheInteropNote>().Single();
await cache.UpsertAsync(note, new CultRecordHandle<CultCacheInteropNote>(new CultRecordKey(note.DocumentId)));
cache.FlushAllBackingStores();
WriteJsonLine(note);
}
// A record with an object list: the cache mints each mark's element id, and every runtime reads it as an ordinary member.
static async Task WriteDeckAsync(string file)
{
var cache = BuildCache(file);
await cache.PullAllBackingStoresAsync();
var deck = new CultCacheInteropDeck
{
DocumentId = "deck:csharp",
Marks = [new CultCacheInteropMark { Label = "twin" }, new CultCacheInteropMark { Label = "twin" }, new CultCacheInteropMark { Label = "other" }]
};
await cache.AddAsync(deck, new CultRecordHandle<CultCacheInteropDeck>(new CultRecordKey(deck.DocumentId)));
cache.FlushAllBackingStores();
WriteDeckJsonLine(deck);
}
static async Task ReadDeckAsync(string file)
{
var cache = BuildCache(file);
await cache.PullAllBackingStoresAsync();
WriteDeckJsonLine(cache.AllEntries.OfType<CultCacheInteropDeck>().Single());
}
static void WriteDeckJsonLine(CultCacheInteropDeck deck) =>
Console.Out.WriteLine(JsonSerializer.Serialize(new { documentId = deck.DocumentId, ids = deck.Marks.Select(mark => mark.Id).ToArray() }));
// Edits a stored record in place, as a consumer holding the object would, and writes it: the file must hold the edit, in whatever
// runtime reads it. payload and override go through the public snapshot API (a decoded record's bytes edited, then
// SerializeSnapshot); collection and object edit the document the cache holds and put the same instance back.
static async Task EditInPlaceAsync(string file, string kind)
{
switch (kind)
{
case "payload":
{
var snapshot = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(file));
OverwriteInPlace(snapshot.Records.Single().Payload, "wrote", "WROTE");
File.WriteAllBytes(file, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot));
break;
}
case "override":
{
var cache = CultCacheMessagePack.Create(file, new CultCacheOpenOptions { Registry = CultDocumentRegistry.ForTypes(new[] { typeof(CultCacheInteropNote) }) });
await cache.PullAllBackingStoresAsync();
var note = cache.AllEntries.OfType<CultCacheInteropNote>().Single();
await cache.UpsertVariantAsync(
new CultRecordKey("note:variant"),
new CultRecordKey(note.DocumentId),
new[]
{
cache.Override<CultCacheInteropNote>(nameof(CultCacheInteropNote.DocumentId), "note:variant"),
cache.Override<CultCacheInteropNote>(nameof(CultCacheInteropNote.Title), "variant title")
});
cache.FlushAllBackingStores();
var snapshot = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(file));
var variant = snapshot.Records.Single(record => record.Variant != null).Variant!;
OverwriteInPlace(variant.Overrides.Single(entry => entry.Path[0].Slot == 3).Value, "variant", "VARIANT");
File.WriteAllBytes(file, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot));
break;
}
case "collection":
{
var cache = BuildCache(file);
await cache.PullAllBackingStoresAsync();
var note = cache.AllEntries.OfType<CultCacheInteropNote>().Single();
note.Tags[0] = "EDITED";
await cache.UpsertAsync(note, new CultRecordHandle<CultCacheInteropNote>(new CultRecordKey(note.DocumentId)));
cache.FlushAllBackingStores();
break;
}
case "object":
{
var cache = BuildCache(file);
await cache.PullAllBackingStoresAsync();
var deck = cache.AllEntries.OfType<CultCacheInteropDeck>().Single();
deck.Marks[0].Label = "EDITED";
await cache.UpsertAsync(deck, new CultRecordHandle<CultCacheInteropDeck>(new CultRecordKey(deck.DocumentId)));
cache.FlushAllBackingStores();
break;
}
default:
throw new InvalidOperationException($"Unknown edit kind {kind}.");
}
Console.Out.WriteLine(JsonSerializer.Serialize(new { kind }));
}
static void OverwriteInPlace(byte[] bytes, string from, string to)
{
var was = System.Text.Encoding.UTF8.GetBytes(from);
var now = System.Text.Encoding.UTF8.GetBytes(to);
var at = bytes.AsSpan().IndexOf(was);
if (at < 0 || now.Length != was.Length)
throw new InvalidOperationException($"Cannot overwrite '{from}' with '{to}' in place.");
now.CopyTo(bytes, at);
}
// Two routed single-file stores: each file is a complete single-store snapshot holding only its own type.
static void WriteRouted(string catalogFile, string runFile)
{
@ -172,7 +292,7 @@ static void WriteJsonLine(CultCacheInteropNote note)
}));
}
[CultDocument("cultcache.interop-note", "cultcache.interop_note.v1")]
[CultDocument("cultcache.interop-note", "cultcache.interop_note.v1", CompatibleSchemaIds = new[] { "cultcache.interop-note.legacy-id" })]
[MessagePackObject]
public sealed class CultCacheInteropNote
{
@ -193,3 +313,18 @@ public sealed class CultCacheInteropRunNote
[Key(2)] public string AuthorRuntimeId { get; set; } = string.Empty;
[Key(3)] public string Body { get; set; } = string.Empty;
}
[CultDocument("cultcache.interop-deck", "cultcache.interop_deck.v1")]
[MessagePackObject]
public sealed class CultCacheInteropDeck
{
[Key(0)] [CultName] public string DocumentId { get; set; } = string.Empty;
[Key(1)] public List<CultCacheInteropMark> Marks { get; set; } = new();
}
[MessagePackObject]
public sealed class CultCacheInteropMark
{
[Key(0)] public string Label { get; set; } = string.Empty;
[Key(1)] [CultElementId] public string Id { get; set; } = string.Empty;
}

3
tests/GameCult.Caching.Tests/AssemblyInfo.cs
View file

@ -6,7 +6,10 @@ using MessagePack.Formatters;
[assembly: CultCacheFormatterResolver(typeof(PairResolver))]
[assembly: CultCacheFormatterResolver(typeof(UnionArmGuardTests.StubArmResolver))]
[assembly: CultCacheFormatterResolver(typeof(UnionArmGuardTests.ClaimArmResolver))]
[assembly: CultCacheFormatterResolver(typeof(UnionArmGuardTests.OrderResolver))]
[assembly: CultCacheFormatterResolver(typeof(UnionArmGuardTests.ListClaimResolver))]
[assembly: CultCacheFormatterResolver(typeof(UnionArmGuardTests.FbLaterClaimResolver))]
namespace GameCult.Caching.Tests
{

140
tests/GameCult.Caching.Tests/C2aGateWriter.cs Normal file
View file

@ -0,0 +1,140 @@
#nullable enable
using System;
using System.IO;
using System.Linq;
using GameCult.Caching;
using GameCult.Caching.MessagePack;
using NUnit.Framework;
namespace GameCult.Caching.Tests
{
// The writer half of the C2a format gate. It writes a record that holds element ids through CultCacheMessagePack.Create, once
// as a single-file store and once as a directory store, and asserts the headers C2a stamps. old-reader-refusal.sh gate runs it
// with CULTLIB_GATE_OUT set, so the 2.0.1 reader probes the very bytes this tree writes; the committed vectors are one such run.
public class C2aGateWriter
{
private const string SingleName = "c2a-written-ids.msgpack";
private const string DirectoryName = "c2a-written-ids-directory";
private const string ManifestName = "store.cc";
private const string DirectoryHeader = "cultcache.store.v5.directory-content-addressed-pages";
private static readonly CultDocumentRegistry Registry = CultDocumentRegistry.ForTypes(new[] { typeof(ElementIdTests.IdDeck) });
// Every id is the author's, so the written bytes and the decoded ids do not depend on minting.
private static readonly string[] Ids = { "00000000c2a1", "00000000c2a2", "00000000c2a3", "00000000c2a4" };
private static ElementIdTests.IdDeck Deck() => new()
{
Name = "gate",
Reels =
{
new ElementIdTests.IdReel
{
Label = "reel",
Id = Ids[0],
Marks = { new ElementIdTests.IdMark { Text = "m", Id = Ids[1] }, new ElementIdTests.IdMark { Text = "m", Id = Ids[2] } }
}
},
Moves = { new ElementIdTests.IdSlash { Reach = 1, Id = Ids[3] } }
};
private static string[] IdsOf(ElementIdTests.IdDeck deck) =>
deck.Reels.Select(reel => reel.Id)
.Concat(deck.Reels.SelectMany(reel => reel.Marks.Select(mark => mark.Id)))
.Concat(deck.Moves.Select(move => move.Id))
.ToArray();
// The one instant every record is stamped with, so the bytes this writes are the same on every run and the committed vectors
// can be compared to them byte for byte. Regenerate the vectors with CULTLIB_GATE_OUT (see C2aStamps...) if C2a's format moves.
private static readonly DateTimeOffset StoredAt = new(2026, 10, 27, 0, 0, 0, TimeSpan.Zero);
private static void Write(string path, bool directory)
{
using var cache = CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = Registry, UseDirectoryStore = directory });
cache.Clock = () => StoredAt;
cache.Commit(batch => batch.Upsert(typeof(ElementIdTests.IdDeck), Deck(), new CultRecordKey("gate")));
}
private static string HeaderOf(string path) =>
CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)).FormatVersion;
private static string[] ReadIds(string path)
{
using var cache = CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = Registry });
return IdsOf(cache.Get<ElementIdTests.IdDeck>(new CultRecordKey("gate"))!);
}
private static string Vectors()
{
for (var dir = new DirectoryInfo(TestContext.CurrentContext.TestDirectory); dir != null; dir = dir.Parent)
{
var vectors = Path.Combine(dir.FullName, "tests", "vectors", "document-variants-c2a");
if (Directory.Exists(vectors)) return vectors;
}
throw new DirectoryNotFoundException($"tests/vectors/document-variants-c2a not found above {TestContext.CurrentContext.TestDirectory}.");
}
// Every file under root by relative path and bytes; a store's lock files hold no state and are not compared.
private static (string Path, byte[] Bytes)[] Tree(string root) =>
Directory.GetFiles(root, "*", SearchOption.AllDirectories)
.Where(file => !file.EndsWith(".lock", StringComparison.Ordinal))
.Select(file => (Path.GetRelativePath(root, file).Replace(Path.DirectorySeparatorChar, '/'), File.ReadAllBytes(file)))
.OrderBy(entry => entry.Item1, StringComparer.Ordinal)
.ToArray();
private string _temp = string.Empty;
[SetUp]
public void CreateTemp()
{
_temp = Path.Combine(Path.GetTempPath(), $"cultlib-c2a-gate-{Guid.NewGuid():N}");
Directory.CreateDirectory(_temp);
}
[TearDown]
public void DeleteTemp()
{
if (Directory.Exists(_temp))
Directory.Delete(_temp, recursive: true);
}
[Test]
public void C2aStampsV3OnASingleFileStoreAndV5OnADirectoryStoreThatHoldIds()
{
var output = Environment.GetEnvironmentVariable("CULTLIB_GATE_OUT");
var root = string.IsNullOrEmpty(output) ? _temp : output;
Directory.CreateDirectory(root);
var single = Path.Combine(root, SingleName);
var directory = Path.Combine(root, DirectoryName);
File.Delete(single);
if (Directory.Exists(directory)) Directory.Delete(directory, recursive: true);
Directory.CreateDirectory(directory);
Write(single, directory: false);
Write(Path.Combine(directory, ManifestName), directory: true);
Assert.That(HeaderOf(single), Is.EqualTo(CultPersistedStoreSnapshot.FormatV3));
Assert.That(HeaderOf(Path.Combine(directory, ManifestName)), Is.EqualTo(DirectoryHeader));
Assert.That(ReadIds(single), Is.EqualTo(Ids));
}
[Test]
public void TheCommittedVectorsAreWhatC2aWrites()
{
var live = Path.Combine(_temp, "live");
var liveDirectory = Path.Combine(live, DirectoryName);
Directory.CreateDirectory(liveDirectory);
Write(Path.Combine(live, SingleName), directory: false);
Write(Path.Combine(liveDirectory, ManifestName), directory: true);
Assert.That(File.ReadAllBytes(Path.Combine(Vectors(), SingleName)), Is.EqualTo(File.ReadAllBytes(Path.Combine(live, SingleName))), SingleName);
var committed = Tree(Path.Combine(Vectors(), DirectoryName));
var written = Tree(liveDirectory);
Assert.That(committed.Select(entry => entry.Path), Is.EqualTo(written.Select(entry => entry.Path)), DirectoryName);
for (var i = 0; i < committed.Length; i++)
Assert.That(committed[i].Bytes, Is.EqualTo(written[i].Bytes), committed[i].Path);
}
}
}

1530
tests/GameCult.Caching.Tests/CachelessSingleFileTests.cs Normal file
View file

File diff suppressed because it is too large Load diff

579
tests/GameCult.Caching.Tests/CompatibleSchemaIdTests.cs Normal file
View file

@ -0,0 +1,579 @@
#nullable enable
using System;
using System.IO;
using System.Linq;
using GameCult.Caching.MessagePack;
using MessagePack;
using NUnit.Framework;
using static GameCult.Caching.Tests.EmittedDocumentTypes;
namespace GameCult.Caching.Tests
{
// A document type declares the older schema ids its records may sit under (CultDocumentAttribute.CompatibleSchemaIds). The
// registered catalog entry lists them, a cache resolves a record under one to that type and stamps it with the type's own id,
// and a store writer never changes the identity of a record it did not receive.
public class CompatibleSchemaIdTests
{
private const string OldId = "old.id";
private const string OwnerId = "sha256:86186abf7d63d8a326c725d39a10f3a4db11b1e8698e8c0fcf6d9720031bc480";
[CultDocument("tests.compat_deck", "tests.compat_deck.v2", CompatibleSchemaIds = new[] { OldId, OldId })]
[MessagePackObject]
public sealed class DeclaringDeck
{
[Key(0)] [CultName] public string Name { get; set; } = "";
}
// The same schema as DeclaringDeck (the class name is not part of a schema id), declaring nothing. Emitted, because a
// declared type that differs from DeclaringDeck only in its declaration is refused beside it by the shared registry.
private static readonly Type BareDeck = Emit("BareDeck", "tests.compat_deck", "tests.compat_deck.v2", new[] { new Field("Name", typeof(string), 0, IsName: true) });
[CultDocument("tests.compat_owner", "tests.compat_owner.v1", CompatibleSchemaIds = new[] { OwnerId, " " })]
[MessagePackObject]
public sealed class Owner
{
[Key(0)] [CultName] public string Name { get; set; } = "";
}
// One schema before and after a member was added, with nothing declared: the later type reads the earlier records by name.
[CultDocument("tests.compat_evolving", "tests.compat_evolving.v1")]
[MessagePackObject]
public sealed class EvolvingV1
{
[Key(0)] [CultName] public string Name { get; set; } = "";
}
[CultDocument("tests.compat_evolving", "tests.compat_evolving.v2")]
[MessagePackObject]
public sealed class EvolvingV2
{
[Key(0)] [CultName] public string Name { get; set; } = "";
[Key(1)] public string Extra { get; set; } = "";
}
private static readonly CultDocumentRegistry Declaring = CultDocumentRegistry.ForTypes(new[] { typeof(DeclaringDeck) });
private static readonly CultDocumentRegistry Bare = CultDocumentRegistry.ForTypes(new[] { BareDeck });
private static readonly CultRecordKey D = new("d");
private string _directory = "";
[SetUp]
public void SetUp()
{
_directory = Path.Combine(Path.GetTempPath(), "cultlib-compat-" + Guid.NewGuid().ToString("N"));
Directory.CreateDirectory(_directory);
}
[TearDown]
public void TearDown()
{
if (Directory.Exists(_directory))
Directory.Delete(_directory, true);
}
private static CultCache Open(string path, CultDocumentRegistry registry) =>
CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = registry });
private static CultPersistedStoreSnapshot Read(string path) =>
CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path));
private static object Deck(string name) => New(BareDeck, ("Name", name));
private static void Rewrite(string path, Action<CultPersistedStoreSnapshot> edit)
{
var snapshot = Read(path);
edit(snapshot);
File.WriteAllBytes(path, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot));
}
// A store written by a runtime that knew the schema under an older id and another name: one entry that owns OldId, one
// record under it. Nothing but the declaration lets this runtime's type claim it.
private string OldStore(string name)
{
var path = Path.Combine(_directory, name);
using (var cache = Open(path, Bare))
cache.Commit(batch => batch.Upsert(BareDeck, Deck("d"), D));
Rewrite(path, snapshot =>
{
var entry = snapshot.SchemaCatalog.Single();
entry.SchemaId = OldId;
entry.SchemaName = "tests.legacy_deck";
entry.ContentHash = "stale";
entry.CompatibleSchemaIds = new[] { OldId };
snapshot.Records.Single().SchemaId = OldId;
});
return path;
}
// The same store with the entry keeping this schema's name and only listing the older id: what a store carries after a
// writer that did not declare it, so a runtime without the declaration can still read it.
private string OldStoreUnderTheSameName(string name)
{
var path = Path.Combine(_directory, name);
using (var cache = Open(path, Bare))
cache.Commit(batch => batch.Upsert(BareDeck, Deck("d"), D));
Rewrite(path, snapshot =>
{
var entry = snapshot.SchemaCatalog.Single();
entry.ContentHash = "stale";
entry.CompatibleSchemaIds = new[] { entry.SchemaId, OldId };
snapshot.Records.Single().SchemaId = OldId;
});
return path;
}
[Test]
public void TheEmittedBareDeckIsTheDeclaringDecksSchema()
{
Assert.That(Bare.GetRequired(BareDeck).SchemaId, Is.EqualTo(Declaring.GetRequired<DeclaringDeck>().SchemaId));
}
[Test]
public void ARecordUnderADeclaredOldIdOpensAndACommitRewritesItUnderTheRegisteredId()
{
var path = OldStore("old.cc");
var descriptor = Declaring.GetRequired<DeclaringDeck>();
using (var cache = Open(path, Declaring))
{
Assert.That(cache.Get<DeclaringDeck>(D)!.Name, Is.EqualTo("d"), "the declaration resolves the record to the type");
Assert.That(cache.Commit(batch => batch.Upsert(typeof(DeclaringDeck), new DeclaringDeck { Name = "d2" }, D)), Is.True);
}
var written = Read(path);
Assert.That(written.Records.Single().SchemaId, Is.EqualTo(descriptor.SchemaId));
var entry = written.SchemaCatalog.Single();
Assert.That(entry.SchemaId, Is.EqualTo(descriptor.SchemaId));
Assert.That(entry.SchemaName, Is.EqualTo("tests.compat_deck"));
Assert.That(entry.CompatibleSchemaIds, Is.EqualTo(new[] { descriptor.SchemaId, OldId }));
foreach (var registry in new[] { Declaring, Bare })
{
using var reopened = Open(path, registry);
Assert.That(reopened.AllEntries.Count(), Is.EqualTo(1));
}
}
// The name a catalog entry carries is metadata: a record resolves by its schema id, so a schema renamed under the id the
// registered type owns opens, with the entry's content hash no longer matching the type's.
[Test]
public void ASchemaRenamedUnderTheIdARegisteredTypeOwnsOpens()
{
var path = Path.Combine(_directory, "renamed.cc");
using (var cache = Open(path, Bare))
cache.Commit(batch => batch.Upsert(BareDeck, Deck("d"), D));
Rewrite(path, snapshot =>
{
snapshot.SchemaCatalog.Single().SchemaName = "tests.renamed_deck";
snapshot.SchemaCatalog.Single().ContentHash = "stale";
});
using var reopened = Open(path, Bare);
Assert.That(EmittedDocumentTypes.Read(reopened.Get(D)!, "Name"), Is.EqualTo("d"));
}
[Test]
public void ARecordUnderAnOldIdNoTypeDeclaresIsCarriedAsForeignNotOpenedByThatSchemaName()
{
var path = OldStore("undeclared.cc");
using var cache = Open(path, Bare);
Assert.That(cache.Get(D), Is.Null, "the entry's name is metadata; only a declared id resolves it");
Assert.That(cache.BackingStores.Single().ForeignRecords.Select(record => (record.Key, record.SchemaId, record.SchemaName)),
Is.EqualTo(new[] { (D.Value, OldId, "tests.legacy_deck") }));
}
// An unconditional commit writes what it staged: the record the batch does not name is copied as the file holds it, under
// the old id and with the entry that publishes that id, though the cache resolved it to the registered type.
[Test]
public void AnUnconditionalCommitCopiesALoadedRecordUnderItsOldIdAsTheFileHoldsIt()
{
var path = OldStore("copied.cc");
var descriptor = Declaring.GetRequired<DeclaringDeck>();
var before = Read(path);
var entryBefore = StoreSlices.CatalogEntry(path, OldId);
using (var cache = Open(path, Declaring))
cache.Commit(batch => batch.Upsert(typeof(DeclaringDeck), new DeclaringDeck { Name = "e" }, new CultRecordKey("e")));
var written = Read(path);
Assert.That(written.Records.Single(record => record.Key == "d").SchemaId, Is.EqualTo(OldId));
Assert.That(written.Records.Single(record => record.Key == "e").SchemaId, Is.EqualTo(descriptor.SchemaId));
var copied = written.Records.Single(record => record.Key == "d");
var original = before.Records.Single();
Assert.That((copied.StoredAt, copied.Payload), Is.EqualTo((original.StoredAt, original.Payload)));
Assert.That(StoreSlices.CatalogEntry(path, OldId), Is.EqualTo(entryBefore),
"the entry that published the old id is carried as its bytes");
}
// A commit onto the file writes its batch and leaves every other record as the file holds it: the store writer never
// changes identity. The record stays under the older id, which the registered entry publishes as compatible.
[Test]
public void ACommitOntoTheFileLeavesARecordUnderADeclaredOldIdAsItIsAndPublishesTheId()
{
var path = OldStoreUnderTheSameName("onto.cc");
var descriptor = Declaring.GetRequired<DeclaringDeck>();
using (var cache = Open(path, Declaring))
{
Assert.That(cache.Get<DeclaringDeck>(D), Is.Not.Null);
Assert.That(cache.Commit(batch =>
{
batch.Expect(new CultRecordKey("e"), null);
batch.Upsert(typeof(DeclaringDeck), new DeclaringDeck { Name = "e" }, new CultRecordKey("e"));
}), Is.True);
}
var written = Read(path);
Assert.That(written.Records.Single(record => record.Key == "d").SchemaId, Is.EqualTo(OldId));
Assert.That(written.Records.Single(record => record.Key == "e").SchemaId, Is.EqualTo(descriptor.SchemaId));
Assert.That(written.SchemaCatalog.Single().CompatibleSchemaIds, Does.Contain(OldId));
using var reopened = Open(path, Declaring);
Assert.That(reopened.Get<DeclaringDeck>(D), Is.Not.Null);
}
[Test]
public void WithoutTheDeclarationACommitOntoTheFileIsRefusedTypedAndTheFileLeftAsItWas()
{
var path = OldStoreUnderTheSameName("undeclared-onto.cc");
var bytes = File.ReadAllBytes(path);
using var cache = Open(path, Bare);
var refusal = Assert.Throws<CultSchemaConflictException>(() => cache.Commit(batch =>
{
batch.Expect(new CultRecordKey("e"), null);
batch.Upsert(BareDeck, Deck("e"), new CultRecordKey("e"));
}))!;
Assert.That(refusal.SchemaId, Is.EqualTo(OldId));
Assert.That(File.ReadAllBytes(path), Is.EqualTo(bytes));
}
// A record under a declared old id is the same record the cache holds, loaded under that id: a condition naming it holds.
[Test]
public void AConditionOnARecordUnderADeclaredOldIdHoldsAndTheWholeStoreCountsAsUnchanged()
{
var path = OldStoreUnderTheSameName("conditions.cc");
using var cache = Open(path, Declaring);
var current = cache.Get<DeclaringDeck>(D)!;
Assert.That(cache.TryCommit(batch =>
{
batch.Expect(D, current);
batch.ExpectUnchanged();
batch.Upsert(typeof(DeclaringDeck), new DeclaringDeck { Name = "e" }, new CultRecordKey("e"));
}), Is.EqualTo(CultCommitOutcome.Committed));
}
// The directory store judges conditions against its manifest, whose record still sits under the older id.
[Test]
public void AConditionOnARecordUnderADeclaredOldIdHoldsInTheDirectoryStore()
{
var path = Path.Combine(_directory, "dir.cc");
var options = new CultCacheOpenOptions { Registry = Bare, UseDirectoryStore = true };
using (var cache = CultCacheMessagePack.Create(path, options))
cache.Commit(batch => batch.Upsert(BareDeck, Deck("d"), D));
Rewrite(path, manifest =>
{
var entry = manifest.SchemaCatalog.Single();
entry.ContentHash = "stale";
entry.CompatibleSchemaIds = new[] { entry.SchemaId, OldId };
manifest.Records.Single().SchemaId = OldId;
});
using var declaring = CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = Declaring, UseDirectoryStore = true });
var current = declaring.Get<DeclaringDeck>(D)!;
Assert.That(declaring.TryCommit(batch =>
{
batch.Expect(D, current);
batch.ExpectUnchanged();
batch.Upsert(typeof(DeclaringDeck), new DeclaringDeck { Name = "e" }, new CultRecordKey("e"));
}), Is.EqualTo(CultCommitOutcome.Committed));
}
// Conditions compare the id a record was loaded under with the file's, exactly. A record resolved by some other path than
// its type's own id is still unchanged when nothing on disk moved: by its schema name under a foreign id, by name across a
// schema change that declared nothing, or by a file entry that lists this runtime's id.
private static void AssertUnchangedConditionsHold(CultCache cache, CultRecordKey key, object current, Type type, object added)
{
Assert.That(cache.TryCommit(batch =>
{
batch.Expect(key, current);
batch.Upsert(type, added, new CultRecordKey("added-" + Guid.NewGuid().ToString("N")));
}), Is.EqualTo(CultCommitOutcome.Committed), "Expect");
Assert.That(cache.TryCommit(batch =>
{
batch.ExpectUnchanged();
batch.Upsert(type, added, new CultRecordKey("added-" + Guid.NewGuid().ToString("N")));
}), Is.EqualTo(CultCommitOutcome.Committed), "ExpectUnchanged");
}
[Test]
public void AConditionOnARecordResolvedByNameUnderAForeignIdHolds()
{
var path = Path.Combine(_directory, "foreign.cc");
using (var cache = Open(path, Bare))
cache.Commit(batch => batch.Upsert(BareDeck, Deck("d"), D));
Rewrite(path, snapshot =>
{
var entry = snapshot.SchemaCatalog.Single();
entry.SchemaId = "foreign.id";
entry.CompatibleSchemaIds = new[] { "foreign.id" };
snapshot.Records.Single().SchemaId = "foreign.id";
});
using var reader = Open(path, Declaring);
AssertUnchangedConditionsHold(reader, D, reader.Get<DeclaringDeck>(D)!, typeof(DeclaringDeck), new DeclaringDeck { Name = "e" });
}
[Test]
public void AConditionOnARecordAnEvolvedTypeResolvedByNameHolds()
{
var path = Path.Combine(_directory, "evolved.cc");
using (var cache = Open(path, CultDocumentRegistry.ForTypes(new[] { typeof(EvolvingV1) })))
cache.Commit(batch => batch.Upsert(typeof(EvolvingV1), new EvolvingV1 { Name = "d" }, D));
using var reader = Open(path, CultDocumentRegistry.ForTypes(new[] { typeof(EvolvingV2) }));
AssertUnchangedConditionsHold(reader, D, reader.Get<EvolvingV2>(D)!, typeof(EvolvingV2), new EvolvingV2 { Name = "e" });
}
[Test]
public void AConditionOnARecordAFileEntryListingTheLocalIdResolvedHolds()
{
var path = Path.Combine(_directory, "listed.cc");
var localId = Bare.GetRequired(BareDeck).SchemaId;
using (var cache = Open(path, Bare))
cache.Commit(batch => batch.Upsert(BareDeck, Deck("d"), D));
Rewrite(path, snapshot =>
{
var entry = snapshot.SchemaCatalog.Single();
entry.SchemaId = OldId;
entry.CompatibleSchemaIds = new[] { OldId, localId };
snapshot.Records.Single().SchemaId = OldId;
});
using var reader = Open(path, Bare);
AssertUnchangedConditionsHold(reader, D, reader.Get(D)!, BareDeck, Deck("e"));
}
// A plain record and a variant loaded under a declared old id carry that id, so a condition on either holds while the file
// is unchanged. An unconditional commit then copies both as they are, still under the old id, and the cache still holds
// each under it: the same conditions hold against the rewritten file.
[Test]
public void ConditionsOnRecordsLoadedUnderAnOldIdHoldBeforeAndAfterAnUnconditionalCommit()
{
var path = Path.Combine(_directory, "restamped.cc");
var b = new CultRecordKey("b");
var v = new CultRecordKey("v");
using (var cache = Open(path, Declaring))
{
cache.Commit(batch => batch.Upsert(typeof(DeclaringDeck), new DeclaringDeck { Name = "b" }, b));
cache.Commit(batch => batch.UpsertVariant(v, b, new[] { cache.Override(typeof(DeclaringDeck), nameof(DeclaringDeck.Name), "v") }));
}
Rewrite(path, snapshot =>
{
foreach (var record in snapshot.Records)
record.SchemaId = OldId;
});
using var reader = Open(path, Declaring);
void ConditionsHold(string phase)
{
foreach (var key in new[] { b, v })
{
Assert.That(reader.TryCommit(batch =>
{
batch.Expect(key, reader.Get(key)!);
batch.Upsert(typeof(DeclaringDeck), new DeclaringDeck { Name = "f" }, new CultRecordKey($"f-{phase}-{key.Value}"));
}), Is.EqualTo(CultCommitOutcome.Committed), $"{phase}: Expect {key.Value}");
}
Assert.That(reader.TryCommit(batch =>
{
batch.ExpectUnchanged();
batch.Upsert(typeof(DeclaringDeck), new DeclaringDeck { Name = "g" }, new CultRecordKey($"g-{phase}"));
}), Is.EqualTo(CultCommitOutcome.Committed), $"{phase}: ExpectUnchanged");
}
ConditionsHold("loaded");
Assert.That(Read(path).Records.Where(record => record.Key is "b" or "v").Select(record => record.SchemaId), Is.All.EqualTo(OldId),
"commits onto the file left the loaded records under the old id");
reader.Commit(batch => batch.Upsert(typeof(DeclaringDeck), new DeclaringDeck { Name = "e" }, new CultRecordKey("e")));
Assert.That(Read(path).Records.Where(record => record.Key is "b" or "v").Select(record => record.SchemaId), Is.All.EqualTo(OldId),
"the unconditional commit copied the loaded records under the old id");
ConditionsHold("rewritten");
}
// The directory store rereads a record whose manifest entry moved to another id with the same storedAt, as a rewrite that
// mints no storedAt leaves it: after a pull the cache holds the record under that id, and a condition on it holds.
[Test]
public void ADirectoryStorePullRereadsARecordWhoseIdAloneChanged()
{
var path = Path.Combine(_directory, "dir-moved.cc");
var options = new CultCacheOpenOptions { Registry = Declaring, UseDirectoryStore = true };
using (var writer = CultCacheMessagePack.Create(path, options))
writer.Commit(batch => batch.Upsert(typeof(DeclaringDeck), new DeclaringDeck { Name = "d" }, D));
using var cache = CultCacheMessagePack.Create(path, options);
Rewrite(path, manifest => manifest.Records.Single().SchemaId = OldId);
cache.PullAllBackingStoresAsync().GetAwaiter().GetResult();
Assert.That(cache.TryCommit(batch =>
{
batch.Expect(D, cache.Get<DeclaringDeck>(D)!);
batch.Upsert(typeof(DeclaringDeck), new DeclaringDeck { Name = "e" }, new CultRecordKey("e"));
}), Is.EqualTo(CultCommitOutcome.Committed));
}
// A rolling deployment shares one single-file store between v1 (one member) and v2 (a second member, declaring v1's id).
private static readonly Field RollName = new("Name", typeof(string), 0, IsName: true);
private static readonly Type RollV1 = Emit("RollV1", "tests.compat_roll", "tests.compat_roll.v1", new[] { RollName });
private static readonly string RollV1Id = CultDocumentRegistry.ForTypes(new[] { RollV1 }).GetRequired(RollV1).SchemaId;
private static readonly Type RollV2 = Emit("RollV2", "tests.compat_roll", "tests.compat_roll.v2", new[] { RollName, new Field("Extra", typeof(string), 1) }, new[] { RollV1Id });
private static readonly CultRecordKey RollK = new("k");
private static readonly CultRecordKey RollY = new("y");
private CultCache OpenRoll(Type type) => Open(Path.Combine(_directory, "roll.cc"), CultDocumentRegistry.ForTypes(new[] { type }));
private string RollPath => Path.Combine(_directory, "roll.cc");
private static string Extra(object? document) => (string?)EmittedDocumentTypes.Read(document!, "Extra") ?? "";
// Writes k with Extra "important" as v2.
private void WriteImportant()
{
using var writer = OpenRoll(RollV2);
writer.Commit(batch => batch.Upsert(RollV2, New(RollV2, ("Name", "k"), ("Extra", "important")), RollK));
}
// A write by a cache of the given version, through a commit or a plain flush, that leaves the key set as it was.
private void UnrelatedWrite(Type version, bool viaFlush)
{
using var cache = OpenRoll(version);
Assert.That(EmittedDocumentTypes.Read(cache.Get(RollK)!, "Name"), Is.EqualTo("k"), $"{version.Name} reads k");
if (viaFlush)
{
cache.BackingStores.Single().PushAll();
return;
}
cache.Commit(batch => batch.Upsert(version, New(version, ("Name", "z")), new CultRecordKey("z")));
cache.Commit(batch => batch.Remove(new CultRecordKey("z")));
}
// The derivation a v2 cache commits from the k it holds, under a condition on that k.
private static CultCommitOutcome Derive(CultCache cache, object held, bool unchangedOnly) => cache.TryCommit(batch =>
{
if (unchangedOnly)
batch.ExpectUnchanged();
else
batch.Expect(RollK, held);
batch.Upsert(RollV2, New(RollV2, ("Name", "y"), ("Extra", "derived-from-" + Extra(held))), RollY);
});
// v1 reads v2's record through the entry that lists v1's id: an id v1 does not own. A write of another record by v1 copies k
// exactly as v2 stored it, the member v1 lacks included, under v2's id and storedAt, so a v2 cache's condition on k still holds
// and what it derives from that member lands.
[Test]
public void AnUnrelatedWriteByAnOlderVersionLeavesANewerRecordAsStored([Values] bool unchangedOnly, [Values] bool viaFlush)
{
WriteImportant();
var written = Read(RollPath);
using var newer = OpenRoll(RollV2);
var current = newer.Get(RollK)!;
UnrelatedWrite(RollV1, viaFlush);
var rewritten = Read(RollPath);
var record = rewritten.Records.Single();
Assert.That((record.SchemaId, record.StoredAt, record.Payload), Is.EqualTo((written.Records.Single().SchemaId, written.Records.Single().StoredAt, written.Records.Single().Payload)));
Assert.That(CultDocumentMessagePackSerialization.SerializeSnapshot(new CultPersistedStoreSnapshot { SchemaCatalog = rewritten.SchemaCatalog }),
Is.EqualTo(CultDocumentMessagePackSerialization.SerializeSnapshot(new CultPersistedStoreSnapshot { SchemaCatalog = written.SchemaCatalog })),
"v2's catalog entry is written as v2 wrote it");
Assert.That(Derive(newer, current, unchangedOnly), Is.EqualTo(CultCommitOutcome.Committed));
using var reopened = OpenRoll(RollV2);
Assert.That(Extra(reopened.Get(RollK)), Is.EqualTo("important"));
Assert.That(Extra(reopened.Get(RollY)), Is.EqualTo("derived-from-important"));
}
// v1 reads v2's k through v2's entry, which lists v1's id: the list is the writer's statement, not v1's declaration, so to v1
// k is read-only. v1's write of k is refused typed before anything is staged, the file is left byte for byte, and a v2
// cache's condition on k still holds.
[Test]
public void AnOlderVersionCannotWriteANewerRecordItReadsThroughTheNewerEntry([Values] bool commit)
{
WriteImportant();
var before = File.ReadAllBytes(RollPath);
using var newer = OpenRoll(RollV2);
var current = newer.Get(RollK)!;
using (var older = OpenRoll(RollV1))
{
var k = older.Get(RollK)!;
Assert.That(EmittedDocumentTypes.Read(k, "Name"), Is.EqualTo("k"), "v1 reads k");
var refusal = Assert.Throws<CultSchemaConflictException>(() =>
{
if (commit)
older.Commit(batch => batch.Upsert(RollV1, k, RollK));
else
older.UpsertAsync(RollV1, k, RollK).GetAwaiter().GetResult();
})!;
Assert.That((refusal.RecordKey, refusal.SchemaId), Is.EqualTo((RollK.Value, Read(RollPath).Records.Single().SchemaId)));
Assert.That(older.BackingStores.Single().IsDirty, Is.False, "nothing was staged");
older.FlushAsync().GetAwaiter().GetResult();
}
Assert.That(File.ReadAllBytes(RollPath), Is.EqualTo(before));
Assert.That(Derive(newer, current, unchangedOnly: false), Is.EqualTo(CultCommitOutcome.Committed));
using var reopened = OpenRoll(RollV2);
Assert.That(Extra(reopened.Get(RollK)), Is.EqualTo("important"));
Assert.That(Extra(reopened.Get(RollY)), Is.EqualTo("derived-from-important"));
}
// A record a write copies keeps its storedAt, so a condition another cache holds on it survives an unrelated write.
[TestCase(false)]
[TestCase(true)]
public void AnUnrelatedWriteKeepsTheStoredAtOfARecordItCopies(bool viaFlush)
{
WriteImportant();
var written = Read(RollPath).Records.Single();
using var held = OpenRoll(RollV2);
var current = held.Get(RollK)!;
UnrelatedWrite(RollV2, viaFlush);
Assert.That(Read(RollPath).Records.Single().StoredAt, Is.EqualTo(written.StoredAt));
Assert.That(Derive(held, current, unchangedOnly: false), Is.EqualTo(CultCommitOutcome.Committed));
}
// The migration CompatibleSchemaIds exists for: v2 declares v1's id and v1's class stays registered beside it, in either
// order. A record under v1's id is v1's, its owner; one under v2's id is v2's.
[TestCase(false)]
[TestCase(true)]
public void AVersionKeptBesideTheVersionDeclaringItsIdReadsItsOwnRecords(bool reversed)
{
using (var older = OpenRoll(RollV1))
older.Commit(batch => batch.Upsert(RollV1, New(RollV1, ("Name", "k")), RollK));
using (var newer = OpenRoll(RollV2))
newer.Commit(batch =>
{
batch.Expect(RollY, null);
batch.Upsert(RollV2, New(RollV2, ("Name", "y"), ("Extra", "e")), RollY);
});
Assert.That(Read(RollPath).Records.Select(record => record.SchemaId), Is.EquivalentTo(new[] { RollV1Id, CultDocumentRegistry.ForTypes(new[] { RollV2 }).GetRequired(RollV2).SchemaId }));
using var both = Open(RollPath, CultDocumentRegistry.ForTypes(reversed ? new[] { RollV2, RollV1 } : new[] { RollV1, RollV2 }));
Assert.That(both.Get(RollK)!.GetType(), Is.EqualTo(RollV1));
Assert.That(both.Get(RollY)!.GetType(), Is.EqualTo(RollV2));
}
[Test]
public void ADeclarationIsRegisteredWithoutItsOwnIdAndItsEntryListsTheOwnIdFirst()
{
var registry = CultDocumentRegistry.ForTypes(new[] { typeof(Owner) });
var descriptor = registry.GetRequired<Owner>();
Assert.That(descriptor.SchemaId, Is.EqualTo(OwnerId), "the pinned id is this schema's id");
Assert.That(descriptor.CompatibleSchemaIds, Is.Empty, "a type's own id and a blank id are not declared compatible ids");
Assert.That(descriptor.ToCatalogEntry().CompatibleSchemaIds, Is.EqualTo(new[] { OwnerId }));
}
}
}

157
tests/GameCult.Caching.Tests/ConditionalCommitTests.cs
View file

@ -83,18 +83,18 @@ namespace GameCult.Caching.Tests
}
[Test]
public void RecordRewrittenUnderAnotherSchemaWithItsStoredAtFailsExpect()
public void RecordRewrittenUnderAnotherSchemaFailsExpect()
{
SeedDrift(directory: false);
using var cache = Drift(false, NarrowDrift).Open();
var observed = cache.Get(DriftKey)!;
var storedAt = cache.GetStored(DriftKey)!.StoredAt;
using (var other = Drift(false, ReusedDrift).Open())
{
other.BackingStores[0].Push(other.GetStored(DriftKey)!);
other.BackingStores[0].PushAll();
using (var reader = Drift(false, ReusedDrift).Open())
Assert.That(reader.GetStored(DriftKey)!.StoredAt, Is.EqualTo(storedAt), "the rewrite kept storedAt");
}
Assert.That(DriftCommit(cache, observed, 2), Is.False, "a record rewritten under another schema passed Expect on storedAt alone");
Assert.That(DriftCommit(cache, observed, 2), Is.False, "a record rewritten under another schema passed Expect");
}
// The directory store rewrites only what is pushed to it, so the other writer pushes the record it read unchanged.
@ -137,51 +137,10 @@ namespace GameCult.Caching.Tests
cache.BackingStores[0].PushAll();
var stored = cache.GetStored(DriftKey)!;
Assert.That(stored.PersistedSchemaId, Is.EqualTo(stored.Descriptor.SchemaId), "the rewrite was not recorded");
Assert.That(stored.StoredSchemaId, Is.EqualTo(stored.Descriptor.SchemaId), "the rewrite was not recorded");
Assert.That(DriftCommit(cache, cache.Get(DriftKey)!, 2), Is.True, "Expect failed after this cache's own push rewrote the record");
}
// A store written outside the library marks its own rewrite through the base class.
[Test]
public void ACustomStoreMarksItsOwnWholeFileRewrite()
{
SeedDrift(directory: false);
using var cache = new CultCache(NarrowDrift);
var store = new WholeFileStore(PathOf("drift.cc"));
cache.AddBackingStore(store);
store.PullAll();
var loaded = cache.GetStored(DriftKey)!;
Assert.That(loaded.PersistedSchemaId, Is.Not.EqualTo(loaded.Descriptor.SchemaId), "the record was not loaded under drift");
store.Push(loaded);
store.PushAll();
Assert.That(cache.GetStored(DriftKey)!.PersistedSchemaId, Is.EqualTo(loaded.Descriptor.SchemaId), "the custom store's rewrite was not recorded");
Assert.That(DriftCommit(cache, cache.Get(DriftKey)!, 2), Is.True, "Expect failed after the custom store rewrote the record");
using var reader = Drift(false, NarrowDrift).Open();
Assert.That(reader.Get<ConditionalDriftNarrow>(DriftKey)!.Value, Is.EqualTo(2));
}
// A store marks only what it holds: another cache's copy of the same record is refused and left as read.
[Test]
public void ACustomStoreCannotMarkAnEntryItDoesNotHold()
{
SeedDrift(directory: false);
using var owner = new CultCache(NarrowDrift);
var ownerStore = new WholeFileStore(PathOf("drift.cc"));
owner.AddBackingStore(ownerStore);
ownerStore.PullAll();
using var other = new CultCache(NarrowDrift);
var otherStore = new WholeFileStore(PathOf("drift.cc"));
other.AddBackingStore(otherStore);
otherStore.PullAll();
var foreign = owner.GetStored(DriftKey)!;
var persisted = foreign.PersistedSchemaId;
Assert.Throws<ArgumentException>(() => otherStore.Mark(foreign), "a store marked another cache's entry");
Assert.That(foreign.PersistedSchemaId, Is.EqualTo(persisted), "a refused mark changed the entry");
}
[Test]
public void OwnUnconditionalCommitKeepsDriftedExpectHolding()
{
@ -215,12 +174,12 @@ namespace GameCult.Caching.Tests
wide = writer.GetStored(DriftKey)!.Descriptor.SchemaId;
using var cache = Drift(false, NarrowDrift).Open();
var loaded = cache.GetStored(DriftKey)!;
Assert.That(loaded.PersistedSchemaId, Is.EqualTo(wide), "the entry carries the schema id its record is stored under");
Assert.That(loaded.StoredSchemaId, Is.EqualTo(wide), "the entry carries the schema id its record is stored under");
Assert.That(loaded.Descriptor.SchemaId, Is.Not.EqualTo(wide));
Assert.That(DriftCommit(cache, cache.Get(DriftKey)!, 2), Is.True);
var written = cache.GetStored(DriftKey)!;
Assert.That(written.PersistedSchemaId, Is.EqualTo(written.Descriptor.SchemaId));
Assert.That(written.StoredSchemaId, Is.EqualTo(written.Descriptor.SchemaId));
Assert.That(DriftCommit(cache, cache.Get(DriftKey)!, 3), Is.True, "Expect failed on the record this cache just wrote");
}
@ -429,6 +388,24 @@ namespace GameCult.Caching.Tests
Assert.That(reads, Is.GreaterThan(0));
}
// With no backing store a record's version is minted with it: an Expect on the held record commits, and once the record is
// replaced after it was observed (here by the same document again) the Expect no longer holds.
[Test]
public void InMemoryExpectHoldsUntilTheRecordIsReplaced()
{
using var cache = new CultCache(CultDocumentRegistry.ForTypes(new[] { typeof(Counter) }));
cache.Commit(batch => batch.Upsert(new Counter { Name = "counter", Value = 1 }, new CultRecordHandle<Counter>(Key)));
var first = cache.Get<Counter>(Key)!;
Assert.That(cache.Commit(batch => batch.Expect(Key, first)), Is.True);
Assert.That(cache.Commit(batch =>
{
batch.Expect(Key, first);
cache.UpsertAsync(first, new CultRecordHandle<Counter>(Key)).GetAwaiter().GetResult();
}), Is.False);
}
[Test]
public void StoredAtIsStrictlyIncreasingPerKey()
{
@ -775,7 +752,7 @@ namespace GameCult.Caching.Tests
}
[Test]
public void UnconditionalCommitIsLastWriterWinsLikeFlush()
public void UnconditionalCommitIsLastWriterWinsPerKeyLikeFlush()
{
var committed = LastWriter(SingleFile(PathOf("commit.cc")), (cache, key) =>
Assert.That(cache.Commit(batch => batch.Upsert(new Counter { Name = key.Value }, new CultRecordHandle<Counter>(key))), Is.True));
@ -785,10 +762,10 @@ namespace GameCult.Caching.Tests
cache.FlushAllBackingStores();
});
Assert.That(committed, Is.EqualTo(new[] { "a", "x" }), "an unconditional commit kept a record its cache never saw");
Assert.That(committed, Is.EqualTo(new[] { "a", "c", "x" }), "an unconditional commit dropped a record another writer added");
Assert.That(committed, Is.EqualTo(flushed));
// A directory store's flush lands staged keys onto the current manifest, and its commit does exactly the same.
// A directory store's flush lands staged keys onto the current manifest, and its commit does exactly the same: so does a single file's.
var directoryCommitted = LastWriter(DirectoryStore(PathOf("commit-dir.cc")), (cache, key) =>
Assert.That(cache.Commit(batch => batch.Upsert(new Counter { Name = key.Value }, new CultRecordHandle<Counter>(key))), Is.True));
var directoryFlushed = LastWriter(DirectoryStore(PathOf("flush-dir.cc")), (cache, key) =>
@ -830,9 +807,11 @@ namespace GameCult.Caching.Tests
writer.FlushAllBackingStores();
}
// A record of a schema this cache does not register is not corruption: the store opens and carries it (ForeignRecordTests).
var bytes = File.ReadAllBytes(unknown);
using var narrow = new CultCache(CultDocumentRegistry.ForTypes(new[] { typeof(Counter) }));
Assert.That(() => narrow.AddBackingStore(new SingleFileMessagePackBackingStore(unknown)), Throws.Exception, "a store with an unknown schema opened");
narrow.AddBackingStore(new SingleFileMessagePackBackingStore(unknown));
Assert.That(narrow.BackingStores.Single().ForeignRecords, Has.Count.EqualTo(1));
Assert.That(File.ReadAllBytes(unknown), Is.EqualTo(bytes), "opening rewrote a store with an unknown schema");
}
@ -1029,6 +1008,10 @@ namespace GameCult.Caching.Tests
private static StoreFactory DirectoryStore(string path, CultDocumentRegistry? registry = null) =>
new(path, () => new DirectoryMessagePackBackingStore(path), registry);
// The id Wide is stored under, which the later versions declare compatible: a cache reads and writes a record under another
// id only when its type declares it. SeedDrift checks that the constant is Wide's id.
private const string WideDriftSchemaId = "sha256:6033aeca8a784899aff1f8c43eb9df6bf3364e4fb6014e423c65afd51a9b8f18";
private static readonly CultRecordKey DriftKey = new("drift");
private static readonly CultDocumentRegistry WideDrift = CultDocumentRegistry.ForTypes(new[] { typeof(ConditionalDriftWide) });
private static readonly CultDocumentRegistry NarrowDrift = CultDocumentRegistry.ForTypes(new[] { typeof(ConditionalDriftNarrow) });
@ -1040,6 +1023,7 @@ namespace GameCult.Caching.Tests
// The record at DriftKey, written under v1.
private void SeedDrift(bool directory)
{
Assert.That(WideDrift.GetRequired(typeof(ConditionalDriftWide)).SchemaId, Is.EqualTo(WideDriftSchemaId));
using var cache = Drift(directory, WideDrift).Open();
Assert.That(cache.Commit(batch => batch.Upsert(
new ConditionalDriftWide { Name = "drift", Value = 1, Extra = "extra" },
@ -1052,65 +1036,6 @@ namespace GameCult.Caching.Tests
batch.Upsert(new ConditionalDriftNarrow { Name = "drift", Value = value }, new CultRecordHandle<ConditionalDriftNarrow>(DriftKey));
});
// A store derived from CacheBackingStore alone, as a consumer would write one: it rewrites the whole file on every
// write, in the single-file MessagePack format, and marks each record it rewrote.
private sealed class WholeFileStore : CacheBackingStore
{
private readonly string _path;
public WholeFileStore(string path) => _path = path;
private CultPersistedStoreSnapshot Read() => CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(_path));
public override void PullAll() => Held(() =>
{
var snapshot = Read();
var loaded = snapshot.Records
.Select(record => ToStoredDocument(record, snapshot.SchemaCatalog,
(type, payload) => CultDocumentMessagePackSerialization.DeserializeUntyped(type, payload, Registry), out _))
.ToArray();
Loaded?.Invoke(loaded, Array.Empty<CultStoredDocument>());
foreach (var stored in loaded)
Entries[stored.Key.Value] = stored;
});
public override void Push(CultStoredDocument entry) => Held(() =>
{
Entries[entry.Key.Value] = entry;
IsDirty = true;
});
public override void Delete(CultStoredDocument entry) => throw new NotSupportedException();
public void Mark(CultStoredDocument entry) => MarkWritten(entry);
public override void PushAll() => Held(Write);
public override CultCommitOutcome CommitBatch(CultCommitRequest request, bool wait) => Held(() =>
{
if (!request.ConditionsHold(Read().Records, Entries.Values))
return CultCommitOutcome.Mismatch;
foreach (var entry in request.Upserts)
Entries[entry.Key.Value] = entry;
Write();
return CultCommitOutcome.Committed;
});
private void Write()
{
var snapshot = new CultPersistedStoreSnapshot
{
Records = Entries.Values.Select(entry => ToPersistedRecord(entry,
document => CultDocumentMessagePackSerialization.SerializeUntyped(document, document.GetType(), Registry))).ToArray(),
SchemaCatalog = Entries.Values.Select(entry => entry.Descriptor.ToCatalogEntry()).ToArray()
};
File.WriteAllBytes(_path, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot));
foreach (var entry in Entries.Values)
MarkWritten(entry);
MarkFlushSucceeded();
}
}
// Pauses a pull between reading the file and handing the records to the cache.
private sealed class PausingStore : SingleFileMessagePackBackingStore
{
@ -1130,9 +1055,9 @@ namespace GameCult.Caching.Tests
base.Push(entry);
}
protected override CultPersistedStoreSnapshot DeserializeSnapshot(byte[] data)
protected override (CultPersistedStoreSnapshot Snapshot, byte[][] CatalogBytes, byte[][] RecordBytes) ReadStore(byte[] data)
{
var snapshot = base.DeserializeSnapshot(data);
var snapshot = base.ReadStore(data);
if (Pause is { } pause)
{
Paused = true;
@ -1196,7 +1121,7 @@ namespace GameCult.Caching.Tests
public string Extra = string.Empty;
}
[CultDocument("tests.conditional_drift", "tests.conditional_drift.v2")]
[CultDocument("tests.conditional_drift", "tests.conditional_drift.v2", CompatibleSchemaIds = new[] { WideDriftSchemaId })]
[MessagePackObject(AllowPrivate = true)]
internal sealed class ConditionalDriftNarrow
{
@ -1208,7 +1133,7 @@ namespace GameCult.Caching.Tests
public int Value;
}
[CultDocument("tests.conditional_drift", "tests.conditional_drift.v3")]
[CultDocument("tests.conditional_drift", "tests.conditional_drift.v3", CompatibleSchemaIds = new[] { WideDriftSchemaId })]
[MessagePackObject(AllowPrivate = true)]
internal sealed class ConditionalDriftReused
{

170
tests/GameCult.Caching.Tests/CultDocumentRegistryTests.cs
View file

@ -1,9 +1,8 @@
using System;
using System.Linq;
using System.Reflection;
using System.Reflection.Emit;
using System.Threading.Tasks;
using NUnit.Framework;
using static GameCult.Caching.Tests.EmittedDocumentTypes;
namespace GameCult.Caching.Tests;
@ -13,8 +12,8 @@ public sealed class CultDocumentRegistryTests
[Test]
public void ForTypesContainsOnlyExplicitDocuments()
{
var requested = CreateDocumentType("explicit", "tests.registry.explicit", "v1");
var unrelated = CreateDocumentType("unrelated", "tests.registry.unrelated", "v1");
var requested = Emit("explicit", "tests.registry.explicit", "v1");
var unrelated = Emit("unrelated", "tests.registry.unrelated", "v1");
var registry = CultDocumentRegistry.ForTypes(new[] { requested, requested });
@ -26,7 +25,7 @@ public sealed class CultDocumentRegistryTests
public void IdenticalTypeRegistrationIsIdempotent()
{
var registry = new CultDocumentRegistry();
var type = CreateDocumentType("idempotent", "tests.registry.idempotent", "v1");
var type = Emit("idempotent", "tests.registry.idempotent", "v1");
var first = registry.GetRequired(type);
var second = registry.GetRequired(type);
@ -34,34 +33,12 @@ public sealed class CultDocumentRegistryTests
Assert.That(second, Is.SameAs(first));
}
[Test]
public void ExactWireCompatibleAliasRegistersByTypeWithOneCanonicalSchemaDescriptor()
{
var registry = new CultDocumentRegistry();
var firstType = CreateDocumentType("alias_owner", "tests.registry.alias", "v1");
var aliasType = CreateDocumentType("alias_claimant", "tests.registry.alias", "v1");
var canonical = registry.GetRequired(firstType);
var alias = registry.GetRequired(aliasType);
Assert.Multiple(() =>
{
Assert.That(alias.DocumentType, Is.EqualTo(aliasType));
Assert.That(alias.SchemaId, Is.EqualTo(canonical.SchemaId));
Assert.That(registry.GetRequired(aliasType), Is.SameAs(alias));
Assert.That(registry.GetRequiredBySchemaId(canonical.SchemaId), Is.SameAs(canonical));
Assert.That(
registry.AllDescriptors.Count(descriptor => descriptor.SchemaId == canonical.SchemaId),
Is.EqualTo(2));
});
}
[Test]
public void DifferentVersionsSharingSchemaNameRegisterWithoutReflectionOrderSelection()
{
var registry = new CultDocumentRegistry();
var firstType = CreateDocumentType("version_one", "tests.registry.versioned", "v1");
var secondType = CreateDocumentType("version_two", "tests.registry.versioned", "v2");
var firstType = Emit("version_one", "tests.registry.versioned", "v1");
var secondType = Emit("version_two", "tests.registry.versioned", "v2");
var first = registry.GetRequired(firstType);
var second = registry.GetRequired(secondType);
@ -86,33 +63,11 @@ public sealed class CultDocumentRegistryTests
.And.Message.Contains("v2"));
}
[Test]
public void IncompatibleTypesClaimingSameSchemaNameAndVersionFailBeforeMutation()
{
var registry = new CultDocumentRegistry();
var firstType = CreateDocumentType("layout_owner", "tests.registry.layout_collision", "v1");
var secondType = CreateDocumentType(
"layout_claimant",
"tests.registry.layout_collision",
"v1",
addStringField: true);
var first = registry.GetRequired(firstType);
Assert.That(
() => registry.GetRequired(secondType),
Throws.TypeOf<InvalidOperationException>()
.With.Message.Contains("schema name 'tests.registry.layout_collision' version 'v1'")
.And.Message.Contains(firstType.FullName)
.And.Message.Contains(secondType.FullName));
Assert.That(registry.GetRequiredBySchemaId(first.SchemaId), Is.SameAs(first));
Assert.That(registry.AllDescriptors.Any(descriptor => descriptor.DocumentType == secondType), Is.False);
}
[Test]
public void ConcurrentRefreshAndReadsObserveCompleteRegistrySnapshots()
{
var registry = new CultDocumentRegistry();
var type = CreateDocumentType("refresh_visible", "tests.registry.refresh_visible", "v1");
var type = Emit("refresh_visible", "tests.registry.refresh_visible", "v1");
var descriptor = registry.GetRequired(type);
Parallel.For(0, 20, _ =>
@ -123,33 +78,94 @@ public sealed class CultDocumentRegistryTests
});
}
private static Type CreateDocumentType(
string typeName,
string schemaName,
string schemaVersion,
bool addStringField = false)
// A second type for one schema is refused, typed, naming both and the claimed id, and the registry keeps what it had.
private static void AssertSecondClaimRefused(Type first, Type second)
{
var assembly = AssemblyBuilder.DefineDynamicAssembly(
new AssemblyName("GameCult.Caching.RegistryTests." + Guid.NewGuid().ToString("N")),
AssemblyBuilderAccess.Run);
var module = assembly.DefineDynamicModule("main");
var type = module.DefineType(
typeName,
TypeAttributes.Class | TypeAttributes.Public | TypeAttributes.Sealed);
var attributeConstructor = typeof(CultDocumentAttribute).GetConstructor(new[] { typeof(string), typeof(string) })!;
type.SetCustomAttribute(new CustomAttributeBuilder(
attributeConstructor,
new object[] { schemaName, schemaVersion }));
type.SetCustomAttribute(new CustomAttributeBuilder(
typeof(global::MessagePack.MessagePackObjectAttribute).GetConstructor(new[] { typeof(bool) })!,
new object[] { false }));
if (addStringField)
var schemaId = CultDocumentRegistry.ForTypes(new[] { second }).GetRequired(second).SchemaId;
var refusal = Assert.Throws<CultSchemaConflictException>(() => CultDocumentRegistry.ForTypes(new[] { first, second }))!;
Assert.Multiple(() =>
{
type.DefineField("Value", typeof(string), FieldAttributes.Public).SetCustomAttribute(new CustomAttributeBuilder(
typeof(global::MessagePack.KeyAttribute).GetConstructor(new[] { typeof(int) })!,
new object[] { 0 }));
}
Assert.That(refusal.SchemaId, Is.EqualTo(schemaId));
Assert.That(refusal.Message, Does.Contain(first.FullName).And.Contain(second.FullName));
});
return type.CreateType()!;
var registry = CultDocumentRegistry.ForTypes(new[] { first });
Assert.That(() => registry.GetRequired(second), Throws.TypeOf<CultSchemaConflictException>());
Assert.That(registry.AllDescriptors.Select(descriptor => descriptor.DocumentType), Is.EqualTo(new[] { first }));
}
private static CultSchemaCatalogEntry[] CatalogOf(params Type[] types) =>
types.Select(type => CultDocumentRegistry.ForTypes(new[] { type }).GetRequired(type).ToCatalogEntry()).ToArray();
// Owner beats lister: a type that owns an id and one that lists it register together in either order (the v1 class kept
// beside a v2 class declaring v1's id). A record under the id resolves to its owner; one under the lister's own id to the
// lister; without the owner registered, the id resolves to the lister.
[TestCase(false)]
[TestCase(true)]
public void AnOwnerAndATypeListingItsIdRegisterTogetherAndTheOwnerWins(bool reversed)
{
var owner = Emit("owns", "tests.registry.owns", "v1");
var ownerId = CultDocumentRegistry.ForTypes(new[] { owner }).GetRequired(owner).SchemaId;
var lister = Emit("lists", "tests.registry.owns", "v2", new[] { new Field("Extra", typeof(string), 0) }, new[] { ownerId });
var listerId = CultDocumentRegistry.ForTypes(new[] { lister }).GetRequired(lister).SchemaId;
var catalog = CatalogOf(owner, lister);
var registry = CultDocumentRegistry.ForTypes(reversed ? new[] { lister, owner } : new[] { owner, lister });
Assert.Multiple(() =>
{
Assert.That(registry.AllDescriptors.Select(descriptor => descriptor.DocumentType), Is.EquivalentTo(new[] { owner, lister }));
Assert.That(registry.ResolvePersistedSchema(ownerId, catalog).DocumentType, Is.EqualTo(owner));
Assert.That(registry.ResolvePersistedSchema(listerId, catalog).DocumentType, Is.EqualTo(lister));
Assert.That(CultDocumentRegistry.ForTypes(new[] { lister }).ResolvePersistedSchema(ownerId, catalog).DocumentType, Is.EqualTo(lister));
});
}
// Two types listing one id register together in either order. With an owner registered, a record under the id is the
// owner's; with none, the id names no single type, and a record under it is refused, typed, naming the listers.
[TestCase(false)]
[TestCase(true)]
public void TwoTypesListingOneIdRegisterAndARecordUnderItNeedsAnOwner(bool reversed)
{
var owner = Emit("shared_owner", "tests.registry.shared", "v1");
var sharedId = CultDocumentRegistry.ForTypes(new[] { owner }).GetRequired(owner).SchemaId;
var a = Emit("declares_a", "tests.registry.declares_a", "v1", compatibleSchemaIds: new[] { sharedId });
var b = Emit("declares_b", "tests.registry.declares_b", "v1", compatibleSchemaIds: new[] { sharedId });
var listers = reversed ? new[] { b, a } : new[] { a, b };
var catalog = CatalogOf(owner);
var ambiguous = CultDocumentRegistry.ForTypes(listers);
var refusal = Assert.Throws<CultSchemaConflictException>(() => ambiguous.ResolvePersistedSchema(sharedId, catalog))!;
Assert.Multiple(() =>
{
Assert.That(refusal.SchemaId, Is.EqualTo(sharedId));
Assert.That(refusal.SchemaNames, Is.EquivalentTo(new[] { "tests.registry.declares_a", "tests.registry.declares_b" }));
Assert.That(refusal.Message, Does.Contain(a.FullName).And.Contain(b.FullName));
});
Assert.That(CultDocumentRegistry.ForTypes(listers.Append(owner)).ResolvePersistedSchema(sharedId, catalog).DocumentType, Is.EqualTo(owner));
Assert.That(CultDocumentRegistry.ForTypes(listers.Prepend(owner)).ResolvePersistedSchema(sharedId, catalog).DocumentType, Is.EqualTo(owner));
}
// One type per schema, in every runtime: a second type of the same schema name and version is refused whichever registers
// first, whether it is identical to the first, declares different compatible ids, or has different members.
[TestCase("identical", false)]
[TestCase("identical", true)]
[TestCase("declaration", false)]
[TestCase("declaration", true)]
[TestCase("members", false)]
[TestCase("members", true)]
public void ASecondTypeForOneSchemaIsRefused(string difference, bool reversed)
{
var schemaName = $"tests.registry.claimed_twice_{difference}";
var first = Emit($"first_{difference}", schemaName, "v1");
var second = difference switch
{
"identical" => Emit($"second_{difference}", schemaName, "v1"),
"declaration" => Emit($"second_{difference}", schemaName, "v1", compatibleSchemaIds: new[] { "tests.registry.older" }),
_ => Emit($"second_{difference}", schemaName, "v1", new[] { new Field("Value", typeof(string), 0) }),
};
AssertSecondClaimRefused(reversed ? second : first, reversed ? first : second);
}
}

79
tests/GameCult.Caching.Tests/CultMathExprVariantTests.cs Normal file
View file

@ -0,0 +1,79 @@
#nullable enable
using System;
using System.IO;
using CultMath.Expr;
using GameCult.Caching;
using GameCult.Caching.MessagePack;
using MessagePack;
using NUnit.Framework;
namespace GameCult.Caching.Tests
{
// An ExprGraph is a plain value with no element-id lists. It survives a single-file cache, a variant inherits it,
// a variant may override it whole, and an edit to the base reaches the inheriting variant.
public class CultMathExprVariantTests
{
[CultDocument("tests.expr_variant", "tests.expr_variant.v1")]
[MessagePackObject(AllowPrivate = true)]
internal sealed class ExprDoc
{
[Key(0)] [CultName] public string Name = string.Empty;
[Key(1)] public ExprGraph Graph = default!;
[Key(2)] public float Scale;
}
private static ExprGraph Graph(float k)
{
var b = new ExprBuilder();
var x = b.Input("x");
var p = b.Param("k", k);
b.Output("o", b.Call("mul", x, p));
return b.Build();
}
private static byte[] Enc(ExprGraph g) => MessagePackSerializer.Serialize(g, CultDocumentMessagePackSerialization.Options);
[Test]
public void ExprGraphRoundTripsThroughACacheAndIsInheritedOrOverriddenByVariants()
{
var dir = Path.Combine(Path.GetTempPath(), "cultlib-exprvariant-" + Guid.NewGuid().ToString("N"));
Directory.CreateDirectory(dir);
try
{
var path = Path.Combine(dir, "expr.cc");
var registry = CultDocumentRegistry.ForTypes(new[] { typeof(ExprDoc) });
var baseKey = new CultRecordKey("base");
var scaledKey = new CultRecordKey("scaled");
var regraphedKey = new CultRecordKey("regraphed");
using (var cache = CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = registry }))
{
cache.Commit(b => b.Upsert(typeof(ExprDoc), new ExprDoc { Name = "base", Graph = Graph(1.5f), Scale = 2f }, baseKey));
cache.Commit(b => b.UpsertVariant(scaledKey, baseKey, new[] { cache.Override<ExprDoc>(nameof(ExprDoc.Name), "scaled"), cache.Override<ExprDoc>(nameof(ExprDoc.Scale), 3f) }));
cache.Commit(b => b.UpsertVariant(regraphedKey, baseKey, new[] { cache.Override<ExprDoc>(nameof(ExprDoc.Name), "regraphed"), cache.Override<ExprDoc>(nameof(ExprDoc.Graph), Graph(4f)) }));
}
using (var cache = CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = registry }))
{
var baseDoc = cache.Get<ExprDoc>(baseKey)!;
var scaled = cache.Get<ExprDoc>(scaledKey)!;
var regraphed = cache.Get<ExprDoc>(regraphedKey)!;
Assert.That(Enc(baseDoc.Graph), Is.EqualTo(Enc(Graph(1.5f))), "the base graph survives the file");
Assert.That(scaled.Scale, Is.EqualTo(3f));
Assert.That(Enc(scaled.Graph), Is.EqualTo(Enc(Graph(1.5f))), "a variant that does not override the graph inherits it");
Assert.That(Enc(regraphed.Graph), Is.EqualTo(Enc(Graph(4f))), "a variant that overrides the graph replaces it whole");
Assert.That(regraphed.Scale, Is.EqualTo(2f), "and keeps the rest of the base");
cache.Commit(b => b.Upsert(typeof(ExprDoc), new ExprDoc { Name = "base", Graph = Graph(9f), Scale = 2f }, baseKey));
Assert.That(Enc(cache.Get<ExprDoc>(scaledKey)!.Graph), Is.EqualTo(Enc(Graph(9f))), "an edit to the base reaches the inheriting variant");
Assert.That(Enc(cache.Get<ExprDoc>(regraphedKey)!.Graph), Is.EqualTo(Enc(Graph(4f))), "and not the overriding one");
}
}
finally
{
Directory.Delete(dir, recursive: true);
}
}
}
}

1087
tests/GameCult.Caching.Tests/ElementIdEngineTests.cs Normal file
View file

File diff suppressed because it is too large Load diff

Some files were not shown because too many files have changed in this diff Show more