diff --git a/.gitignore b/.gitignore index 1428a57..f5ad87f 100644 --- a/.gitignore +++ b/.gitignore @@ -10,7 +10,6 @@ state/scratch-runtime/ .weksa-runs/ .weksa-smoke/ .weksa/*.cc -*.cc.lock .weksa/*.json .weksa/*.log .weksa/*.pid @@ -21,4 +20,3 @@ dist/ build/ .venv/ venv/ -target/ diff --git a/.voidbot/state/weksa.cc b/.voidbot/state/weksa.cc index 7960180..be51ad6 100644 Binary files a/.voidbot/state/weksa.cc and b/.voidbot/state/weksa.cc differ diff --git a/AGENTS.md b/AGENTS.md index 55d488a..8befd5d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -46,18 +46,8 @@ The live state architecture is ported from EpiphanyAgent in reduced form: - `state/map.yaml` is canonical and slow-changing. - `state/scratch.md` is temporary and disposable. - `schemas/` explains state contracts in human language. -- `crates/weksa` owns executable typed state. +- `packages/weksa-state-model` owns executable typed state. Project truth does not belong in agent personality memory. If Weksa later grows role dossiers or heartbeat scheduling, those surfaces must remain separate from language-project truth. - -## Building And Verifying - -- `crates/weksa` depends on CultLib's `cultcache-rs` as a git dependency pinned by - `rev`, fetched from GitHub. With the default `CARGO_HOME` that fetch fails on a - Mind's host. Use the pre-fetched home: - `CARGO_HOME=/var/lib/mind-weksa-self/cargo-home cargo test --workspace`. -- `weksa packet proof ` admits the nibu proof packet under the key - `packet:6e6962752d303031` (hex of `nibu-001`), not `nibu-proof-001`. Use that key - with `weksa lower `. diff --git a/CLAUDE.md b/CLAUDE.md index 17a040b..640c1a1 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,3 +1,3 @@ -Repository doctrine for this project lives in AGENTS.md, shared with other coding agents. +Repository doctrine for this project lives in AGENTS.md, shared with Codex. @AGENTS.md diff --git a/Cargo.lock b/Cargo.lock deleted file mode 100644 index 38e73c8..0000000 --- a/Cargo.lock +++ /dev/null @@ -1,1009 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -version = 4 - -[[package]] -name = "android_system_properties" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc" -dependencies = [ - "libc", -] - -[[package]] -name = "anstream" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" -dependencies = [ - "anstyle", - "anstyle-parse", - "anstyle-query", - "anstyle-wincon", - "colorchoice", - "is_terminal_polyfill", - "utf8parse", -] - -[[package]] -name = "anstyle" -version = "1.0.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" - -[[package]] -name = "anstyle-parse" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" -dependencies = [ - "utf8parse", -] - -[[package]] -name = "anstyle-query" -version = "1.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" -dependencies = [ - "windows-sys 0.61.2", -] - -[[package]] -name = "anstyle-wincon" -version = "3.0.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" -dependencies = [ - "anstyle", - "once_cell_polyfill", - "windows-sys 0.61.2", -] - -[[package]] -name = "anyhow" -version = "1.0.104" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" - -[[package]] -name = "ar_archive_writer" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73cd58deff2140a0a8eae87e417bd01db68a33e148aa93d1e8cd837e55e312b6" -dependencies = [ - "object", -] - -[[package]] -name = "autocfg" -version = "1.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" - -[[package]] -name = "base64" -version = "0.23.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" - -[[package]] -name = "bitflags" -version = "2.13.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" - -[[package]] -name = "bumpalo" -version = "3.20.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" - -[[package]] -name = "bytes" -version = "1.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" - -[[package]] -name = "caseless" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b6fd507454086c8edfd769ca6ada439193cdb209c7681712ef6275cccbfe5d8" -dependencies = [ - "unicode-normalization", -] - -[[package]] -name = "cc" -version = "1.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86e055fa6e3e40ad6187ba25b6855eeb635d5271f8e0f0e1e9a5826b8d75bea0" -dependencies = [ - "find-msvc-tools", - "shlex", -] - -[[package]] -name = "cfg-if" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" - -[[package]] -name = "chrono" -version = "0.4.45" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" -dependencies = [ - "iana-time-zone", - "js-sys", - "num-traits", - "serde", - "wasm-bindgen", - "windows-link", -] - -[[package]] -name = "clap" -version = "4.6.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa8876b300ab35ba921adea3dfd70157a46249b33f95c9084ae5709785478946" -dependencies = [ - "clap_builder", - "clap_derive", -] - -[[package]] -name = "clap_builder" -version = "4.6.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec0797fb7aeb1406c84efac526901f7ec3ead2124f946b494e72879d4b54704d" -dependencies = [ - "anstream", - "anstyle", - "clap_lex", - "strsim", -] - -[[package]] -name = "clap_derive" -version = "4.6.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9c751b79415d4e559e3d1fcf128e09e720eb673a06d26cf6f392d37d75b66e0" -dependencies = [ - "heck", - "proc-macro2", - "quote", - "syn 3.0.9", -] - -[[package]] -name = "clap_lex" -version = "1.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c133bc6a41be0d194c306b5506d15e6feeea7b1d6604bd3f8310dfb2ca96486" - -[[package]] -name = "colorchoice" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" - -[[package]] -name = "core-foundation-sys" -version = "0.8.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" - -[[package]] -name = "cultcache-rs" -version = "0.3.1" -source = "git+https://github.com/GameCult/CultLib.git?rev=64b86e0a273a3936939357e9deb186b7bf3bc417#64b86e0a273a3936939357e9deb186b7bf3bc417" -dependencies = [ - "anyhow", - "chrono", - "cultcache-rs-derive", - "fs2", - "redb", - "rmp-serde", - "serde", - "serde_bytes", - "serde_json", - "serde_stacker", - "stacker", - "uuid", - "windows-sys 0.59.0", -] - -[[package]] -name = "cultcache-rs-derive" -version = "0.2.0" -source = "git+https://github.com/GameCult/CultLib.git?rev=64b86e0a273a3936939357e9deb186b7bf3bc417#64b86e0a273a3936939357e9deb186b7bf3bc417" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", -] - -[[package]] -name = "dyn-clone" -version = "1.0.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" - -[[package]] -name = "errno" -version = "0.3.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" -dependencies = [ - "libc", - "windows-sys 0.61.2", -] - -[[package]] -name = "fastrand" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" - -[[package]] -name = "find-msvc-tools" -version = "0.1.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c66915cc5391097de7a2b0b972e8505aa33c0df401fd3a6811dbdb32881ba48" - -[[package]] -name = "fs2" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213" -dependencies = [ - "libc", - "winapi", -] - -[[package]] -name = "futures-core" -version = "0.3.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" - -[[package]] -name = "futures-task" -version = "0.3.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" - -[[package]] -name = "futures-util" -version = "0.3.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" -dependencies = [ - "futures-core", - "futures-task", - "pin-project-lite", - "slab", -] - -[[package]] -name = "getrandom" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" -dependencies = [ - "cfg-if", - "libc", - "r-efi", -] - -[[package]] -name = "heck" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" - -[[package]] -name = "http" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" -dependencies = [ - "bytes", - "itoa", -] - -[[package]] -name = "httparse" -version = "1.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" - -[[package]] -name = "iana-time-zone" -version = "0.1.65" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" -dependencies = [ - "android_system_properties", - "core-foundation-sys", - "iana-time-zone-haiku", - "js-sys", - "log", - "wasm-bindgen", - "windows-core", -] - -[[package]] -name = "iana-time-zone-haiku" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" -dependencies = [ - "cc", -] - -[[package]] -name = "is_terminal_polyfill" -version = "1.70.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" - -[[package]] -name = "itoa" -version = "1.0.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" - -[[package]] -name = "js-sys" -version = "0.3.106" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7883d941dae510fb2d978fc3fe018c71c9e2892fd38854de3e8b92c2e5ad9cc5" -dependencies = [ - "cfg-if", - "futures-util", - "wasm-bindgen", -] - -[[package]] -name = "libc" -version = "0.2.191" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a" - -[[package]] -name = "linux-raw-sys" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" - -[[package]] -name = "log" -version = "0.4.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" - -[[package]] -name = "memchr" -version = "2.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" - -[[package]] -name = "num-traits" -version = "0.2.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" -dependencies = [ - "autocfg", -] - -[[package]] -name = "object" -version = "0.39.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e5a6c098c7a3b6547378093f5cc30bc54fd361ce711e05293a5cc589562739b" -dependencies = [ - "memchr", -] - -[[package]] -name = "once_cell" -version = "1.21.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" - -[[package]] -name = "once_cell_polyfill" -version = "1.70.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" - -[[package]] -name = "percent-encoding" -version = "2.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" - -[[package]] -name = "pin-project-lite" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" - -[[package]] -name = "proc-macro2" -version = "1.0.107" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "psm" -version = "0.1.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dcd034599e63b970727f70d79e02d62390a4a84f7c6b827c27c46d5ac3fa622" -dependencies = [ - "ar_archive_writer", - "cc", -] - -[[package]] -name = "quote" -version = "1.0.47" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" -dependencies = [ - "proc-macro2", -] - -[[package]] -name = "r-efi" -version = "6.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" - -[[package]] -name = "redb" -version = "4.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39178e0a867f9e3eaa6292a4ab1dd3613f2519c5b564341e1e7b29b270736cc4" -dependencies = [ - "libc", -] - -[[package]] -name = "ref-cast" -version = "1.0.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e440fb4e4b4147295338efb76001ab9e4efc0e5839df2c47fc5ac2381d365c3" -dependencies = [ - "ref-cast-impl", -] - -[[package]] -name = "ref-cast-impl" -version = "1.0.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92ecd8964f8453721699a1ed72037b0db49ce2f5a5138486ee89bed6f67cdf3a" -dependencies = [ - "proc-macro2", - "quote", - "syn 3.0.9", -] - -[[package]] -name = "rmp" -version = "0.8.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ba8be72d372b2c9b35542551678538b562e7cf86c3315773cae48dfbfe7790c" -dependencies = [ - "num-traits", -] - -[[package]] -name = "rmp-serde" -version = "1.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72f81bee8c8ef9b577d1681a70ebbc962c232461e397b22c208c43c04b67a155" -dependencies = [ - "rmp", - "serde", -] - -[[package]] -name = "rustix" -version = "1.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5e76742e5294153d4dde1207dbec5bdd81eef6fabdc0d2118babadabfd858aa" -dependencies = [ - "bitflags", - "errno", - "libc", - "linux-raw-sys", - "windows-sys 0.61.2", -] - -[[package]] -name = "rustversion" -version = "1.0.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" - -[[package]] -name = "schemars" -version = "1.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab508826f74a77ca9d5aba6ff19b522583ee3eaf28a19384ff3d0e5835fadf6e" -dependencies = [ - "dyn-clone", - "ref-cast", - "schemars_derive", - "serde", - "serde_json", -] - -[[package]] -name = "schemars_derive" -version = "1.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1c3a92094fa7d61aa124645844facb6b554dfc797136d0f5fd1f890e2bffc69" -dependencies = [ - "proc-macro2", - "quote", - "serde_derive_internals", - "syn 3.0.9", -] - -[[package]] -name = "serde" -version = "1.0.229" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" -dependencies = [ - "serde_core", - "serde_derive", -] - -[[package]] -name = "serde_bytes" -version = "0.11.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5d440709e79d88e51ac01c4b72fc6cb7314017bb7da9eeff678aa94c10e3ea8" -dependencies = [ - "serde", - "serde_core", -] - -[[package]] -name = "serde_core" -version = "1.0.229" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" -dependencies = [ - "serde_derive", -] - -[[package]] -name = "serde_derive" -version = "1.0.229" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" -dependencies = [ - "proc-macro2", - "quote", - "syn 3.0.9", -] - -[[package]] -name = "serde_derive_internals" -version = "0.30.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f852137cce035d6a4df67ccce505ff6b3e9fd3a10e3e52b24dc71e650bb1a9bd" -dependencies = [ - "proc-macro2", - "quote", - "syn 3.0.9", -] - -[[package]] -name = "serde_json" -version = "1.0.154" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6" -dependencies = [ - "itoa", - "memchr", - "serde", - "serde_core", - "zmij", -] - -[[package]] -name = "serde_stacker" -version = "0.1.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4936375d50c4be7eff22293a9344f8e46f323ed2b3c243e52f89138d9bb0f4a" -dependencies = [ - "serde", - "serde_core", - "stacker", -] - -[[package]] -name = "shlex" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" - -[[package]] -name = "slab" -version = "0.4.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" - -[[package]] -name = "stacker" -version = "0.1.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "707f49d46706bacf8a2b00d51dace3f9de527c13eec3778f570c411f89e69967" -dependencies = [ - "cc", - "cfg-if", - "libc", - "psm", - "windows-sys 0.61.2", -] - -[[package]] -name = "strsim" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" - -[[package]] -name = "syn" -version = "2.0.119" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "syn" -version = "3.0.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "tempfile" -version = "3.27.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" -dependencies = [ - "fastrand", - "getrandom", - "once_cell", - "rustix", - "windows-sys 0.61.2", -] - -[[package]] -name = "tinyvec" -version = "1.13.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd3ca314f692efd6c868f8408f53fe444634a845f96c028b97d35f6a1f79f0ee" - -[[package]] -name = "unicode-ident" -version = "1.0.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" - -[[package]] -name = "unicode-normalization" -version = "0.1.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75cd3fb03d5288b3ba9a1ee103ac7e9cbb59f0b03d64f8ddfcc04dd42758a85d" -dependencies = [ - "tinyvec", -] - -[[package]] -name = "unicode-segmentation" -version = "1.13.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" - -[[package]] -name = "ureq" -version = "3.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cde93ff05758a36a851a398a772b0b79a82fbe3c186a46f2696baaeaf84c966b" -dependencies = [ - "base64", - "log", - "percent-encoding", - "ureq-proto", - "utf8-zero", -] - -[[package]] -name = "ureq-proto" -version = "0.6.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "841583fc36089d7d59419568ae157f43a5a03a2e5a81ba87c77d1596ef46e922" -dependencies = [ - "base64", - "http", - "httparse", - "log", -] - -[[package]] -name = "utf8-zero" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8c0a043c9540bae7c578c88f91dda8bd82e59ae27c21baca69c8b191aaf5a6e" - -[[package]] -name = "utf8parse" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" - -[[package]] -name = "uuid" -version = "1.29.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a0e2d11620c50695fe12bbbfde96b3477fe606e7284c22d505c12ce51214514" -dependencies = [ - "getrandom", - "js-sys", - "wasm-bindgen", -] - -[[package]] -name = "wasm-bindgen" -version = "0.2.129" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9bb54f33acc68fd454578d9820b0bde1a1a3d17aa17bb7b6595806d02886d409" -dependencies = [ - "cfg-if", - "once_cell", - "rustversion", - "wasm-bindgen-macro", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-macro" -version = "0.2.129" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e29d0c35b16e224a7eeb5cd2d25e3e1968fbd65604117b44d3b789d00ee8535" -dependencies = [ - "quote", - "wasm-bindgen-macro-support", -] - -[[package]] -name = "wasm-bindgen-macro-support" -version = "0.2.129" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f501a8bc3719dba86ef8ae4728879c08001bea749eb1333ac5b91e040e2a6b7" -dependencies = [ - "bumpalo", - "proc-macro2", - "quote", - "syn 3.0.9", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-shared" -version = "0.2.129" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23f0c9c52aa7cd7d77769a4cfe2a9adb1b331f489a41d912ce14513d5ab995c6" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "weksa" -version = "0.1.0" -dependencies = [ - "anyhow", - "caseless", - "clap", - "cultcache-rs", - "schemars", - "serde", - "serde_json", - "tempfile", - "unicode-normalization", - "unicode-segmentation", - "ureq", -] - -[[package]] -name = "winapi" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" -dependencies = [ - "winapi-i686-pc-windows-gnu", - "winapi-x86_64-pc-windows-gnu", -] - -[[package]] -name = "winapi-i686-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" - -[[package]] -name = "winapi-x86_64-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" - -[[package]] -name = "windows-core" -version = "0.62.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" -dependencies = [ - "windows-implement", - "windows-interface", - "windows-link", - "windows-result", - "windows-strings", -] - -[[package]] -name = "windows-implement" -version = "0.60.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", -] - -[[package]] -name = "windows-interface" -version = "0.59.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", -] - -[[package]] -name = "windows-link" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" - -[[package]] -name = "windows-result" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" -dependencies = [ - "windows-link", -] - -[[package]] -name = "windows-strings" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" -dependencies = [ - "windows-link", -] - -[[package]] -name = "windows-sys" -version = "0.59.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" -dependencies = [ - "windows-targets", -] - -[[package]] -name = "windows-sys" -version = "0.61.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" -dependencies = [ - "windows-link", -] - -[[package]] -name = "windows-targets" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" -dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_gnullvm", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", -] - -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" - -[[package]] -name = "windows_i686_gnu" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" - -[[package]] -name = "windows_i686_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" - -[[package]] -name = "windows_i686_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" - -[[package]] -name = "zmij" -version = "1.0.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/Cargo.toml b/Cargo.toml deleted file mode 100644 index 956fefd..0000000 --- a/Cargo.toml +++ /dev/null @@ -1,3 +0,0 @@ -[workspace] -resolver = "3" -members = ["crates/weksa"] diff --git a/README.md b/README.md index d327e48..a2a51bf 100644 --- a/README.md +++ b/README.md @@ -1,40 +1,12 @@ # Weksa Weksa is a procedural language engine for alien grammars grounded in concept, -culture, and constraint, and the planned lowerer that turns an agent's -meaning packet into the reader's display language. +culture, and constraint. -**What is valuable here.** The interlingua standard and the lowering design: a -meaning packet owns what is meant, a target-language profile owns register, -address and politeness, and a lowerer owns surface text and leaves a trace. - -**The problem it solves.** An agent's outward speech today is one English -string the model chose, so meaning, register and voice are fused and nobody can -inspect or re-target it. Packets separate them, so the same meaning can be -lowered into English, other languages and conlangs, and the loss can be -measured. - -**Entry points.** - -- [Interlingua Standard](./docs/interlingua-standard.md): the packet contract. -- `crates/weksa`: the packet, inventory and corpus types, `validate`, the - derived split, the scorer (`score`: chrF, meaning cosine through an Ollama - embedder, whole output against whole reference with no mask, leak diagnostics) and the `weksa` CLI - (`weksa validate`, `corpus`, `entity`, `inventory`, `split`, `schema`); - published schemas in `schemas/cultnet/`. -- `data/corpus/corpus.cc`, `data/entity/entity.cc` and `data/interlingua/inventory.cc`: the English - corpus, its entities and the inventory, read with `weksa corpus dump`, - `weksa entity dump` and `weksa inventory dump`. -- [Lowering target](./docs/targets/interlingua-lowering.md) and - [its map](./docs/targets/interlingua-lowering-map.md): the design, scoring - and cut sequence. -- `data/`: the cultural ontology and target-language profile data. -- `settings/`: setting-specific proving grounds for language projects. - -**Live versus planned.** Live: docs, fixtures, examples, ontology data and the -packet crate `crates/weksa`, including the scoring library. Planned: lowering, -the `weksa score` subcommand, and the golf loop that scores lowered English against an expressive corpus. No -daemon runs; the earlier Node daemon is parked at tag `parked/weksa-node-body`. +The project starts from a blunt rule: English words are not the source of truth. +Meaning is decomposed into an ontology, passed through a worldview, constrained +by grammar, shaped by phonology, distorted by history, and only then rendered as +surface text. ## Ownership Model @@ -68,10 +40,9 @@ for the LLM-free hot path. See [Interlingua Standard](./docs/interlingua-standard.md) for the shared meaning-packet contract Weksa clients should target. -See [Verse Service Contract](./docs/verse-service-contract.md) for the planned -service contract: Weksa would own typed conversational intent, pronunciation -plans, utterance handoffs, and the typed CultMesh state that makes those -documents inspectable. No daemon runs yet. +See [Verse Service Contract](./docs/verse-service-contract.md) for the daemon +contract: Weksa owns typed conversational intent, pronunciation plans, utterance +handoffs, and the typed CultMesh state that makes those documents inspectable. Eve is deprecated and Thing 2 is its successor; the operator Eve surface is quarantined (see the contract). See [Multilingual Utterance Lowering](./docs/multilingual-utterance-lowering.md) for the shared lowering target: interlingua projected through speaker state into @@ -92,12 +63,11 @@ AquaSynth or expanding pronunciation coverage. ## Implementation Status -The persistent state spine is live: map, scratch, schemas, docs, examples and -data. Packet types, validation and the corpus and inventory stores are live in -`crates/weksa`, and so is the scoring library. Lowering and the service are planned (see the cut -sequence in [the lowering map](./docs/targets/interlingua-lowering-map.md)). -Writing and agent state will enter Weksa as intent pressure, and typed utterance -documents will leave for AquaSynth. +The first committed artifact is the persistent state spine: map, scratch, +schemas, docs, examples, and a read-only provider advertisement fixture. The +daemon runtime still needs implementation, but its authority is no longer +ambiguous: writing and agent state enter Weksa as intent pressure, and typed +utterance documents leave for AquaSynth. [Eusocial Interbeing / Zyphos](./settings/eusocial-interbeing/README.md) is the current higher-priority setting target. [Rust to Dust](./settings/rust-to-dust/README.md) diff --git a/crates/weksa/Cargo.toml b/crates/weksa/Cargo.toml deleted file mode 100644 index 5b9c407..0000000 --- a/crates/weksa/Cargo.toml +++ /dev/null @@ -1,28 +0,0 @@ -[package] -name = "weksa" -version = "0.1.0" -edition = "2024" -publish = false - -[lib] -name = "weksa" -path = "src/lib.rs" - -[[bin]] -name = "weksa" -path = "src/main.rs" - -[dependencies] -anyhow = "1" -caseless = "0.2" -clap = { version = "4", features = ["derive"] } -cultcache-rs = { git = "https://github.com/GameCult/CultLib.git", rev = "64b86e0a273a3936939357e9deb186b7bf3bc417", package = "cultcache-rs" } -schemars = "1" -serde = { version = "1", features = ["derive"] } -serde_json = "1" -unicode-normalization = "0.1" -unicode-segmentation = "1" -ureq = { version = "3", default-features = false } - -[dev-dependencies] -tempfile = "3" diff --git a/crates/weksa/src/corpus.rs b/crates/weksa/src/corpus.rs deleted file mode 100644 index d67cc60..0000000 --- a/crates/weksa/src/corpus.rs +++ /dev/null @@ -1,142 +0,0 @@ -//! The English target corpus. An item's text is immutable; a correction is a new -//! item that supersedes the old, and an item is withdrawn, never deleted. The -//! source enum has no variant for a source whose terms forbid training. - -use cultcache_rs::DatabaseEntry; -use schemars::JsonSchema; -use serde::{Deserialize, Serialize}; - -use crate::form::{FormError, is_slug, string_form}; -use crate::packet::EntityId; - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] -pub struct Turn { - pub speaker: EntityId, - pub text: String, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] -pub enum Source { - /// A line from a cast member's hand-authored fixture, with the human who - /// wrote it. A fixture that records no author is `ModelWritten` with family - /// `Unrecorded`, never this (`corpus-labels-tell-truth`). - CastAuthored { - persona: String, - fixture: String, - author: String, - }, - OperatorWritten, - OperatorApproved { - note: String, - }, - PublicDomain { - work: String, - author: String, - edition_url: String, - }, - /// Labelled model output; never in the test split, never judged by its own family. - /// `family` is `Unrecorded` when nothing records one. `generator` is the - /// string the fixture or run wrote, kept verbatim. - ModelWritten { - family: ModelFamily, - generator: Option, - prompt_digest: Option, - }, -} - -/// The family of the model that wrote a line. -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] -pub enum ModelFamily { - Anthropic, - OpenAi, - Google, - Alibaba, - Meta, - Mistral, - /// Nothing records which family wrote the line. - Unrecorded, -} - -/// Key: `item::`. -#[derive(Clone, Debug, PartialEq, Eq, JsonSchema, DatabaseEntry)] -#[cultcache(type = "weksa.corpus_item")] -pub struct CorpusItem { - #[cultcache(key = 0)] - pub id: String, - #[cultcache(key = 1)] - pub text: String, - #[cultcache(key = 2)] - pub language: String, - #[cultcache(key = 3)] - pub speaker: EntityId, - /// The unit the split hashes: a scene, a fixture, a work. - #[cultcache(key = 4)] - pub group: String, - #[cultcache(key = 5)] - pub prior_turns: Vec, - #[cultcache(key = 6)] - pub source: Source, - #[cultcache(key = 7)] - pub admitted_on: String, - /// Key of the item this one corrects. - #[cultcache(key = 8)] - pub supersedes: Option, - #[cultcache(key = 9)] - pub withdrawn: bool, - /// Which entities the text names, and where. Fixed with the text at intake. - #[cultcache(key = 10)] - pub names: Vec, -} - -/// A byte range `[start, end)` of an item's text that names an entity. The -/// store admits it only when both ends sit on extended grapheme cluster -/// boundaries (UAX #29) of the NFC text and the range's bytes equal one of the -/// entity's forms in the item's language. -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] -pub struct NameSpan { - pub entity: EntityId, - pub start: u32, - pub end: u32, -} - -/// The key of a corpus item, `item::` as `key_slug` reads it. A packet -/// names the item it was minted for by this type, so the key cannot be free text. -#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] -#[serde(try_from = "String", into = "String")] -pub struct CorpusItemId(String); - -impl CorpusItemId { - pub fn parse(s: &str) -> Result { - key_slug(s) - .map(|_| Self(s.to_string())) - .ok_or_else(|| FormError::new("CorpusItemId", s)) - } - - pub fn as_str(&self) -> &str { - &self.0 - } -} - -impl std::fmt::Display for CorpusItemId { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.write_str(&self.0) - } -} - -string_form!(CorpusItemId, { "type": "string", "pattern": "^item:[a-z0-9]([a-z0-9-]{0,46}[a-z0-9])?:[1-9][0-9]*$" }); - -/// `item::`. -pub fn item_key(slug: &str, n: u32) -> String { - format!("item:{slug}:{n}") -} - -/// The slug of an item key, if it is canonical `item::`: a slug of -/// `[a-z0-9]` and inner `-` (at most 48 bytes, no edge `-`), and `n` a decimal -/// of at least 1 without sign or leading zero that fits `u32`. -pub fn key_slug(key: &str) -> Option<&str> { - let rest = key.strip_prefix("item:")?; - let (slug, n) = rest.rsplit_once(':')?; - let canonical_n = - n.bytes().all(|b| b.is_ascii_digit()) && !n.starts_with('0') && n.parse::().is_ok(); - (canonical_n && is_slug(slug)).then_some(slug) -} diff --git a/crates/weksa/src/entity.rs b/crates/weksa/src/entity.rs deleted file mode 100644 index 33d22e1..0000000 --- a/crates/weksa/src/entity.rs +++ /dev/null @@ -1,53 +0,0 @@ -//! The entity store's documents. An entity is a named thing a packet may cite by -//! id. Its forms per language are data, citation form first, then inflected or -//! particle-bound forms; no rule here reads what a form means, only its shape. - -use cultcache_rs::DatabaseEntry; -use schemars::JsonSchema; -use serde::{Deserialize, Serialize}; - -use crate::packet::EntityId; - -/// An entity's forms in one language, as the corpus item language tag spells it. -/// The citation form comes first. -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] -pub struct LanguageForms { - pub language: String, - /// 1 to 64 bytes each, no control or invisible format character, no edge - /// whitespace, none repeated. - pub forms: Vec, -} - -/// Where an entity was minted. -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] -pub enum EntityOrigin { - /// From a cast member's fixture: the persona and the fixture path the - /// entity came from. - Cast { persona: String, fixture: String }, - /// From a public-domain work. - PublicDomain { work: String }, - /// Minted live by the named persona before it names the entity. - Live { persona: String }, -} - -/// Key: the entity id. Forms only grow: see `EntityStore::put`. -#[derive(Clone, Debug, PartialEq, Eq, JsonSchema, DatabaseEntry)] -#[cultcache(type = "weksa.entity")] -pub struct Entity { - #[cultcache(key = 0)] - pub id: EntityId, - #[cultcache(key = 1)] - pub forms: Vec, - #[cultcache(key = 2)] - pub origin: EntityOrigin, -} - -impl Entity { - /// The forms stored for `language`, if any. - pub fn forms_in(&self, language: &str) -> Option<&[String]> { - self.forms - .iter() - .find(|f| f.language == language) - .map(|f| f.forms.as_slice()) - } -} diff --git a/crates/weksa/src/form.rs b/crates/weksa/src/form.rs deleted file mode 100644 index 9de8eb8..0000000 --- a/crates/weksa/src/form.rs +++ /dev/null @@ -1,199 +0,0 @@ -//! Form types. A form lives in a newtype whose only constructor is a fallible -//! `parse`; deserializing goes through the same parse, so a malformed value can -//! neither be built in Rust nor read back from a store. - -use std::fmt; - -/// A value that is not of the form its type names. -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct FormError { - pub form: &'static str, - pub got: String, -} - -impl FormError { - pub fn new(form: &'static str, got: impl fmt::Display) -> Self { - Self { - form, - got: got.to_string(), - } - } -} - -impl fmt::Display for FormError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{:?} is not a {}", self.got, self.form) - } -} - -impl std::error::Error for FormError {} - -/// Wires a string form type (with `parse` and `Display`) to FromStr, serde's -/// `try_from`/`into` String and a published schema. -macro_rules! string_form { - ($t:ident, $schema:tt) => { - impl std::str::FromStr for $t { - type Err = $crate::form::FormError; - fn from_str(s: &str) -> Result { - Self::parse(s) - } - } - impl TryFrom for $t { - type Error = $crate::form::FormError; - fn try_from(s: String) -> Result { - Self::parse(&s) - } - } - impl From<$t> for String { - fn from(v: $t) -> String { - v.to_string() - } - } - impl schemars::JsonSchema for $t { - fn schema_name() -> std::borrow::Cow<'static, str> { - stringify!($t).into() - } - fn json_schema(_: &mut schemars::SchemaGenerator) -> schemars::Schema { - schemars::json_schema!($schema) - } - } - }; -} -pub(crate) use string_form; - -/// A BCP 47-form language tag: 1 to 35 bytes of ASCII alphanumerics in -/// hyphen-separated subtags of 1 to 8 bytes; the first is 2 to 8 letters, or the -/// single letter `x` of a private-use tag. Form only: no registry is consulted. -pub(crate) fn is_language_tag(s: &str) -> bool { - if s.len() > 35 { - return false; - } - let mut subtags = s.split('-'); - let first = subtags.next().unwrap_or(""); - let first_ok = first.bytes().all(|b| b.is_ascii_alphabetic()) - && ((2..=8).contains(&first.len()) || first == "x"); - first_ok - && subtags - .all(|t| (1..=8).contains(&t.len()) && t.bytes().all(|b| b.is_ascii_alphanumeric())) -} - -/// A slug: `[a-z0-9]` and inner `-`, 1 to 48 bytes, no edge `-`. -pub(crate) fn is_slug(s: &str) -> bool { - let b = s.as_bytes(); - (1..=48).contains(&b.len()) - && b.iter() - .all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || *c == b'-') - && b[0] != b'-' - && b[b.len() - 1] != b'-' -} - -/// A General_Category Cf code point, or the line and paragraph separators -/// U+2028 and U+2029, other than the joiners ZWNJ U+200C and ZWJ U+200D. The -/// ranges are Unicode 15.1 Cf; `tests/format_refused.txt` is the full list from -/// Python's `unicodedata` that the table is swept against. -pub(crate) fn is_format(c: char) -> bool { - matches!( - c, - '\u{ad}' - | '\u{600}'..='\u{605}' - | '\u{61c}' - | '\u{6dd}' - | '\u{70f}' - | '\u{890}'..='\u{891}' - | '\u{8e2}' - | '\u{180e}' - | '\u{200b}' - | '\u{200e}'..='\u{200f}' - | '\u{2028}'..='\u{202e}' - | '\u{2060}'..='\u{2064}' - | '\u{2066}'..='\u{206f}' - | '\u{feff}' - | '\u{fff9}'..='\u{fffb}' - | '\u{110bd}' - | '\u{110cd}' - | '\u{13430}'..='\u{1343f}' - | '\u{1bca0}'..='\u{1bca3}' - | '\u{1d173}'..='\u{1d17a}' - | '\u{e0001}' - | '\u{e0020}'..='\u{e007f}' - ) -} - -/// ZWNJ and ZWJ: Cf, but scripts and emoji sequences need them inside a form. -pub(crate) fn is_joiner(c: char) -> bool { - matches!(c, '\u{200c}' | '\u{200d}') -} - -/// A character a name or form may not hold: every `is_format` code point, and -/// the invisible fillers that are Default_Ignorable without being Cf (Hangul -/// U+115F, U+1160, U+3164, U+FFA0 and Khmer U+17B4, U+17B5). Joiners, -/// variation selectors, CGJ and the Mongolian selectors are not refused. -pub(crate) fn is_refused_format(c: char) -> bool { - is_format(c) - || matches!( - c, - '\u{115f}' | '\u{1160}' | '\u{17b4}' | '\u{17b5}' | '\u{3164}' | '\u{ffa0}' - ) -} - -/// A joiner, a variation selector (U+FE00..U+FE0F, U+E0100..U+E01EF), CGJ -/// U+034F or a Mongolian free variation selector (U+180B..U+180D, U+180F): it -/// shapes a neighbour and shows nothing alone. -pub(crate) fn is_joining_only(c: char) -> bool { - is_joiner(c) - || matches!( - c, - '\u{34f}' - | '\u{180b}'..='\u{180d}' - | '\u{180f}' - | '\u{fe00}'..='\u{fe0f}' - | '\u{e0100}'..='\u{e01ef}' - ) -} - -/// An attestation string (author, persona, fixture, note, work, url, generator, -/// digest): at most 200 bytes, at least one alphanumeric, no control, format or -/// joiner character. Whitespace between words is fine; it attests something. -pub(crate) fn is_attestation(s: &str) -> bool { - s.len() <= 200 - && s.chars().any(char::is_alphanumeric) - && !s - .chars() - .any(|c| c.is_control() || is_refused_format(c) || is_joiner(c)) -} - -#[cfg(test)] -mod tests { - use super::*; - - /// The rule against every code point: `is_format` is exactly the listed - /// Cf-minus-joiners-plus-U+2028/9 set, so each range has both of its - /// edges and the gaps between ranges pinned. - #[test] - fn format_rule_matches_the_unicode_list_at_every_code_point() { - let listed: std::collections::BTreeSet = include_str!("../tests/format_refused.txt") - .lines() - .filter(|l| !l.starts_with('#')) - .map(|l| u32::from_str_radix(l, 16).unwrap()) - .collect(); - for code in 0..=0x10ffff_u32 { - if let Some(c) = char::from_u32(code) { - assert_eq!(is_format(c), listed.contains(&code), "U+{code:X}"); - } - } - assert!(!is_format('\u{200c}') && !is_format('\u{200d}')); - } - - #[test] - fn an_attestation_is_bounded_readable_text() { - assert!(is_attestation("a")); - assert!(is_attestation(&"x".repeat(200))); - assert!(!is_attestation(&"x".repeat(201))); - assert!(is_attestation("two words, 2 more")); - for bad in [ - "", " ", "\t", "\u{200b}", "\u{200d}", "\0", "--", "a\nb", "a\u{ad}b", - ] { - assert!(!is_attestation(bad), "{bad:?}"); - } - } -} diff --git a/crates/weksa/src/inventory.rs b/crates/weksa/src/inventory.rs deleted file mode 100644 index 22538dd..0000000 --- a/crates/weksa/src/inventory.rs +++ /dev/null @@ -1,510 +0,0 @@ -//! The inventory: the ids packets name, as CultCache documents under appended -//! revisions. An entry is never edited in place; a changed meaning is a retire -//! plus an add, and a retired id still resolves for packets of earlier revisions. - -use std::collections::BTreeMap; -use std::fmt; -use std::num::NonZeroU32; - -use cultcache_rs::DatabaseEntry; -use schemars::JsonSchema; -use serde::{Deserialize, Serialize}; - -use crate::form::{FormError, string_form}; - -#[derive( - Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize, JsonSchema, -)] -pub enum Family { - Concept, - Frame, - Role, - Relation, - SpeechAct, - Feature, - ReferentKind, - /// A game- or loop-declared packet field; the entry carries its value kind. - Extension, -} - -impl Family { - pub const ALL: [Family; 8] = [ - Family::Concept, - Family::Frame, - Family::Role, - Family::Relation, - Family::SpeechAct, - Family::Feature, - Family::ReferentKind, - Family::Extension, - ]; - - pub fn prefix(self) -> &'static str { - match self { - Family::Concept => "concept", - Family::Frame => "frame", - Family::Role => "role", - Family::Relation => "relation", - Family::SpeechAct => "speech_act", - Family::Feature => "feature", - Family::ReferentKind => "referent_kind", - Family::Extension => "extension", - } - } - - /// The id of the `n`th entry of this family. - pub fn id(self, n: NonZeroU32) -> InvId { - InvId { family: self, n } - } -} - -/// Id of an inventory entry: `:`, `n` a canonical decimal -/// (no sign, no leading zero) of at least 1. -#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] -#[serde(try_from = "String", into = "String")] -pub struct InvId { - family: Family, - n: NonZeroU32, -} - -impl InvId { - pub fn parse(s: &str) -> Result { - let bad = || FormError::new("InvId", s); - let (prefix, n) = s.split_once(':').ok_or_else(bad)?; - let family = Family::ALL - .into_iter() - .find(|f| f.prefix() == prefix) - .ok_or_else(bad)?; - if n.starts_with('0') || !n.bytes().all(|b| b.is_ascii_digit()) { - return Err(bad()); - } - let n = n.parse::().map_err(|_| bad())?; - Ok(Self { family, n }) - } - - pub fn family(self) -> Family { - self.family - } -} - -impl fmt::Display for InvId { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}:{}", self.family.prefix(), self.n) - } -} - -string_form!(InvId, { "type": "string", "pattern": "^(concept|frame|role|relation|speech_act|feature|referent_kind|extension):[1-9][0-9]*$" }); - -/// A value label of an options feature: `^[a-z][a-z0-9_]{0,23}$`. -#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] -#[serde(try_from = "String", into = "String")] -pub struct OptionLabel(String); - -impl OptionLabel { - pub fn parse(s: &str) -> Result { - let mut bytes = s.bytes(); - let ok = s.len() <= 24 - && bytes.next().is_some_and(|b| b.is_ascii_lowercase()) - && bytes.all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'_'); - if ok { - Ok(Self(s.to_string())) - } else { - Err(FormError::new("OptionLabel", s)) - } - } - - pub fn as_str(&self) -> &str { - &self.0 - } -} - -impl fmt::Display for OptionLabel { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str(&self.0) - } -} - -string_form!(OptionLabel, { "type": "string", "pattern": "^[a-z][a-z0-9_]{0,23}$" }); - -/// 1 to 16 distinct labels. -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(try_from = "Vec", into = "Vec")] -pub struct OptionSet(Vec); - -impl OptionSet { - pub const MAX: usize = 16; - - pub fn new(labels: Vec) -> Result { - let mut distinct: Vec<&OptionLabel> = labels.iter().collect(); - distinct.sort(); - distinct.dedup(); - if (1..=Self::MAX).contains(&labels.len()) && distinct.len() == labels.len() { - Ok(Self(labels)) - } else { - Err(FormError::new( - "OptionSet", - format!("{} labels", labels.len()), - )) - } - } - - pub fn contains(&self, label: &OptionLabel) -> bool { - self.0.contains(label) - } -} - -impl TryFrom> for OptionSet { - type Error = FormError; - fn try_from(v: Vec) -> Result { - Self::new(v) - } -} - -impl From for Vec { - fn from(s: OptionSet) -> Self { - s.0 - } -} - -impl JsonSchema for OptionSet { - fn schema_name() -> std::borrow::Cow<'static, str> { - "OptionSet".into() - } - fn json_schema(g: &mut schemars::SchemaGenerator) -> schemars::Schema { - schemars::json_schema!({ - "type": "array", - "items": g.subschema_for::(), - "minItems": 1, - "maxItems": 16, - "uniqueItems": true - }) - } -} - -/// Where a feature may sit in a packet. -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] -pub enum Slot { - Discourse, - Referent, - Predication, - Relation, - Stance, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] -pub enum ValueKind { - Grade, - Options(OptionSet), - Flag, -} - -/// Key: the entry's id, `:`. -#[derive(Clone, Debug, PartialEq, Eq, JsonSchema, DatabaseEntry)] -#[cultcache(type = "weksa.inventory_entry")] -pub struct InventoryEntry { - #[cultcache(key = 0)] - pub id: InvId, - /// 1 to 120 bytes, no control character (checked at admission). - #[cultcache(key = 1)] - pub gloss: String, - /// Present exactly on features and extensions. - #[cultcache(key = 2)] - pub value_kind: Option, - /// Non-empty exactly on features. - #[cultcache(key = 3)] - pub applies_to: Vec, - #[cultcache(key = 4)] - pub introduced_rev: u32, - #[cultcache(key = 5)] - pub retired_rev: Option, -} - -impl InventoryEntry { - pub fn family(&self) -> Family { - self.id.family() - } - - /// Whether the entry's shape matches its family: features carry a value kind - /// and slots, extensions a value kind and no slots, nothing else either. - pub fn well_formed(&self) -> bool { - match self.family() { - Family::Feature => self.value_kind.is_some() && !self.applies_to.is_empty(), - Family::Extension => self.value_kind.is_some() && self.applies_to.is_empty(), - _ => self.value_kind.is_none() && self.applies_to.is_empty(), - } - } - - /// Whether a packet of inventory revision `rev` may name this entry. - pub fn live_at(&self, rev: u32) -> bool { - self.introduced_rev <= rev && self.retired_rev.is_none_or(|r| r > rev) - } -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] -pub enum Change { - Add(InvId), - Retire(InvId), -} - -/// Key: `rev:`. -#[derive(Clone, Debug, PartialEq, Eq, JsonSchema, DatabaseEntry)] -#[cultcache(type = "weksa.inventory_revision")] -pub struct InventoryRevision { - #[cultcache(key = 0)] - pub n: u32, - #[cultcache(key = 1)] - pub parent: Option, - #[cultcache(key = 2)] - pub changes: Vec, - #[cultcache(key = 3)] - pub frozen: bool, -} - -pub fn revision_key(n: u32) -> String { - format!("rev:{n}") -} - -/// The whole inventory as the store holds it. -#[derive(Clone, Debug, Default)] -pub struct Inventory { - entries: BTreeMap, - revisions: BTreeMap, -} - -impl Inventory { - /// An inventory exists only as `InventoryStore::inventory` returns it, so - /// no caller can validate or admit against a fabricated one. - /// - /// ```compile_fail,E0624 - /// // `Inventory::new` is not callable outside the crate. - /// let _ = weksa::Inventory::new(vec![], vec![]); - /// ``` - pub(crate) fn new(entries: Vec, revisions: Vec) -> Self { - Self { - entries: entries.into_iter().map(|e| (e.id, e)).collect(), - revisions: revisions.into_iter().map(|r| (r.n, r)).collect(), - } - } - - pub fn entries(&self) -> impl Iterator { - self.entries.values() - } - - pub fn entry(&self, id: &InvId) -> Option<&InventoryEntry> { - self.entries.get(id) - } - - pub fn revisions(&self) -> impl Iterator { - self.revisions.values() - } - - pub fn has_revision(&self, n: u32) -> bool { - self.revisions.contains_key(&n) - } - - pub fn latest(&self) -> Option<&InventoryRevision> { - self.revisions.values().next_back() - } - - /// The entry `id`, if a packet of revision `rev` may name it. - pub fn resolve(&self, id: &InvId, rev: u32) -> Option<&InventoryEntry> { - self.entry(id).filter(|e| e.live_at(rev)) - } -} - -/// Revision 1: the standard's enumerated values and the nuance features the -/// loop starts from. Concepts start empty. -pub fn seed() -> (InventoryRevision, Vec) { - let mut entries: Vec = Vec::new(); - let mut add = |family: Family, gloss: &str, kind: Option, slots: &[Slot]| { - let taken = entries.iter().filter(|e| e.family() == family).count() as u32; - entries.push(InventoryEntry { - id: family.id(NonZeroU32::MIN.saturating_add(taken)), - gloss: gloss.to_string(), - value_kind: kind, - applies_to: slots.to_vec(), - introduced_rev: 1, - retired_rev: None, - }); - }; - for g in [ - "person", - "group", - "organism", - "place", - "artifact", - "institution", - "ecology", - "route", - "memory", - "event", - "abstract", - ] { - add(Family::ReferentKind, g, None, &[]); - } - for g in [ - "coreference", - "part_of", - "member_of", - "located_at", - "near", - "before", - "after", - "causes", - "enables", - "prevents", - "owns", - "owes", - "knows", - "names", - "represents", - "contrasts_with", - ] { - add(Family::Relation, g, None, &[]); - } - for g in [ - "agent", - "patient", - "experiencer", - "instrument", - "source", - "goal", - "location", - "beneficiary", - ] { - add(Family::Role, g, None, &[]); - } - for g in [ - "assert", - "ask", - "request", - "promise", - "warn", - "greet", - "thank", - "apologise", - ] { - add(Family::SpeechAct, g, None, &[]); - } - let options = |labels: &[&str]| { - let labels = labels - .iter() - .map(|l| OptionLabel::parse(l).expect("seed labels are well formed")) - .collect(); - Some(ValueKind::Options( - OptionSet::new(labels).expect("seed option sets are well formed"), - )) - }; - let pred = [Slot::Predication]; - add( - Family::Feature, - "polarity", - options(&["positive", "negative"]), - &pred, - ); - add( - Family::Feature, - "tense", - options(&["past", "present", "future", "timeless"]), - &pred, - ); - add( - Family::Feature, - "aspect", - options(&[ - "perfective", - "imperfective", - "progressive", - "habitual", - "perfect", - ]), - &pred, - ); - add( - Family::Feature, - "modality", - options(&[ - "actual", - "potential", - "possible", - "necessary", - "desired", - "obligated", - ]), - &pred, - ); - add( - Family::Feature, - "evidentiality", - options(&["direct", "inferred", "reported", "assumed"]), - &pred, - ); - let nuance = [Slot::Discourse, Slot::Predication]; - for g in ["hedge", "certainty"] { - add(Family::Feature, g, Some(ValueKind::Grade), &nuance); - } - add(Family::Feature, "irony", Some(ValueKind::Flag), &nuance); - // Felt affect (permille of how strongly it is felt) and the restraint the speaker - // puts on it: slot Stance only. Affection covers fondness toward the - // addressee. Displayed affect is each language's ontology's business. - let stance = [Slot::Stance]; - for g in [ - "anger", - "fear", - "sadness", - "joy", - "disgust", - "surprise", - "contempt", - "affection", - "shame", - "restraint", - ] { - add(Family::Feature, g, Some(ValueKind::Grade), &stance); - } - let changes = entries.iter().map(|e| Change::Add(e.id)).collect(); - ( - InventoryRevision { - n: 1, - parent: None, - changes, - frozen: false, - }, - entries, - ) -} - -/// Revision 2: the concepts and frames the `nibu-proof-001` packet names, which -/// the seed leaves open. Hand-seeded like [`seed`] until the golf loop's -/// minting round owns concepts and frames. -pub fn proof_revision() -> (InventoryRevision, Vec) { - let mut entries: Vec = Vec::new(); - for (family, glosses) in [ - ( - Family::Concept, - ["endanger", "prevent", "player", "control", "ship_mind"].as_slice(), - ), - (Family::Frame, ["hazard_warning", "intervention"].as_slice()), - ] { - for (i, gloss) in glosses.iter().enumerate() { - entries.push(InventoryEntry { - id: family.id(NonZeroU32::MIN.saturating_add(i as u32)), - gloss: gloss.to_string(), - value_kind: None, - applies_to: vec![], - introduced_rev: 2, - retired_rev: None, - }); - } - } - let changes = entries.iter().map(|e| Change::Add(e.id)).collect(); - ( - InventoryRevision { - n: 2, - parent: Some(1), - changes, - frozen: false, - }, - entries, - ) -} diff --git a/crates/weksa/src/lib.rs b/crates/weksa/src/lib.rs deleted file mode 100644 index 0bfdb8c..0000000 --- a/crates/weksa/src/lib.rs +++ /dev/null @@ -1,35 +0,0 @@ -//! Weksa's typed interlingua state: packet, inventory and corpus documents in -//! CultCache, the validator that decides whether a packet exists, the derived -//! train/validation/test split, the stores that admit them, and the scorer -//! (chrF, meaning cosine, leak diagnostics). - -pub mod corpus; -pub mod entity; -pub mod form; -pub mod inventory; -pub mod lower; -pub mod packet; -pub mod schema; -pub mod score; -pub mod split; -pub mod store; -pub mod validate; - -pub use corpus::{CorpusItem, CorpusItemId, NameSpan, Source, Turn}; -pub use entity::{Entity, EntityOrigin, LanguageForms}; -pub use form::FormError; -pub use inventory::{ - Change, Family, InvId, Inventory, InventoryEntry, InventoryRevision, OptionLabel, OptionSet, - Slot, ValueKind, -}; -pub use lower::{LowerError, lower}; -pub use packet::{ - EncoderId, EntityId, Grade, NodeId, Packet, PacketId, PacketKind, SCHEMA_VERSION, Stance, -}; -pub use score::{Embedder, ItemScore, OllamaEmbedder, Scorer, chrf}; -pub use split::{Split, SplitScheme}; -pub use store::{ - AddOutcome, CorpusStore, EntityBound, EntityStore, InventoryStore, PacketStore, StoreError, - StoreRefusal, -}; -pub use validate::{LoweringInput, Refusal, validate}; diff --git a/crates/weksa/src/lower.rs b/crates/weksa/src/lower.rs deleted file mode 100644 index d6fcdfb..0000000 --- a/crates/weksa/src/lower.rs +++ /dev/null @@ -1,205 +0,0 @@ -//! The English lowerer: the validated lowering view of a packet to text, by a -//! fixed table keyed by inventory ids. It renders predications, referents and the -//! speech act, and nothing else: no style, no model, no corpus text. It takes a -//! [`LoweringInput`], so it cannot see the trace or the corpus item. A packet -//! carrying a feature, relation, extension, role fill or referent it does not -//! render is refused, because rendering part of a meaning would assert a different -//! one (`polarity: negative` dropped says the opposite). Context, kind and -//! constraints steer delivery and are not rendered; the concept of a predication -//! must resolve but the frame owns the verb. -//! -//! The table is a temporary owner. Each row speaks for an inventory entry and -//! names the gloss it speaks for, so a row cannot outlive a change of what its id -//! means. `language-ontology` moves the rows into ontology documents. - -use std::fmt; - -use crate::inventory::{Family, InvId, Inventory}; -use crate::packet::{NodeId, Predication}; -use crate::validate::LoweringInput; - -/// `(inventory id, the gloss the row speaks for, surface)`. -type Row = (&'static str, &'static str, &'static str); - -/// Referent concepts, as the noun phrase after "the". -const CONCEPTS: &[Row] = &[ - ("concept:3", "player", "player"), - ("concept:4", "control", "control"), - ("concept:5", "ship_mind", "ship mind"), -]; - -/// Frames: the clause a predication of that frame says. `{gloss}` is the filler -/// of the role whose inventory gloss it is. -const FRAMES: &[Row] = &[ - ( - "frame:1", - "hazard_warning", - "{agent} may endanger {patient} by touching {instrument}", - ), - ( - "frame:2", - "intervention", - "{agent} will prevent {patient} to protect {beneficiary}", - ), -]; - -/// Speech acts: the opener of the first sentence. -const SPEECH_ACTS: &[Row] = &[ - ("speech_act:1", "assert", ""), - ("speech_act:5", "warn", "Warning: "), -]; - -#[derive(Clone, Debug, PartialEq, Eq)] -pub enum LowerError { - /// The packet carries a slot the lowerer does not render. - Unrendered { what: &'static str }, - /// No table row for an id the packet uses. - NoRow { id: InvId }, - /// The row's gloss is not the gloss the inventory gives its id at the - /// packet's revision, or the id does not resolve there. - GlossMismatch { id: InvId, row: &'static str }, - /// A frame template names a role the inventory has no live entry for. - UnknownRole { frame: InvId, role: String }, - /// The predication fills no role its frame's template needs. - MissingRole { predication: NodeId, role: String }, -} - -impl fmt::Display for LowerError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{self:?}") - } -} - -impl std::error::Error for LowerError {} - -fn row( - table: &'static [Row], - id: InvId, - inv: &Inventory, - rev: u32, -) -> Result<&'static str, LowerError> { - let (_, gloss, surface) = table - .iter() - .find(|(key, _, _)| *key == id.to_string()) - .ok_or(LowerError::NoRow { id })?; - match inv.resolve(&id, rev) { - Some(e) if e.gloss == *gloss => Ok(surface), - _ => Err(LowerError::GlossMismatch { id, row: gloss }), - } -} - -pub fn lower(input: &LoweringInput, inv: &Inventory) -> Result { - let rev = input.inventory_rev(); - let unrendered = |what| Err(LowerError::Unrendered { what }); - if input.predications().is_empty() { - return unrendered("no predication"); - } - if !input.discourse().features.is_empty() - || input.referents().iter().any(|r| !r.features.is_empty()) - || input.predications().iter().any(|p| !p.features.is_empty()) - { - return unrendered("features"); - } - if !input.relations().is_empty() { - return unrendered("relations"); - } - if !input.extensions().is_empty() { - return unrendered("extensions"); - } - let opener = match input.discourse().speech_act { - Some(act) => row(SPEECH_ACTS, act, inv, rev)?, - None => "", - }; - let mut sentences = Vec::new(); - for p in input.predications() { - sentences.push(clause(input, inv, p)?); - } - let mentioned = |r: &NodeId| { - input - .predications() - .iter() - .any(|p| p.roles.iter().any(|f| &f.filler == r)) - }; - if input.referents().iter().any(|r| !mentioned(&r.id)) { - return unrendered("a referent no role mentions"); - } - let mut out = String::new(); - for (i, clause) in sentences.iter().enumerate() { - if i > 0 { - out.push(' '); - } - if i == 0 { - out.push_str(opener); - } - let mut chars = clause.chars(); - if let Some(first) = chars.next() { - if i == 0 && !opener.is_empty() { - out.push(first); - } else { - out.extend(first.to_uppercase()); - } - out.push_str(chars.as_str()); - } - out.push('.'); - } - Ok(out) -} - -fn clause(input: &LoweringInput, inv: &Inventory, p: &Predication) -> Result { - let rev = input.inventory_rev(); - let template = row(FRAMES, p.frame, inv, rev)?; - let mut said: Vec<&NodeId> = Vec::new(); - let mut out = String::new(); - let mut rest = template; - while let Some(open) = rest.find('{') { - out.push_str(&rest[..open]); - let close = rest[open..].find('}').map_or(rest.len(), |c| open + c); - let name = &rest[open + 1..close]; - rest = &rest[(close + 1).min(rest.len())..]; - let role = inv - .entries() - .find(|e| e.family() == Family::Role && e.gloss == name && e.live_at(rev)) - .ok_or_else(|| LowerError::UnknownRole { - frame: p.frame, - role: name.to_string(), - })?; - let fill = - p.roles - .iter() - .find(|r| r.role == role.id) - .ok_or_else(|| LowerError::MissingRole { - predication: p.id.clone(), - role: name.to_string(), - })?; - out.push_str(&filler(input, inv, &fill.filler, &said)?); - said.push(&fill.filler); - } - out.push_str(rest); - if p.roles.len() != said.len() { - return Err(LowerError::Unrendered { - what: "a role the frame does not name", - }); - } - Ok(out) -} - -/// A role filler as a phrase: the addressee is "you", a predication is "that", -/// another referent is "the" and its concept word. The addressee said twice in -/// one predication is reflexive; no other referent is, because the packet has no -/// animacy to choose "themself" or "itself" by. -fn filler( - input: &LoweringInput, - inv: &Inventory, - node: &NodeId, - said: &[&NodeId], -) -> Result { - let again = said.contains(&node); - if let Some(r) = input.referents().iter().find(|r| &r.id == node) { - if input.discourse().addressee.as_ref() == Some(node) { - return Ok(if again { "yourself" } else { "you" }.to_string()); - } - let word = row(CONCEPTS, r.concept, inv, input.inventory_rev())?; - return Ok(format!("the {word}")); - } - Ok("that".to_string()) -} diff --git a/crates/weksa/src/main.rs b/crates/weksa/src/main.rs deleted file mode 100644 index f36ba27..0000000 --- a/crates/weksa/src/main.rs +++ /dev/null @@ -1,553 +0,0 @@ -//! The `weksa` command line: the same validator and store admission the library -//! exposes, plus text projections (`dump`) that are for review and are not stores. - -use std::path::PathBuf; - -use anyhow::{Context, Result, bail}; -use clap::{Args, Parser, Subcommand, ValueEnum}; -use unicode_normalization::UnicodeNormalization; -use weksa::corpus::{CorpusItem, ModelFamily, NameSpan, Source, Turn}; -use weksa::entity::{Entity, EntityOrigin, LanguageForms}; -use weksa::inventory::{proof_revision, seed}; -use weksa::packet::proof_packet; -use weksa::score::ScoreInput; -use weksa::{ - CorpusStore, EntityId, EntityStore, InventoryStore, OllamaEmbedder, PacketStore, Scorer, - SplitScheme, StoreError, lower, validate, -}; - -#[derive(Parser)] -#[command( - name = "weksa", - about = "Weksa interlingua packet, inventory and corpus tools" -)] -struct Cli { - /// The corpus store. - #[arg(long, global = true, default_value = "data/corpus/corpus.cc")] - corpus: PathBuf, - /// The inventory store. - #[arg(long, global = true, default_value = "data/interlingua/inventory.cc")] - inventory: PathBuf, - /// The entity store. - #[arg(long, global = true, default_value = "data/entity/entity.cc")] - entities: PathBuf, - #[command(subcommand)] - command: Command, -} - -#[derive(Subcommand)] -enum Command { - /// Validate the packets in a store (or one packet) against the inventory. - Validate { store: PathBuf, key: Option }, - #[command(subcommand)] - Corpus(CorpusCommand), - #[command(subcommand)] - Inventory(InventoryCommand), - #[command(subcommand)] - Entity(EntityCommand), - #[command(subcommand)] - Packet(PacketCommand), - /// Lower a stored packet to English text. - Lower { store: PathBuf, key: String }, - /// Score output text (stdin, or --output) against corpus items, one embed call. - Score(ScoreArgs), - /// Print the split of a corpus item. - Split { item_key: String }, - /// Write the published JSON Schemas. - Schema { - #[arg(long, default_value = "schemas/cultnet")] - out: PathBuf, - }, -} - -#[derive(Subcommand)] -enum PacketCommand { - /// Admit the nibu-proof-001 packet through `PacketStore::admit`. - Proof { store: PathBuf }, -} - -#[derive(Args)] -struct ScoreArgs { - /// Corpus item keys, the references. - #[arg(required = true)] - items: Vec, - /// The text to score; stdin when absent. - #[arg(long)] - output: Option, - #[arg(long, default_value = "http://127.0.0.1:11434")] - ollama_url: String, - #[arg(long, default_value = "qwen3-embedding:0.6b")] - model: String, -} - -#[derive(Subcommand)] -enum CorpusCommand { - Add(Box), - /// Text projection of the corpus for review; not a store. - Dump, - Withdraw { - key: String, - }, -} - -#[derive(Subcommand)] -enum InventoryCommand { - /// Text projection of the inventory for review; not a store. - Dump, - /// Admit revision 1, the standard's enumerated values and the nuance features. - Seed, - /// Admit revision 2, the concepts and frames the nibu-proof-001 packet names. - Proof, -} - -#[derive(Subcommand)] -enum EntityCommand { - /// Admit an entity, or append forms to a stored one. - Put(PutArgs), - /// Text projection of the entity store for review; not a store. - Dump, -} - -#[derive(Args)] -struct PutArgs { - /// `entity:`. - id: String, - /// `language=form`, repeated; a language's forms keep the order given, - /// citation form first. Forms are NFC-normalized here. - #[arg(long = "form", required = true)] - forms: Vec, - #[arg(long, value_enum)] - origin: OriginKind, - /// The persona of a `cast` or `live` origin; refused with any other origin. - #[arg(long)] - persona: Option, - /// The fixture path a `cast` entity came from; refused with any other origin. - #[arg(long)] - fixture: Option, - /// The work of a `public-domain` origin; refused with any other origin. - #[arg(long)] - work: Option, -} - -#[derive(Clone, Copy, ValueEnum)] -enum OriginKind { - Cast, - PublicDomain, - Live, -} - -impl PutArgs { - fn entity(self) -> Result { - let mut forms: Vec = Vec::new(); - for f in &self.forms { - let (language, form) = f.split_once('=').context("--form is not language=form")?; - match forms.iter_mut().find(|lf| lf.language == language) { - Some(lf) => lf.forms.push(form.nfc().collect()), - None => forms.push(LanguageForms { - language: language.to_string(), - forms: vec![form.nfc().collect()], - }), - } - } - let origin = match (self.origin, self.persona, self.fixture, self.work) { - (OriginKind::Cast, persona, fixture, None) => EntityOrigin::Cast { - persona: need(persona, "persona")?, - fixture: need(fixture, "fixture")?, - }, - (OriginKind::PublicDomain, None, None, work) => EntityOrigin::PublicDomain { - work: need(work, "work")?, - }, - (OriginKind::Live, persona, None, None) => EntityOrigin::Live { - persona: need(persona, "persona")?, - }, - _ => bail!( - "--fixture is only for --origin cast, --persona only for --origin cast or live and --work only for --origin public-domain" - ), - }; - Ok(Entity { - id: self.id.parse()?, - forms, - origin, - }) - } -} - -#[derive(Clone, Copy, ValueEnum)] -enum SourceKind { - CastAuthored, - OperatorWritten, - OperatorApproved, - PublicDomain, - ModelWritten, -} - -#[derive(Clone, Copy, ValueEnum)] -enum ModelFamilyArg { - Anthropic, - OpenAi, - Google, - Alibaba, - Meta, - Mistral, - Unrecorded, -} - -impl From for ModelFamily { - fn from(f: ModelFamilyArg) -> Self { - match f { - ModelFamilyArg::Anthropic => ModelFamily::Anthropic, - ModelFamilyArg::OpenAi => ModelFamily::OpenAi, - ModelFamilyArg::Google => ModelFamily::Google, - ModelFamilyArg::Alibaba => ModelFamily::Alibaba, - ModelFamilyArg::Meta => ModelFamily::Meta, - ModelFamilyArg::Mistral => ModelFamily::Mistral, - ModelFamilyArg::Unrecorded => ModelFamily::Unrecorded, - } - } -} - -#[derive(Args)] -struct AddArgs { - #[arg(long)] - slug: String, - /// NFC-normalized here; `--name` byte offsets index the normalized text. - #[arg(long)] - text: String, - #[arg(long, default_value = "en-US")] - language: String, - /// An entity id. - #[arg(long)] - speaker: EntityId, - /// `start..end=entity:`, repeated: a byte range of the text naming an entity. - #[arg(long = "name")] - names: Vec, - #[arg(long)] - group: String, - /// `speaker=text`, repeated, oldest first. - #[arg(long = "prior-turn")] - prior_turns: Vec, - #[arg(long, value_enum)] - source: SourceKind, - #[arg(long)] - persona: Option, - #[arg(long)] - fixture: Option, - #[arg(long)] - note: Option, - #[arg(long)] - work: Option, - /// Who wrote the line: for `cast-authored` and `public-domain`. - #[arg(long)] - author: Option, - #[arg(long)] - edition_url: Option, - #[arg(long = "model-family", value_enum)] - family: Option, - /// The generator string the fixture or run wrote, verbatim. - #[arg(long)] - generator: Option, - #[arg(long)] - prompt_digest: Option, - #[arg(long)] - admitted_on: String, - #[arg(long)] - supersedes: Option, -} - -fn need(value: Option, flag: &str) -> Result { - value.with_context(|| format!("this source needs --{flag}")) -} - -impl AddArgs { - fn source(&self) -> Result { - let get = |v: &Option, flag: &str| need(v.clone(), flag); - let applies: &[&str] = match self.source { - SourceKind::CastAuthored => &["persona", "fixture", "author"], - SourceKind::OperatorWritten => &[], - SourceKind::OperatorApproved => &["note"], - SourceKind::PublicDomain => &["work", "author", "edition-url"], - SourceKind::ModelWritten => &["model-family", "generator", "prompt-digest"], - }; - let given = [ - ("persona", self.persona.is_some()), - ("fixture", self.fixture.is_some()), - ("author", self.author.is_some()), - ("note", self.note.is_some()), - ("work", self.work.is_some()), - ("edition-url", self.edition_url.is_some()), - ("model-family", self.family.is_some()), - ("generator", self.generator.is_some()), - ("prompt-digest", self.prompt_digest.is_some()), - ]; - let stray: Vec = given - .into_iter() - .filter(|(flag, given)| *given && !applies.contains(flag)) - .map(|(flag, _)| format!("--{flag}")) - .collect(); - if !stray.is_empty() { - bail!("{} does not apply to this --source", stray.join(", ")); - } - Ok(match self.source { - SourceKind::CastAuthored => Source::CastAuthored { - persona: get(&self.persona, "persona")?, - fixture: get(&self.fixture, "fixture")?, - author: get(&self.author, "author")?, - }, - SourceKind::OperatorWritten => Source::OperatorWritten, - SourceKind::OperatorApproved => Source::OperatorApproved { - note: get(&self.note, "note")?, - }, - SourceKind::PublicDomain => Source::PublicDomain { - work: get(&self.work, "work")?, - author: get(&self.author, "author")?, - edition_url: get(&self.edition_url, "edition-url")?, - }, - SourceKind::ModelWritten => Source::ModelWritten { - family: self - .family - .map(ModelFamily::from) - .context("this source needs --model-family")?, - generator: self.generator.clone(), - prompt_digest: self.prompt_digest.clone(), - }, - }) - } - - fn item(self, id: String) -> Result { - let prior_turns = self - .prior_turns - .iter() - .map(|t| { - let (speaker, text) = t - .split_once('=') - .context("--prior-turn is not speaker=text")?; - Ok(Turn { - speaker: speaker.parse().map_err(|_| { - anyhow::anyhow!("--prior-turn speaker is not entity:") - })?, - text: text.nfc().collect(), - }) - }) - .collect::>>()?; - let names = self - .names - .iter() - .map(|n| { - let bad = || anyhow::anyhow!("--name is not start..end=entity:"); - let (range, entity) = n.split_once('=').ok_or_else(bad)?; - let (start, end) = range.split_once("..").ok_or_else(bad)?; - Ok(NameSpan { - entity: entity.parse().map_err(|_| bad())?, - start: start.parse().map_err(|_| bad())?, - end: end.parse().map_err(|_| bad())?, - }) - }) - .collect::>>()?; - Ok(CorpusItem { - id, - source: self.source()?, - text: self.text.nfc().collect(), - language: self.language, - speaker: self.speaker, - group: self.group, - prior_turns, - admitted_on: self.admitted_on, - supersedes: self.supersedes, - withdrawn: false, - names, - }) - } -} - -/// The split of an item as `Train`, `Validation`, `Test` or `excluded`. -fn split_text(item: &CorpusItem) -> String { - SplitScheme::V1 - .split(item) - .map_or_else(|| "excluded".to_string(), |s| format!("{s:?}")) -} - -fn refused(e: StoreError) -> anyhow::Error { - anyhow::anyhow!("{e}") -} - -fn main() -> Result<()> { - let cli = Cli::parse(); - match cli.command { - Command::Validate { store, key } => { - let inventory = InventoryStore::open(&cli.inventory)?.inventory()?; - let packets = PacketStore::open(&store)?; - let packets = match key { - Some(key) => vec![ - packets - .get(&key)? - .with_context(|| format!("no packet {key}"))?, - ], - None => packets.all()?, - }; - let mut bad = 0; - for p in &packets { - match validate(p, &inventory) { - Ok(_) => println!("ok {}", p.id), - Err(refusals) => { - bad += 1; - for r in refusals { - println!("refused {} {r}", p.id); - } - } - } - } - if bad > 0 { - bail!("{bad} of {} packets refused", packets.len()); - } - } - Command::Corpus(CorpusCommand::Add(args)) => { - let mut store = CorpusStore::open(&cli.corpus)?; - let key = store.next_key(&args.slug).map_err(refused)?; - let entities = EntityStore::open(&cli.entities)?; - let outcome = store - .add((*args).item(key.clone())?, &entities) - .map_err(refused)?; - println!("{outcome:?} {key}"); - } - Command::Corpus(CorpusCommand::Dump) => { - for i in CorpusStore::open(&cli.corpus)?.all()? { - let split = split_text(&i); - let withdrawn = if i.withdrawn { " withdrawn" } else { "" }; - println!( - "{} [{} {} group={} {split}{withdrawn}]\n {:?}\n names: {:?}\n source: {:?}", - i.id, i.language, i.speaker, i.group, i.text, i.names, i.source - ); - } - } - Command::Corpus(CorpusCommand::Withdraw { key }) => { - CorpusStore::open(&cli.corpus)? - .withdraw(&key) - .map_err(refused)?; - println!("withdrawn {key}"); - } - Command::Inventory(InventoryCommand::Dump) => { - let inv = InventoryStore::open(&cli.inventory)?.inventory()?; - for r in inv.revisions() { - println!( - "rev:{} parent={:?} frozen={} changes={}", - r.n, - r.parent, - r.frozen, - r.changes.len() - ); - } - for e in inv.entries() { - println!( - "{} introduced={} retired={:?} {:?} {:?} {}", - e.id, e.introduced_rev, e.retired_rev, e.value_kind, e.applies_to, e.gloss - ); - } - } - Command::Entity(EntityCommand::Put(args)) => { - let entity = args.entity()?; - let id = entity.id.clone(); - let outcome = EntityStore::open(&cli.entities)? - .put(entity) - .map_err(refused)?; - println!("{outcome:?} {id}"); - } - Command::Entity(EntityCommand::Dump) => { - for e in EntityStore::open(&cli.entities)?.all()? { - println!("{} {:?}\n forms: {:?}", e.id, e.origin, e.forms); - } - } - Command::Inventory(InventoryCommand::Seed) => { - let (rev, entries) = seed(); - let count = entries.len(); - InventoryStore::open(&cli.inventory)? - .admit_revision(rev, entries) - .map_err(refused)?; - println!("seeded rev:1 with {count} entries"); - } - Command::Inventory(InventoryCommand::Proof) => { - let (rev, entries) = proof_revision(); - let count = entries.len(); - InventoryStore::open(&cli.inventory)? - .admit_revision(rev, entries) - .map_err(refused)?; - println!("admitted rev:2 with {count} entries"); - } - Command::Packet(PacketCommand::Proof { store }) => { - let packet = proof_packet(); - let key = packet.id.to_string(); - PacketStore::open(&store)? - .admit( - packet, - &InventoryStore::open(&cli.inventory)?, - &CorpusStore::open(&cli.corpus)?, - &EntityStore::open(&cli.entities)?, - ) - .map_err(refused)?; - println!("admitted {key}"); - } - Command::Lower { store, key } => { - let inventory_store = InventoryStore::open(&cli.inventory)?; - let inventory = inventory_store.inventory()?; - let mut packets = PacketStore::open(&store)?; - let packet = packets - .get(&key)? - .with_context(|| format!("no packet {key}"))?; - // Admission is the only source of a LoweringInput; re-admitting the - // stored packet is a no-op write that re-runs every door check. - let view = packets - .admit( - packet, - &inventory_store, - &CorpusStore::open(&cli.corpus)?, - &EntityStore::open(&cli.entities)?, - ) - .map_err(refused)?; - println!("{}", lower(&view, &inventory)?); - } - Command::Score(args) => { - let output = match &args.output { - Some(path) => std::fs::read_to_string(path)?, - None => std::io::read_to_string(std::io::stdin())?, - }; - let corpus = CorpusStore::open(&cli.corpus)?; - let references = args - .items - .iter() - .map(|key| { - Ok(corpus - .get(key)? - .with_context(|| format!("no item {key}"))? - .text) - }) - .collect::>>()?; - let inputs: Vec = references - .iter() - .map(|reference| ScoreInput { - reference, - output: output.trim(), - }) - .collect(); - let scorer = Scorer::new(OllamaEmbedder::new(&args.ollama_url, &args.model)); - for (key, score) in args.items.iter().zip(scorer.score(&inputs)?) { - match score.meaning { - Some(m) => println!("{key} chrf={:.2} meaning={m:.4}", score.chrf), - None => println!("{key} chrf={:.2} meaning=none", score.chrf), - } - } - } - Command::Split { item_key } => { - let item = CorpusStore::open(&cli.corpus)? - .get(&item_key)? - .with_context(|| format!("no item {item_key}"))?; - println!("{}", split_text(&item)); - } - Command::Schema { out } => { - std::fs::create_dir_all(&out)?; - for (name, json) in weksa::schema::published() { - std::fs::write(out.join(&name), json)?; - println!("wrote {name}"); - } - } - } - Ok(()) -} diff --git a/crates/weksa/src/packet.rs b/crates/weksa/src/packet.rs deleted file mode 100644 index cae001b..0000000 --- a/crates/weksa/src/packet.rs +++ /dev/null @@ -1,432 +0,0 @@ -//! The interlingua packet. Every id is an opaque form type: a node id `n`, a -//! packet id `packet:`, an encoder id `encoder:`, an inventory id -//! `:`. The only free text is the trace, which is stripped before -//! lowering. A malformed id cannot be built, and cannot be read back from a -//! store, so a packet cannot carry target-language wording through an id slot. - -use std::fmt; - -use cultcache_rs::DatabaseEntry; -use schemars::JsonSchema; -use serde::{Deserialize, Serialize}; - -use crate::corpus::CorpusItemId; -use crate::form::{FormError, is_slug, string_form}; -pub use crate::inventory::InvId; -use crate::inventory::OptionLabel; - -/// Bumped by a commit that changes these types; the published schema moves with it. -pub const SCHEMA_VERSION: u32 = 4; - -/// Id of a referent or predication within one packet: `n`, `k` a canonical -/// decimal 1..=9999. -#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] -#[serde(try_from = "String", into = "String")] -pub struct NodeId(String); - -impl NodeId { - pub fn parse(s: &str) -> Result { - let k = s - .strip_prefix('n') - .filter(|d| !d.starts_with('0') && d.bytes().all(|b| b.is_ascii_digit())) - .and_then(|d| d.parse::().ok()) - .filter(|k| (1..=9999).contains(k)); - k.map(|_| Self(s.to_string())) - .ok_or_else(|| FormError::new("NodeId", s)) - } - - pub fn as_str(&self) -> &str { - &self.0 - } -} - -impl fmt::Display for NodeId { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str(&self.0) - } -} - -string_form!(NodeId, { "type": "string", "pattern": "^n[1-9][0-9]{0,3}$" }); - -/// `:<16 lowercase hex>`. -fn hex16(prefix: &str, s: &str) -> bool { - s.strip_prefix(prefix) - .is_some_and(|h| h.len() == 16 && h.bytes().all(|b| matches!(b, b'0'..=b'9' | b'a'..=b'f'))) -} - -/// Key of a packet: `packet:`, a digest the loop derives from the item, -/// the inventory revision and the encoder. -#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] -#[serde(try_from = "String", into = "String")] -pub struct PacketId(String); - -impl PacketId { - pub fn parse(s: &str) -> Result { - if hex16("packet:", s) { - Ok(Self(s.to_string())) - } else { - Err(FormError::new("PacketId", s)) - } - } - - pub fn as_str(&self) -> &str { - &self.0 - } -} - -impl fmt::Display for PacketId { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str(&self.0) - } -} - -string_form!(PacketId, { "type": "string", "pattern": "^packet:[0-9a-f]{16}$" }); - -/// `encoder:`, a digest of the model id and the prompt digest. -#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] -#[serde(try_from = "String", into = "String")] -pub struct EncoderId(String); - -impl EncoderId { - pub fn parse(s: &str) -> Result { - if hex16("encoder:", s) { - Ok(Self(s.to_string())) - } else { - Err(FormError::new("EncoderId", s)) - } - } - - pub fn as_str(&self) -> &str { - &self.0 - } -} - -impl fmt::Display for EncoderId { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str(&self.0) - } -} - -string_form!(EncoderId, { "type": "string", "pattern": "^encoder:[0-9a-f]{16}$" }); - -/// Id of an entity: `entity:`, a slug as an item key's (`[a-z0-9]` and -/// inner `-`, 1 to 48 bytes, no edge `-`). -#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] -#[serde(try_from = "String", into = "String")] -pub struct EntityId(String); - -impl EntityId { - pub fn parse(s: &str) -> Result { - match s.strip_prefix("entity:") { - Some(slug) if is_slug(slug) => Ok(Self(s.to_string())), - _ => Err(FormError::new("EntityId", s)), - } - } - - pub fn as_str(&self) -> &str { - &self.0 - } -} - -impl fmt::Display for EntityId { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str(&self.0) - } -} - -string_form!(EntityId, { "type": "string", "pattern": "^entity:[a-z0-9]([a-z0-9-]{0,46}[a-z0-9])?$" }); - -/// Permille, 0..=1000. -#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] -#[serde(try_from = "u16", into = "u16")] -pub struct Grade(u16); - -impl Grade { - pub const MAX: u16 = 1000; - - pub fn new(permille: u16) -> Result { - if permille <= Self::MAX { - Ok(Self(permille)) - } else { - Err(FormError::new("Grade", permille)) - } - } - - pub fn get(self) -> u16 { - self.0 - } -} - -impl TryFrom for Grade { - type Error = FormError; - fn try_from(v: u16) -> Result { - Self::new(v) - } -} - -impl From for u16 { - fn from(g: Grade) -> u16 { - g.0 - } -} - -impl JsonSchema for Grade { - fn schema_name() -> std::borrow::Cow<'static, str> { - "Grade".into() - } - fn json_schema(_: &mut schemars::SchemaGenerator) -> schemars::Schema { - schemars::json_schema!({ "type": "integer", "minimum": 0, "maximum": 1000 }) - } -} - -#[derive(Clone, Debug, PartialEq, Eq, JsonSchema, DatabaseEntry)] -#[cultcache(type = "weksa.interlingua_packet")] -pub struct Packet { - #[cultcache(key = 0)] - pub id: PacketId, - #[cultcache(key = 1)] - pub schema_version: u32, - #[cultcache(key = 2)] - pub inventory_rev: u32, - #[cultcache(key = 3)] - pub kind: PacketKind, - #[cultcache(key = 4)] - pub provenance: Provenance, - #[cultcache(key = 5)] - pub context: Context, - #[cultcache(key = 6)] - pub discourse: Discourse, - #[cultcache(key = 7)] - pub referents: Vec, - #[cultcache(key = 8)] - pub predications: Vec, - #[cultcache(key = 9)] - pub relations: Vec, - #[cultcache(key = 10)] - pub constraints: Constraints, - #[cultcache(key = 11)] - pub extensions: Vec, - #[cultcache(key = 12)] - pub trace: Trace, - #[cultcache(key = 13)] - pub stance: Stance, -} - -impl Packet { - /// The most referents, predications and relations one packet holds: a - /// calibration constant of ours, capacity and not meaning. - pub const MAX_NODES: usize = 64; -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] -pub enum PacketKind { - Utterance, - NameRequest, - CommentaryIntent, - Narration, - DialogueLine, - CorpusExample, - DebugProbe, -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] -pub enum ProvenanceSource { - Authored, - ModelDecomposition, - GameTelemetry, - ImportedCorpus, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] -pub struct Provenance { - pub source: ProvenanceSource, - /// The corpus item this packet was minted for. Required: the packet store - /// admits only packets that name their item. - pub corpus_item: CorpusItemId, - pub encoder: EncoderId, - pub confidence: Grade, -} - -#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] -pub struct Context { - /// Concept ids. - pub setting: Option, - pub medium: Option, - pub participants: Vec, - pub audience: Vec, -} - -#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] -pub struct Discourse { - /// Speech-act id. - pub speech_act: Option, - pub addressee: Option, - pub features: Vec, -} - -/// The speaker's felt affect and restraint, never displayed affect: how the -/// speaker shows a feeling is each language's ontology's business. A -/// held-back feeling is not a claim, so it is never a predication. -#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] -pub struct Stance { - pub features: Vec, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] -pub struct FeatureValue { - /// Feature id. - pub feature: InvId, - pub value: FeatValue, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] -pub enum FeatValue { - Grade(Grade), - /// A label the feature's inventory entry declares. - Option(OptionLabel), - Flag(bool), -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] -pub struct Referent { - pub id: NodeId, - /// Referent-kind id. - pub kind: InvId, - /// Concept id. - pub concept: InvId, - /// The entity this referent names, when it has one. The entity store holds - /// its wording; the packet holds only the id. - pub entity: Option, - pub features: Vec, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] -pub struct Predication { - pub id: NodeId, - /// Concept id. - pub concept: InvId, - /// Frame id. - pub frame: InvId, - pub roles: Vec, - pub features: Vec, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] -pub struct RoleFill { - /// Role id. - pub role: InvId, - pub filler: NodeId, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] -pub struct Relation { - /// Relation id. - pub relation: InvId, - pub from: NodeId, - pub to: NodeId, - pub features: Vec, -} - -#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] -pub struct Constraints { - pub preserve: Vec, - pub deterministic: bool, - pub max_latency_ms: Option, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] -pub struct Extension { - /// Extension-entry id; the inventory declares its value kind. - pub key: InvId, - pub value: FeatValue, -} - -/// Authoring and debug notes. Stripped before lowering. -#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] -pub struct Trace { - pub notes: Vec, - pub rejected_readings: Vec, -} - -/// The `nibu-proof-001` packet: the meaning of `examples/english-lowering/nibu-proof-001/interlingua.yaml` -/// (warn the player off the damaged control; Nibu prevents it for them) as typed -/// ids of inventory revision 2, for the corpus item `item:nibu:1`. No features: -/// nothing here is style. -pub fn proof_packet() -> Packet { - fn id(s: &str) -> InvId { - InvId::parse(s).expect("proof ids are well formed") - } - fn node(s: &str) -> NodeId { - NodeId::parse(s).expect("proof nodes are well formed") - } - let referent = |n: &str, kind: &str, concept: &str, entity: Option<&str>| Referent { - id: node(n), - kind: id(kind), - concept: id(concept), - entity: entity.map(|e| EntityId::parse(e).expect("proof entity is well formed")), - features: vec![], - }; - let fill = |role: &str, filler: &str| RoleFill { - role: id(role), - filler: node(filler), - }; - Packet { - id: PacketId::parse("packet:6e6962752d303031").expect("proof id"), - schema_version: SCHEMA_VERSION, - inventory_rev: 2, - kind: PacketKind::DialogueLine, - provenance: Provenance { - source: ProvenanceSource::Authored, - corpus_item: CorpusItemId::parse("item:nibu:1").expect("proof item"), - encoder: EncoderId::parse("encoder:6f70657261746f72").expect("proof encoder"), - confidence: Grade::new(900).expect("proof grade"), - }, - context: Context::default(), - discourse: Discourse { - speech_act: Some(id("speech_act:5")), - addressee: Some(node("n1")), - features: vec![], - }, - referents: vec![ - referent("n1", "referent_kind:1", "concept:3", None), - referent("n2", "referent_kind:5", "concept:4", None), - referent("n3", "referent_kind:1", "concept:5", Some("entity:nibu")), - ], - predications: vec![ - Predication { - id: node("n4"), - concept: id("concept:1"), - frame: id("frame:1"), - roles: vec![ - fill("role:1", "n1"), - fill("role:4", "n2"), - fill("role:2", "n1"), - ], - features: vec![], - }, - Predication { - id: node("n5"), - concept: id("concept:2"), - frame: id("frame:2"), - roles: vec![ - fill("role:1", "n3"), - fill("role:2", "n4"), - fill("role:8", "n1"), - ], - features: vec![], - }, - ], - relations: vec![], - constraints: Constraints { - preserve: vec![node("n1"), node("n2")], - deterministic: true, - max_latency_ms: None, - }, - extensions: vec![], - trace: Trace { - notes: vec!["First proof packet for Nibu flavored-English lowering.".to_string()], - rejected_readings: vec![], - }, - stance: Stance::default(), - } -} diff --git a/crates/weksa/src/schema.rs b/crates/weksa/src/schema.rs deleted file mode 100644 index 72c27e4..0000000 --- a/crates/weksa/src/schema.rs +++ /dev/null @@ -1,25 +0,0 @@ -//! JSON Schema publication. JSON is only the xenos-boundary export of the typed -//! documents; nothing in this crate reads JSON. - -use crate::corpus::CorpusItem; -use crate::entity::Entity; -use crate::inventory::{InventoryEntry, InventoryRevision}; -use crate::packet::Packet; -use cultcache_rs::DatabaseEntry; -use schemars::{JsonSchema, schema_for}; - -fn published_for() -> (String, String) { - let json = serde_json::to_string_pretty(&schema_for!(T)).expect("a schema serializes"); - (format!("{}.schema.json", T::TYPE), format!("{json}\n")) -} - -/// `(file name, contents)` for every document type, as `schemas/cultnet/` holds them. -pub fn published() -> Vec<(String, String)> { - vec![ - published_for::(), - published_for::(), - published_for::(), - published_for::(), - published_for::(), - ] -} diff --git a/crates/weksa/src/score/chrf.rs b/crates/weksa/src/score/chrf.rs deleted file mode 100644 index 7650518..0000000 --- a/crates/weksa/src/score/chrf.rs +++ /dev/null @@ -1,60 +0,0 @@ -//! chrF as sacrebleu 2.4.3 computes it by default: character n-gram orders 1 to -//! 6, whitespace removed, case kept, beta 2. - -use std::collections::HashMap; - -/// The metric this module implements; a score stored beside it names its definition. -pub const CHRF_VERSION: &str = "chrF2|nc:6|nw:0|space:no|sacrebleu:2.4.3"; - -const ORDER: usize = 6; -const BETA_SQ: f64 = 4.0; - -fn grams(chars: &[char], n: usize) -> HashMap<&[char], u32> { - let mut counts = HashMap::new(); - for window in chars.windows(n) { - *counts.entry(window).or_insert(0) += 1; - } - counts -} - -/// Python `str.split`'s whitespace, which sacrebleu strips before gramming: -/// Unicode White_Space plus U+001C..=U+001F. -pub(super) fn is_split_space(c: char) -> bool { - c.is_whitespace() || ('\u{1c}'..='\u{1f}').contains(&c) -} - -/// chrF of `hypothesis` against `reference`, 0 to 100. Beta 2 weights recall, -/// so the arguments are not interchangeable. 0 when no order has n-grams on -/// both sides. -pub fn chrf(hypothesis: &str, reference: &str) -> f64 { - let strip = |s: &str| { - s.chars() - .filter(|c| !is_split_space(*c)) - .collect::>() - }; - let (hyp, reference) = (strip(hypothesis), strip(reference)); - let (mut precision, mut recall, mut orders) = (0.0, 0.0, 0u32); - for n in 1..=ORDER { - let (hyp_grams, ref_grams) = (grams(&hyp, n), grams(&reference, n)); - let (hyp_total, ref_total): (u32, u32) = - (hyp_grams.values().sum(), ref_grams.values().sum()); - if hyp_total == 0 || ref_total == 0 { - continue; - } - let matched: u32 = hyp_grams - .iter() - .map(|(gram, count)| (*count).min(ref_grams.get(gram).copied().unwrap_or(0))) - .sum(); - precision += f64::from(matched) / f64::from(hyp_total); - recall += f64::from(matched) / f64::from(ref_total); - orders += 1; - } - if orders == 0 { - return 0.0; - } - let (p, r) = (precision / f64::from(orders), recall / f64::from(orders)); - if p + r == 0.0 { - return 0.0; - } - 100.0 * (1.0 + BETA_SQ) * p * r / (BETA_SQ * p + r) -} diff --git a/crates/weksa/src/score/diagnostics.rs b/crates/weksa/src/score/diagnostics.rs deleted file mode 100644 index 9d91cf2..0000000 --- a/crates/weksa/src/score/diagnostics.rs +++ /dev/null @@ -1,169 +0,0 @@ -//! Leak diagnostics: values reported beside scores, never refusals. Pooled -//! coverage at n >= 2 above zero beside single-word glosses, or a high pairwise -//! chrF, means a flow bug or a corpus duplicate across splits. - -use std::collections::HashSet; - -use super::chrf::chrf; -use super::words::words; -use crate::corpus::CorpusItem; -use crate::split::{Split, SplitScheme}; - -/// Word n-gram orders read, 1 to this. -const ORDERS: usize = 3; - -/// The pair that reached the highest chrF. -#[derive(Clone, Debug, PartialEq)] -pub struct PairChrf { - pub chrf: f64, - pub minted: String, - pub item: String, -} - -/// One word-order's pooled novel coverage over a split's items. -#[derive(Clone, Debug, PartialEq)] -pub struct OrderCoverage { - pub n: usize, - /// Distinct n-grams of the split's items absent from Train, summed over items. - pub novel: usize, - /// Of those, the ones that occur in the minted text. - pub covered: usize, - /// `covered / novel`; `None` when `novel` is 0, never NaN. - pub share: Option, -} - -/// The item that read most at `n` >= 2. -#[derive(Clone, Debug, PartialEq)] -pub struct WorstItem { - pub item: String, - pub n: usize, - pub covered: usize, - pub novel: usize, -} - -/// One held-out split's readings. -#[derive(Clone, Debug, PartialEq, Default)] -pub struct SplitLeak { - /// Items of the split. - pub items: usize, - /// Max chrF of any minted string (hypothesis) against any item (reference). - pub max_chrf: Option, - /// Pooled novel coverage for n in 1..=3, in order. - pub coverage_by_n: Vec, - /// The item and order (n >= 2) with the largest covered count, then the - /// larger share; the first item, then the smaller n, on ties. `None` when - /// no item has a covered n-gram at n >= 2. - pub worst: Option, -} - -#[derive(Clone, Debug, PartialEq, Default)] -pub struct LeakDiagnostic { - pub validation: SplitLeak, - pub test: SplitLeak, -} - -/// The distinct word n-grams of a text for each order 1..=ORDERS, over its -/// normalised word segments. -fn grams(text: &str) -> Vec> { - let keys = words(text); - (1..=ORDERS) - .map(|n| keys.windows(n).map(|w| w.join(" ")).collect()) - .collect() -} - -fn union<'a>(texts: impl Iterator) -> Vec> { - let mut all = vec![HashSet::new(); ORDERS]; - for text in texts { - for (into, from) in all.iter_mut().zip(grams(text)) { - into.extend(from); - } - } - all -} - -fn split_leak( - minted: &[&str], - train: &[HashSet], - minted_grams: &[HashSet], - items: &[&CorpusItem], -) -> SplitLeak { - let mut leak = SplitLeak { - items: items.len(), - ..SplitLeak::default() - }; - let mut totals = [(0usize, 0usize); ORDERS]; - for item in items { - for m in minted { - let score = chrf(m, &item.text); - if leak.max_chrf.as_ref().is_none_or(|best| score > best.chrf) { - leak.max_chrf = Some(PairChrf { - chrf: score, - minted: (*m).to_string(), - item: item.id.clone(), - }); - } - } - for (i, item_grams) in grams(&item.text).iter().enumerate() { - let novel: Vec<&String> = item_grams - .iter() - .filter(|g| !train[i].contains(*g)) - .collect(); - let covered = novel - .iter() - .filter(|g| minted_grams[i].contains(**g)) - .count(); - totals[i].0 += novel.len(); - totals[i].1 += covered; - if i == 0 || covered == 0 { - continue; - } - let share = covered as f64 / novel.len() as f64; - let better = leak.worst.as_ref().is_none_or(|w| { - covered > w.covered - || (covered == w.covered && share > w.covered as f64 / w.novel as f64) - }); - if better { - leak.worst = Some(WorstItem { - item: item.id.clone(), - n: i + 1, - covered, - novel: novel.len(), - }); - } - } - } - leak.coverage_by_n = totals - .iter() - .enumerate() - .map(|(i, (novel, covered))| OrderCoverage { - n: i + 1, - novel: *novel, - covered: *covered, - share: (*novel > 0).then(|| *covered as f64 / *novel as f64), - }) - .collect(); - leak -} - -/// Reads minted text (inventory glosses, option labels, later ontology text) -/// against the held-out items. Per held-out item and for n in 1..=3, its word -/// n-grams that occur in no Train item are its novel n-grams; coverage at each -/// n is the covered over the novel n-grams summed across the split's items -/// (unigrams over all minted strings, longer grams within one string). Honest -/// single-word glosses saturate n = 1, so the leak signature is pooled n >= 2 -/// and the worst item at n >= 2. Train items are not measured; items in no -/// split are dropped. -pub fn leak_diagnostic(minted: &[&str], items: &[CorpusItem]) -> LeakDiagnostic { - let of = |want: Split| -> Vec<&CorpusItem> { - items - .iter() - .filter(|i| SplitScheme::V1.split(i) == Some(want)) - .collect() - }; - let train = union(of(Split::Train).iter().map(|i| i.text.as_str())); - let minted_grams = union(minted.iter().copied()); - LeakDiagnostic { - validation: split_leak(minted, &train, &minted_grams, &of(Split::Validation)), - test: split_leak(minted, &train, &minted_grams, &of(Split::Test)), - } -} diff --git a/crates/weksa/src/score/embed.rs b/crates/weksa/src/score/embed.rs deleted file mode 100644 index 45d1bcf..0000000 --- a/crates/weksa/src/score/embed.rs +++ /dev/null @@ -1,113 +0,0 @@ -//! The meaning metric's embedding port and its Ollama adapter. - -use std::time::Duration; - -use anyhow::{Context, Result, bail, ensure}; -use serde::Deserialize; -use serde_json::json; - -/// Sentence embeddings. Every embedder (Ollama now, a style embedder later) -/// enters the scorer through this port. -pub trait Embedder { - fn model(&self) -> &str; - /// One vector per text, in order, all of one dimension. - fn embed(&self, texts: &[&str]) -> Result>>; -} - -/// Cosine similarity. Precondition: `a` and `b` have one equal nonzero length -/// and nonzero norm; `Scorer::score` refuses embedder output that does not. -/// Violating it panics rather than reading as a score. -pub fn cosine(a: &[f32], b: &[f32]) -> f64 { - assert!( - !a.is_empty() && a.len() == b.len(), - "cosine needs vectors of one nonzero length, got {} and {}", - a.len(), - b.len() - ); - let dot: f64 = a - .iter() - .zip(b) - .map(|(x, y)| f64::from(*x) * f64::from(*y)) - .sum(); - let norm = |v: &[f32]| v.iter().map(|x| f64::from(*x).powi(2)).sum::().sqrt(); - let denom = norm(a) * norm(b); - assert!(denom > 0.0, "cosine of a zero vector"); - dot / denom -} - -/// Ollama's `/api/embed` over plain HTTP. One server serves Huginn and the -/// loop, so batches stay small and `num_thread` is sent (without it a batch of -/// 32 timed out at 120 s under host load). -#[derive(Clone, Debug)] -pub struct OllamaEmbedder { - pub url: String, - pub model: String, - /// Sent as `options.num_thread`; omitted when `None`. - pub num_thread: Option, - /// Texts per request, clamped to 1..=32. - pub batch: usize, -} - -impl OllamaEmbedder { - pub const MAX_BATCH: usize = 32; - - pub fn new(url: impl Into, model: impl Into) -> Self { - Self { - url: url.into(), - model: model.into(), - num_thread: Some(8), - batch: Self::MAX_BATCH, - } - } - - fn request(&self, agent: &ureq::Agent, batch: &[&str]) -> Result>> { - let mut body = json!({ "model": self.model, "input": batch }); - if let Some(threads) = self.num_thread { - body["options"] = json!({ "num_thread": threads }); - } - let endpoint = format!("{}/api/embed", self.url.trim_end_matches('/')); - let mut response = agent - .post(&endpoint) - .header("Content-Type", "application/json") - .send(body.to_string()) - .with_context(|| format!("POST {endpoint}"))?; - #[derive(Deserialize)] - struct Reply { - embeddings: Vec>, - } - let reply: Reply = serde_json::from_str(&response.body_mut().read_to_string()?) - .context("ollama /api/embed reply")?; - ensure!( - reply.embeddings.len() == batch.len(), - "asked for {} embeddings, got {}", - batch.len(), - reply.embeddings.len() - ); - Ok(reply.embeddings) - } -} - -impl Embedder for OllamaEmbedder { - fn model(&self) -> &str { - &self.model - } - - fn embed(&self, texts: &[&str]) -> Result>> { - if texts.is_empty() { - return Ok(Vec::new()); - } - let agent: ureq::Agent = ureq::Agent::config_builder() - .timeout_global(Some(Duration::from_secs(120))) - .build() - .into(); - let mut out: Vec> = Vec::with_capacity(texts.len()); - for batch in texts.chunks(self.batch.clamp(1, Self::MAX_BATCH)) { - out.extend(self.request(&agent, batch)?); - } - let dim = out[0].len(); - if dim == 0 || out.iter().any(|v| v.len() != dim) { - bail!("embeddings disagree on dimension"); - } - Ok(out) - } -} diff --git a/crates/weksa/src/score/mod.rs b/crates/weksa/src/score/mod.rs deleted file mode 100644 index 713dede..0000000 --- a/crates/weksa/src/score/mod.rs +++ /dev/null @@ -1,144 +0,0 @@ -//! What a score counts: chrF and the meaning cosine of the whole lowered -//! output against the whole reference. Nothing is deleted, hidden or replaced -//! before either is read, and the scorer takes no input from a packet, so no -//! text the output carries can raise a score except by resembling the -//! reference. No door judges text; the diagnostics report. - -mod chrf; -mod diagnostics; -mod embed; -mod words; - -use anyhow::{Result, ensure}; - -pub use chrf::{CHRF_VERSION, chrf}; -pub use diagnostics::{ - LeakDiagnostic, OrderCoverage, PairChrf, SplitLeak, WorstItem, leak_diagnostic, -}; -pub use embed::{Embedder, OllamaEmbedder, cosine}; - -use crate::corpus::CorpusItem; -use crate::split::{Split, SplitScheme}; - -/// One reference/output pair; both are scored whole. -pub struct ScoreInput<'a> { - pub reference: &'a str, - pub output: &'a str, -} - -#[derive(Clone, Debug, PartialEq)] -pub struct ItemScore { - /// chrF of the output against the reference. - pub chrf: f64, - /// Embedding cosine of the output and the reference; `None` when either - /// is empty after trimming. - pub meaning: Option, -} - -pub struct Scorer { - embedder: E, -} - -/// Every vector has one nonzero length and a nonzero norm. -fn check_vectors(vectors: &[Vec]) -> Result<()> { - let dim = vectors.first().map_or(0, Vec::len); - for vector in vectors { - ensure!( - dim > 0 && vector.len() == dim, - "embedder returned vectors of length {} and {}", - dim, - vector.len() - ); - let norm: f64 = vector.iter().map(|x| f64::from(*x).powi(2)).sum(); - ensure!( - norm > 0.0 && norm.is_finite(), - "embedder returned a zero or non-finite vector" - ); - } - Ok(()) -} - -impl Scorer { - pub fn new(embedder: E) -> Self { - Self { embedder } - } - - pub fn embedder(&self) -> &E { - &self.embedder - } - - /// Scores every input with one embed call; the caller chunks `inputs`. - /// Errs when the embedder's vectors disagree in length or are empty or zero. - pub fn score(&self, inputs: &[ScoreInput<'_>]) -> Result> { - let mut texts: Vec<&str> = Vec::new(); - let slots: Vec> = inputs - .iter() - .map(|input| { - if input.output.trim().is_empty() || input.reference.trim().is_empty() { - return None; - } - texts.push(input.reference); - texts.push(input.output); - Some(texts.len() - 2) - }) - .collect(); - let vectors = self.embedder.embed(&texts)?; - ensure!( - vectors.len() == texts.len(), - "embedder returned {} vectors for {} texts", - vectors.len(), - texts.len() - ); - check_vectors(&vectors)?; - Ok(inputs - .iter() - .zip(slots) - .map(|(input, slot)| ItemScore { - chrf: chrf(input.output, input.reference), - meaning: slot.map(|i| cosine(&vectors[i], &vectors[i + 1])), - }) - .collect()) - } -} - -/// Mean scores of one derived split. -#[derive(Clone, Debug, PartialEq, Default)] -pub struct SplitMean { - pub count: usize, - /// `None` when the split has no items. - pub chrf: Option, - /// Mean over the items that have a meaning score; `None` when none do. - pub meaning: Option, -} - -#[derive(Clone, Debug, PartialEq, Default)] -pub struct Aggregate { - pub train: SplitMean, - pub validation: SplitMean, - pub test: SplitMean, -} - -/// Groups scores by the item's derived split. Items in no split are counted -/// nowhere. The scorer only groups: the loop decides when Test is read. -pub fn aggregate(scored: &[(&CorpusItem, ItemScore)]) -> Aggregate { - let mean = |want: Split| { - let of: Vec<&ItemScore> = scored - .iter() - .filter(|(item, _)| SplitScheme::V1.split(item) == Some(want)) - .map(|(_, score)| score) - .collect(); - let average = |values: Vec| { - (!values.is_empty()).then(|| values.iter().sum::() / values.len() as f64) - }; - SplitMean { - count: of.len(), - chrf: average(of.iter().map(|s| s.chrf).collect()), - meaning: average(of.iter().filter_map(|s| s.meaning).collect()), - } - }; - Aggregate { - train: mean(Split::Train), - validation: mean(Split::Validation), - test: mean(Split::Test), - } -} diff --git a/crates/weksa/src/score/words.rs b/crates/weksa/src/score/words.rs deleted file mode 100644 index 58e7c1f..0000000 --- a/crates/weksa/src/score/words.rs +++ /dev/null @@ -1,44 +0,0 @@ -//! The one way the scorer reads words: UAX #29 word segmentation, then NFC and -//! full Unicode case folding. The novel-coverage diagnostic reads text through -//! here, so every script is treated by one rule. - -use caseless::default_case_fold_str; -use unicode_normalization::UnicodeNormalization; -use unicode_segmentation::UnicodeSegmentation; - -/// NFC, then full default case folding. -pub(super) fn normalise(s: &str) -> String { - default_case_fold_str(&s.nfc().collect::()) -} - -/// The pieces of one segment: split at `_` and `.` (which are dropped) and -/// at every lowercase-to-uppercase transition, so `kick_the_bucket`, -/// `kick.the.bucket` and `kickTheBucket` read as `kick the bucket`. -fn pieces(segment: &str) -> Vec<&str> { - let mut out = Vec::new(); - let mut start = 0; - let mut previous: Option = None; - for (at, c) in segment.char_indices() { - let separator = c == '_' || c == '.'; - let camel = previous.is_some_and(char::is_lowercase) && c.is_uppercase(); - if separator || camel { - out.push(&segment[start..at]); - start = if separator { at + c.len_utf8() } else { at }; - } - previous = Some(c); - } - out.push(&segment[start..]); - out -} - -/// The normalised words of `text` in order: UAX #29 segments containing an -/// alphanumeric char (digits kept), each split by `pieces`, empty or -/// non-alphanumeric pieces dropped. -pub(super) fn words(text: &str) -> Vec { - text.split_word_bounds() - .filter(|segment| segment.chars().any(char::is_alphanumeric)) - .flat_map(pieces) - .filter(|piece| piece.chars().any(char::is_alphanumeric)) - .map(normalise) - .collect() -} diff --git a/crates/weksa/src/split.rs b/crates/weksa/src/split.rs deleted file mode 100644 index d2258a2..0000000 --- a/crates/weksa/src/split.rs +++ /dev/null @@ -1,54 +0,0 @@ -//! The train/validation/test split is derived from an item's group and nothing -//! else. No stored assignment exists, so no writer can reassign one. - -use crate::corpus::{CorpusItem, Source}; - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum Split { - Train, - Validation, - Test, -} - -/// A scheme is a constant; a new scheme is a new variant, never a reassignment. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum SplitScheme { - V1, -} - -pub fn fnv1a64(bytes: &[u8]) -> u64 { - let mut h: u64 = 0xcbf2_9ce4_8422_2325; - for b in bytes { - h ^= u64::from(*b); - h = h.wrapping_mul(0x0000_0100_0000_01b3); - } - h -} - -impl SplitScheme { - /// 0..100, FNV-1a 64 of `weksa-split-v1:` + group, mod 100. - pub fn bucket(self, group: &str) -> u64 { - match self { - SplitScheme::V1 => fnv1a64(format!("weksa-split-v1:{group}").as_bytes()) % 100, - } - } - - /// <40 Train, <80 Validation, else Test. - pub fn of_bucket(self, bucket: u64) -> Split { - match self { - SplitScheme::V1 if bucket < 40 => Split::Train, - SplitScheme::V1 if bucket < 80 => Split::Validation, - SplitScheme::V1 => Split::Test, - } - } - - /// The split of an item: its group's, except that a model-written item of a - /// test group is in no split. It is never moved to train: a model line of - /// the test meaning would teach the test. - pub fn split(self, item: &CorpusItem) -> Option { - match self.of_bucket(self.bucket(&item.group)) { - Split::Test if matches!(item.source, Source::ModelWritten { .. }) => None, - split => Some(split), - } - } -} diff --git a/crates/weksa/src/store.rs b/crates/weksa/src/store.rs deleted file mode 100644 index 1caa6ac..0000000 --- a/crates/weksa/src/store.rs +++ /dev/null @@ -1,722 +0,0 @@ -//! Admission to the corpus, inventory and packet stores. Each store is a -//! CultCache over one `SingleFileMessagePackBackingStore` file. These functions -//! are the only writers, so the rules S1..S4 hold for every caller. - -use std::collections::BTreeSet; -use std::fmt; -use std::path::Path; - -use caseless::default_case_fold_str; -use cultcache_rs::{CultCache, SingleFileMessagePackBackingStore}; -use unicode_normalization::{UnicodeNormalization, char::is_combining_mark, is_nfc}; -use unicode_segmentation::UnicodeSegmentation; - -use crate::corpus::{CorpusItem, CorpusItemId, Source, item_key, key_slug}; -use crate::entity::{Entity, EntityOrigin}; -use crate::form::{is_attestation, is_joining_only, is_language_tag, is_refused_format}; -use crate::inventory::{Change, InvId, Inventory, InventoryEntry, InventoryRevision, revision_key}; -use crate::packet::{EntityId, Packet}; -use crate::validate::{LoweringInput, Refusal, admissible}; - -/// A broken store rule, named. -#[derive(Clone, Debug, PartialEq, Eq)] -pub enum StoreRefusal { - /// S1: an existing item key re-added with any field different. - S1ItemImmutable { key: String }, - /// S2: a revision that is not latest+1 with parent latest. - S2RevisionChain { expected: u32, got: u32 }, - /// S3: a revision after a frozen one. - S3Frozen { frozen: u32 }, - /// S4: an entry whose introduced_rev is not the revision adding it. - S4IntroducedRev { - id: InvId, - introduced: u32, - revision: u32, - }, - /// S5: a superseding item's group differs from its predecessor's. - S5SupersedeGroup { - key: String, - expected: String, - got: String, - }, - /// S6: an item's normalized text equals another key's. - S6TextHeld { key: String, held_by: String }, - /// A `provenance.corpus_item` key names no item. - UnknownCorpusItem { key: String }, - /// A packet key already holds a different packet. - PacketImmutable { key: String }, - /// An entity id the entity store does not hold: a speaker, a prior-turn - /// speaker, a name span's entity or a referent's entity. - UnknownEntity { entity: EntityId }, - /// A name span with `start >= end`, `end` past the text, or an end off a - /// character boundary. - NameSpanRange { key: String, start: u32, end: u32 }, - /// A name span that starts or ends inside an extended grapheme cluster - /// (UAX #29) of the item's text. - NameSpanSplitsCluster { key: String, start: u32, end: u32 }, - /// An item text or prior-turn text, or an entity form, that is not NFC. - /// Callers normalize at intake; the stores refuse what they were handed. - NotNfc { key: String }, - /// An entity beyond a capacity bound. - EntityBounds { id: EntityId, bound: EntityBound }, - /// Two name spans of one item that overlap. - NameSpanOverlap { key: String }, - /// A name span whose bytes equal none of its entity's forms in the item's - /// language (an entity with no forms in that language included). - NameSpanNotAForm { - key: String, - entity: EntityId, - start: u32, - end: u32, - }, - /// A `put` that drops, reorders or edits a stored language, form or origin. - EntityNotAppendOnly { id: EntityId }, - /// A corpus packet's referent cites an entity that is neither named on its - /// item nor the item's speaker or a prior-turn speaker. - EntityNotOnItem { entity: EntityId, item: String }, - /// A new packet names a corpus item already withdrawn. Packets admitted - /// before the withdrawal stay stored and readable. - WithdrawnCorpusItem { key: String }, - /// Any other malformed document or change. - Malformed { key: String, why: &'static str }, -} - -/// The capacity bound an entity broke. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum EntityBound { - /// A language that is not a BCP 47-form tag (1 to 35 bytes). - LanguageTag, - /// More than 8 languages. - Languages, - /// More than 8 forms in one language. - Forms, - /// A form of more than 64 bytes. - FormBytes, -} - -impl EntityBound { - pub const MAX_LANGUAGES: usize = 8; - pub const MAX_FORMS: usize = 8; - pub const MAX_FORM_BYTES: usize = 64; -} - -#[derive(Debug)] -pub enum StoreError { - Refused(StoreRefusal), - /// The packet broke validator rules. - Invalid(Vec), - Io(anyhow::Error), -} - -impl fmt::Display for StoreError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - StoreError::Refused(r) => write!(f, "refused: {r:?}"), - StoreError::Invalid(rs) => { - write!(f, "invalid packet:")?; - rs.iter().try_for_each(|r| write!(f, " [{r}]")) - } - StoreError::Io(e) => write!(f, "{e:#}"), - } - } -} - -impl std::error::Error for StoreError {} - -impl From for StoreError { - fn from(e: anyhow::Error) -> Self { - StoreError::Io(e) - } -} - -impl From for StoreError { - fn from(r: StoreRefusal) -> Self { - StoreError::Refused(r) - } -} - -fn malformed(key: &str, why: &'static str) -> StoreError { - StoreRefusal::Malformed { - key: key.to_string(), - why, - } - .into() -} - -fn open_cache( - path: &Path, - register: impl FnOnce(&mut CultCache) -> anyhow::Result<()>, -) -> anyhow::Result { - if let Some(dir) = path.parent() { - std::fs::create_dir_all(dir)?; - } - let mut cache = CultCache::new(); - register(&mut cache)?; - cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(path))?; - cache.pull_all_backing_stores()?; - Ok(cache) -} - -#[derive(Debug, PartialEq, Eq)] -pub enum AddOutcome { - Added, - /// The key already held this text; nothing was written. - Unchanged, -} - -pub struct CorpusStore { - cache: CultCache, -} - -impl CorpusStore { - pub fn open(path: &Path) -> anyhow::Result { - Ok(Self { - cache: open_cache(path, |c| c.register_entry_type::())?, - }) - } - - pub fn all(&self) -> anyhow::Result> { - let mut items = self.cache.get_all::()?; - items.sort_by(|a, b| a.id.cmp(&b.id)); - Ok(items) - } - - pub fn get(&self, key: &str) -> anyhow::Result> { - self.cache.get::(key) - } - - /// The next free key for `slug`; withdrawn items keep their numbers. - pub fn next_key(&self, slug: &str) -> Result { - let highest = self - .all()? - .iter() - .filter(|i| key_slug(&i.id) == Some(slug)) - .filter_map(|i| i.id.rsplit_once(':')?.1.parse::().ok()) - .max() - .unwrap_or(0); - let next = highest - .checked_add(1) - .ok_or_else(|| malformed(slug, "slug has no free key"))?; - Ok(item_key(slug, next)) - } - - /// The item `id` names, or `UnknownCorpusItem` when it holds nothing. - fn resolve(&self, id: &CorpusItemId) -> Result { - self.get(id.as_str())?.ok_or_else(|| { - StoreRefusal::UnknownCorpusItem { - key: id.to_string(), - } - .into() - }) - } - - /// S1: an item key holds its whole item for good. S6: a text held under - /// one key (live or withdrawn) is not re-added under another unless the new - /// item supersedes that key. S5: a correction stays in its predecessor's - /// group, so it cannot move across the split. - pub fn add( - &mut self, - item: CorpusItem, - entities: &EntityStore, - ) -> Result { - if key_slug(&item.id).is_none() { - return Err(malformed(&item.id, "item key is not item::")); - } - if item.withdrawn { - return Err(malformed( - &item.id, - "an item is admitted live, then withdrawn", - )); - } - if let Some(existing) = self.get(&item.id)? { - if existing == item { - return Ok(AddOutcome::Unchanged); - } - return Err(StoreRefusal::S1ItemImmutable { key: item.id }.into()); - } - if !is_language_tag(&item.language) { - return Err(malformed(&item.id, "language is not a BCP 47-form tag")); - } - if !is_nfc(&item.text) || item.prior_turns.iter().any(|t| !is_nfc(&t.text)) { - return Err(StoreRefusal::NotNfc { key: item.id }.into()); - } - let text = normalized(&item.text); - if !text.chars().any(char::is_alphanumeric) { - return Err(malformed(&item.id, "text has no alphanumeric character")); - } - if let Some(why) = empty_attestation(&item.source) { - return Err(malformed(&item.id, why)); - } - check_entities(&item, entities)?; - for held in self.all()? { - if normalized(&held.text) == text && item.supersedes.as_deref() != Some(&held.id) { - return Err(StoreRefusal::S6TextHeld { - key: item.id, - held_by: held.id, - } - .into()); - } - } - if let Some(old) = &item.supersedes { - let Some(old) = self.get(old)? else { - return Err(malformed( - &item.id, - "supersedes an item that does not exist", - )); - }; - if old.group != item.group { - return Err(StoreRefusal::S5SupersedeGroup { - key: item.id, - expected: old.group, - got: item.group, - } - .into()); - } - } - self.cache.put(item.id.clone(), &item)?; - Ok(AddOutcome::Added) - } - - /// Marks an item withdrawn; its text stays. - pub fn withdraw(&mut self, key: &str) -> Result<(), StoreError> { - let Some(mut item) = self.get(key)? else { - return Err(malformed(key, "no such item")); - }; - if !item.withdrawn { - item.withdrawn = true; - self.cache.put(key, &item)?; - } - Ok(()) - } -} - -/// Every entity an item cites exists, and every name span is a well-formed, -/// non-overlapping byte range of the text, starting and ending on extended -/// grapheme cluster boundaries, equal to one of its entity's forms in the -/// item's language. Id lookup, UAX #29 boundaries and byte equality: no reading -/// of meaning, so every script is held to the same check. -fn check_entities(item: &CorpusItem, entities: &EntityStore) -> Result<(), StoreError> { - let known = |id: &EntityId| -> Result { - entities - .get(id)? - .ok_or_else(|| StoreRefusal::UnknownEntity { entity: id.clone() }.into()) - }; - known(&item.speaker)?; - for turn in &item.prior_turns { - known(&turn.speaker)?; - } - let mut ranges = Vec::with_capacity(item.names.len()); - let cuts: BTreeSet = item - .text - .grapheme_indices(true) - .map(|(i, _)| i) - .chain([item.text.len()]) - .collect(); - for span in &item.names { - let entity = known(&span.entity)?; - let (start, end) = (span.start as usize, span.end as usize); - let Some(bytes) = item - .text - .get(start..end) - .filter(|_| start < end) - .map(str::as_bytes) - else { - return Err(StoreRefusal::NameSpanRange { - key: item.id.clone(), - start: span.start, - end: span.end, - } - .into()); - }; - if !cuts.contains(&start) || !cuts.contains(&end) { - return Err(StoreRefusal::NameSpanSplitsCluster { - key: item.id.clone(), - start: span.start, - end: span.end, - } - .into()); - } - let is_form = entity - .forms_in(&item.language) - .is_some_and(|forms| forms.iter().any(|f| f.as_bytes() == bytes)); - if !is_form { - return Err(StoreRefusal::NameSpanNotAForm { - key: item.id.clone(), - entity: span.entity.clone(), - start: span.start, - end: span.end, - } - .into()); - } - ranges.push((span.start, span.end)); - } - ranges.sort_unstable(); - if ranges.windows(2).any(|w| w[1].0 < w[0].1) { - return Err(StoreRefusal::NameSpanOverlap { - key: item.id.clone(), - } - .into()); - } - Ok(()) -} - -/// The S6 key of a text: Unicode compatibility caseless matching (D146: NFKC, -/// default case fold, NFKC) keeping letters, digits and combining marks -/// (General_Category L, N, M) so tone marks and viramas keep words apart. Composed and decomposed, -/// fullwidth, ligature and case-fold variants (straße, STRASSE) and spacing and -/// punctuation variants compare equal. Homoglyphs across scripts (Cyrillic a for -/// Latin a) stay apart: no key here reads what a character looks like. -fn normalized(text: &str) -> String { - let folded = default_case_fold_str(&text.nfkc().collect::()); - folded - .nfkc() - .filter(|c| c.is_alphanumeric() || is_combining_mark(*c)) - .collect() -} - -/// The field that holds an unattested provenance string, if any: a string with -/// no alphanumeric, a control or format character, or over 200 bytes attests -/// nothing. -fn empty_attestation(source: &Source) -> Option<&'static str> { - let empty = |s: &String| !is_attestation(s); - let empty_some = |s: &Option| s.as_ref().is_some_and(empty); - match source { - Source::CastAuthored { persona, .. } if empty(persona) => { - Some("a cast-authored source names its persona") - } - Source::CastAuthored { fixture, .. } if empty(fixture) => { - Some("a cast-authored source names its fixture") - } - Source::CastAuthored { author, .. } if empty(author) => { - Some("a cast-authored source names its author") - } - Source::OperatorApproved { note } if empty(note) => { - Some("an operator-approved source carries its note") - } - Source::PublicDomain { work, .. } if empty(work) => { - Some("a public-domain source names its work") - } - Source::PublicDomain { author, .. } if empty(author) => { - Some("a public-domain source names its author") - } - Source::PublicDomain { edition_url, .. } if empty(edition_url) => { - Some("a public-domain source names its edition url") - } - Source::ModelWritten { generator, .. } if empty_some(generator) => { - Some("a model-written source names a non-empty generator") - } - Source::ModelWritten { prompt_digest, .. } if empty_some(prompt_digest) => { - Some("a model-written source carries a non-empty prompt digest") - } - _ => None, - } -} - -pub struct EntityStore { - cache: CultCache, -} - -impl EntityStore { - pub fn open(path: &Path) -> anyhow::Result { - Ok(Self { - cache: open_cache(path, |c| c.register_entry_type::())?, - }) - } - - pub fn all(&self) -> anyhow::Result> { - let mut entities = self.cache.get_all::()?; - entities.sort_by(|a, b| a.id.cmp(&b.id)); - Ok(entities) - } - - pub fn get(&self, id: &EntityId) -> anyhow::Result> { - self.cache.get::(id.as_str()) - } - - /// Admits a new entity, an identical replay, or a replacement that keeps - /// every stored language and form in order and origin as it was, and only - /// appends: forms grow, never shrink, reorder or change. An entity has 1 to - /// 8 languages, each a BCP 47-form tag named once, with 1 to 8 NFC forms of - /// 1 to 64 bytes, no control or invisible format character (joiners and - /// variation selectors stay), no edge whitespace, none repeated; the origin - /// attests where it came from with strings that hold an alphanumeric, no control or format character and at most 200 bytes. - pub fn put(&mut self, entity: Entity) -> Result { - let key = entity.id.to_string(); - check_entity_shape(&entity)?; - if let Some(old) = self.get(&entity.id)? { - if old == entity { - return Ok(AddOutcome::Unchanged); - } - let grows = old.origin == entity.origin - && old.forms.len() <= entity.forms.len() - && old.forms.iter().zip(&entity.forms).all(|(o, n)| { - o.language == n.language - && o.forms.len() <= n.forms.len() - && n.forms.starts_with(&o.forms) - }); - if !grows { - return Err(StoreRefusal::EntityNotAppendOnly { id: entity.id }.into()); - } - } - self.cache.put(key, &entity)?; - Ok(AddOutcome::Added) - } -} - -fn check_entity_shape(entity: &Entity) -> Result<(), StoreError> { - let bad = |why| malformed(entity.id.as_str(), why); - let bound = |bound| -> StoreError { - StoreRefusal::EntityBounds { - id: entity.id.clone(), - bound, - } - .into() - }; - let (persona, fixture, work) = match &entity.origin { - EntityOrigin::Cast { persona, fixture } => (Some(persona), Some(fixture), None), - EntityOrigin::Live { persona } => (Some(persona), None, None), - EntityOrigin::PublicDomain { work } => (None, None, Some(work)), - }; - if persona.is_some_and(|s| !is_attestation(s)) { - return Err(bad("an origin names its persona")); - } - if fixture.is_some_and(|s| !is_attestation(s)) { - return Err(bad("a cast origin names its fixture")); - } - if work.is_some_and(|s| !is_attestation(s)) { - return Err(bad("a public-domain origin names its work")); - } - if entity.forms.is_empty() { - return Err(bad("an entity has forms in at least one language")); - } - if entity.forms.len() > EntityBound::MAX_LANGUAGES { - return Err(bound(EntityBound::Languages)); - } - let mut languages = BTreeSet::new(); - for lf in &entity.forms { - if !is_language_tag(&lf.language) { - return Err(bound(EntityBound::LanguageTag)); - } - if !languages.insert(&lf.language) { - return Err(bad("a language is named once")); - } - if lf.forms.is_empty() { - return Err(bad("a language has at least one form")); - } - if lf.forms.len() > EntityBound::MAX_FORMS { - return Err(bound(EntityBound::Forms)); - } - let mut seen = BTreeSet::new(); - for f in &lf.forms { - if f.len() > EntityBound::MAX_FORM_BYTES { - return Err(bound(EntityBound::FormBytes)); - } - if !is_nfc(f) { - return Err(StoreRefusal::NotNfc { - key: entity.id.to_string(), - } - .into()); - } - if f.is_empty() || f.chars().any(char::is_control) || f.trim() != f.as_str() { - return Err(bad( - "a form is not empty and has no control character or edge whitespace", - )); - } - if f.chars().any(is_refused_format) { - return Err(bad("a form holds an invisible format character")); - } - if f.chars().all(is_joining_only) { - return Err(bad("a form is more than joiners and selectors")); - } - if !seen.insert(f) { - return Err(bad("a form is not repeated within its language")); - } - } - } - Ok(()) -} - -pub struct InventoryStore { - cache: CultCache, -} - -impl InventoryStore { - pub fn open(path: &Path) -> anyhow::Result { - let cache = open_cache(path, |c| { - c.register_entry_type::()?; - c.register_entry_type::() - })?; - Ok(Self { cache }) - } - - pub fn inventory(&self) -> anyhow::Result { - Ok(Inventory::new( - self.cache.get_all::()?, - self.cache.get_all::()?, - )) - } - - /// Admits revision `rev` and the entries it adds, whole or not at all. - /// S2: latest+1 with parent latest. S3: nothing follows a frozen revision. - /// S4: an added entry's introduced_rev is this revision. - /// An added entry's gloss is 1 to 120 bytes without a control character. - /// No door judges text against the corpus: leak is owned by the loop's - /// context builder and the scorer. - pub fn admit_revision( - &mut self, - rev: InventoryRevision, - entries: Vec, - ) -> Result<(), StoreError> { - let inv = self.inventory()?; - let latest = inv.latest(); - if let Some(l) = latest.filter(|l| l.frozen) { - return Err(StoreRefusal::S3Frozen { frozen: l.n }.into()); - } - let (expected, parent) = latest.map_or((1, None), |l| (l.n + 1, Some(l.n))); - if rev.n != expected || rev.parent != parent { - return Err(StoreRefusal::S2RevisionChain { - expected, - got: rev.n, - } - .into()); - } - let mut touched: BTreeSet = BTreeSet::new(); - let mut staged = Vec::new(); - for change in &rev.changes { - match change { - Change::Add(id) => { - let Some(entry) = entries.iter().find(|e| &e.id == id) else { - return Err(malformed(&id.to_string(), "added id has no entry")); - }; - if entry.introduced_rev != rev.n { - let (id, introduced, revision) = (*id, entry.introduced_rev, rev.n); - return Err(StoreRefusal::S4IntroducedRev { - id, - introduced, - revision, - } - .into()); - } - let key = id.to_string(); - if !entry.well_formed() || entry.retired_rev.is_some() { - return Err(malformed(&key, "entry shape does not match its family")); - } - if inv.entry(id).is_some() || !touched.insert(*id) { - return Err(malformed(&key, "id is already in the inventory")); - } - check_entry_text(entry)?; - staged.push(self.cache.prepare_entry_named(key, entry)?.0); - } - Change::Retire(id) => { - let key = id.to_string(); - let Some(current) = inv.entry(id).filter(|e| e.retired_rev.is_none()) else { - return Err(malformed( - &key, - "retire target is missing or already retired", - )); - }; - if !touched.insert(*id) { - return Err(malformed(&key, "id changes twice in one revision")); - } - let retired = InventoryEntry { - retired_rev: Some(rev.n), - ..current.clone() - }; - staged.push(self.cache.prepare_entry_named(key, &retired)?.0); - } - } - } - if let Some(stray) = entries.iter().find(|e| !touched.contains(&e.id)) { - return Err(malformed( - &stray.id.to_string(), - "entry is not in the revision's changes", - )); - } - staged.push(self.cache.prepare_entry_named(revision_key(rev.n), &rev)?.0); - self.cache.put_prepared_batch(staged)?; - Ok(()) - } -} - -fn check_entry_text(entry: &InventoryEntry) -> Result<(), StoreError> { - if !(1..=120).contains(&entry.gloss.len()) || entry.gloss.chars().any(char::is_control) { - return Err(malformed( - &entry.id.to_string(), - "gloss is 1 to 120 bytes with no control character", - )); - } - Ok(()) -} - -pub struct PacketStore { - cache: CultCache, -} - -impl PacketStore { - pub fn open(path: &Path) -> anyhow::Result { - Ok(Self { - cache: open_cache(path, |c| c.register_entry_type::())?, - }) - } - - pub fn all(&self) -> anyhow::Result> { - let mut packets = self.cache.get_all::()?; - packets.sort_by(|a, b| a.id.cmp(&b.id)); - Ok(packets) - } - - pub fn get(&self, key: &str) -> anyhow::Result> { - self.cache.get::(key) - } - - /// The only door, and the only source of a [`LoweringInput`]: a packet - /// exists only if the validator's rules pass against the inventory the - /// store holds at the packet's revision, its `corpus_item` resolves to an - /// item that is not withdrawn, every referent's entity is in the entity store and is named on - /// that item or is its speaker or a prior-turn speaker (an id-set check). A - /// key holds its packet for good. A packet that names no corpus item has no - /// door here. - pub fn admit( - &mut self, - packet: Packet, - inventory: &InventoryStore, - corpus: &CorpusStore, - entities: &EntityStore, - ) -> Result { - let view = admissible(&packet, &inventory.inventory()?).map_err(StoreError::Invalid)?; - let item = corpus.resolve(&packet.provenance.corpus_item)?; - if item.withdrawn { - return Err(StoreRefusal::WithdrawnCorpusItem { - key: item.id.clone(), - } - .into()); - } - for entity in packet.referents.iter().filter_map(|r| r.entity.as_ref()) { - if entities.get(entity)?.is_none() { - return Err(StoreRefusal::UnknownEntity { - entity: entity.clone(), - } - .into()); - } - let on_item = item.names.iter().any(|n| &n.entity == entity) - || &item.speaker == entity - || item.prior_turns.iter().any(|t| &t.speaker == entity); - if !on_item { - return Err(StoreRefusal::EntityNotOnItem { - entity: entity.clone(), - item: item.id.clone(), - } - .into()); - } - } - let key = packet.id.to_string(); - if let Some(existing) = self.get(&key)? { - if existing == packet { - return Ok(view); - } - return Err(StoreRefusal::PacketImmutable { key }.into()); - } - self.cache.put(key, &packet)?; - Ok(view) - } -} diff --git a/crates/weksa/src/validate.rs b/crates/weksa/src/validate.rs deleted file mode 100644 index 7fbf6aa..0000000 --- a/crates/weksa/src/validate.rs +++ /dev/null @@ -1,525 +0,0 @@ -//! The validator owns the inventory and form rules a packet must meet. Every -//! path that admits a packet applies them, but `validate` only judges: it -//! returns `Ok(())` or the refusals. [`LoweringInput`], the only thing a -//! lowerer sees, comes from `PacketStore::admit` alone, which builds it through -//! the crate-private `admissible`. The stance is checked like every feature -//! list, in slot `Stance`. The rules are V1..V8 and V10..V12: the shape of an id -//! or a grade is its type's parse, not a rule, so V9 is retired and not -//! renumbered. -//! -//! `validate` owns form and capacity and makes no leak decision. A packet's one -//! free text is its trace (`notes` and `rejected_readings`): `validate` does not -//! bound or read it, and [`LoweringInput`] drops it, so no lowerer sees it. -//! Every other slot is an id type, so the lists are the bounded part. What bounds -//! each list (V11, V12): a feature list holds at -//! most one value per feature id and `extensions` at most one per extension id, -//! so each is at most the inventory's ids of that kind; `participants`, -//! `audience` and `preserve` hold each node id at most once and every id names -//! a node (V4), so each is at most `Packet::MAX_NODES`; a predication's `roles` -//! hold each role id at most once, so at most the inventory's roles; and the -//! nodes themselves are at most `Packet::MAX_NODES` (V12). Whether a score may -//! rise because corpus text reached a packet is owned by the golf loop's -//! context builder and the scorer (ruling `leak-owned-by-flow-and-scorer-3`), -//! never by a text rule here, in `admit_revision` or in `PacketStore::admit`. - -use std::collections::BTreeSet; -use std::fmt; - -use crate::inventory::{Family, InvId, Inventory, InventoryEntry, Slot, ValueKind}; -use crate::packet::{ - Constraints, Context, Discourse, Extension, FeatValue, FeatureValue, NodeId, Packet, - PacketKind, Predication, Referent, Relation, SCHEMA_VERSION, Stance, -}; - -/// A broken rule, named. -#[derive(Clone, Debug, PartialEq, Eq)] -pub enum Refusal { - /// V1: the packet's schema_version is not the one these types define. - V1SchemaVersion { found: u32 }, - /// V2: no referents, predications or relations. - V2PayloadRequired, - /// V3: a node id repeats across referents and predications. - V3DuplicateNode { id: NodeId }, - /// V4: a reference names no node. - V4DanglingRef { id: NodeId, at: &'static str }, - /// V5: an inventory id does not resolve at the packet's revision. - V5Unresolved { id: InvId }, - /// V6: an inventory id is of the wrong family for its slot. - V6WrongFamily { - id: InvId, - expected: Family, - at: &'static str, - }, - /// V7: a feature or extension value does not fit its value kind. - V7ValueKind { what: String }, - /// V8: a feature is not declared for the slot it sits on. - V8NotApplicable { feature: InvId, slot: Slot }, - /// V10: the packet's inventory revision has no record. - V10UnknownRevision { rev: u32 }, - /// V11: a feature id repeats in one slot's list (`slot` is `None` for an - /// extension id repeated in the packet). - V11DuplicateFeature { feature: InvId, slot: Option }, - /// V11: a node id repeats in `participants`, `audience` or `preserve` - /// (`at` names the list). - V11DuplicateNodeRef { id: NodeId, at: &'static str }, - /// V11: a role id repeats in one predication's roles. - V11DuplicateRole { role: InvId, predication: NodeId }, - /// V12: more referents, predications and relations than `Packet::MAX_NODES`. - V12TooManyNodes { count: usize, max: usize }, -} - -impl Refusal { - pub fn rule(&self) -> &'static str { - match self { - Refusal::V1SchemaVersion { .. } => "V1", - Refusal::V2PayloadRequired => "V2", - Refusal::V3DuplicateNode { .. } => "V3", - Refusal::V4DanglingRef { .. } => "V4", - Refusal::V5Unresolved { .. } => "V5", - Refusal::V6WrongFamily { .. } => "V6", - Refusal::V7ValueKind { .. } => "V7", - Refusal::V8NotApplicable { .. } => "V8", - Refusal::V10UnknownRevision { .. } => "V10", - Refusal::V11DuplicateFeature { .. } - | Refusal::V11DuplicateNodeRef { .. } - | Refusal::V11DuplicateRole { .. } => "V11", - Refusal::V12TooManyNodes { .. } => "V12", - } - } -} - -impl fmt::Display for Refusal { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}: {self:?}", self.rule()) - } -} - -/// Whether `packet` meets the rules under `inventory`. All broken rules are -/// returned. A pass carries no lowering view: that comes from -/// `PacketStore::admit` only. -/// -/// ``` -/// use weksa::{Inventory, Refusal, packet::Packet}; -/// fn judge(p: &Packet, i: &Inventory) -> Result<(), Vec> { weksa::validate(p, i) } -/// ``` -/// -/// ```compile_fail -/// use weksa::{Inventory, LoweringInput, Refusal, packet::Packet}; -/// fn lower(p: &Packet, i: &Inventory) -> Result> { weksa::validate(p, i) } -/// ``` -pub fn validate(packet: &Packet, inventory: &Inventory) -> Result<(), Vec> { - admissible(packet, inventory).map(|_| ()) -} - -/// The one implementation of the rules. Crate-private: `PacketStore::admit` is -/// the only caller that keeps the view. -/// -/// The first block is the same call through the public name, so the second -/// fails for the private path and nothing else. -/// -/// ``` -/// use weksa::{Inventory, packet::Packet}; -/// fn judge(p: &Packet, i: &Inventory) { let _ = weksa::validate::validate(p, i); } -/// ``` -/// -/// ```compile_fail -/// use weksa::{Inventory, packet::Packet}; -/// fn lower(p: &Packet, i: &Inventory) { let _ = weksa::validate::admissible(p, i); } -/// ``` -pub(crate) fn admissible( - packet: &Packet, - inventory: &Inventory, -) -> Result> { - let mut out = Vec::new(); - if packet.schema_version != SCHEMA_VERSION { - out.push(Refusal::V1SchemaVersion { - found: packet.schema_version, - }); - } - if packet.referents.is_empty() && packet.predications.is_empty() && packet.relations.is_empty() - { - out.push(Refusal::V2PayloadRequired); - } - let count = packet.referents.len() + packet.predications.len() + packet.relations.len(); - if count > Packet::MAX_NODES { - out.push(Refusal::V12TooManyNodes { - count, - max: Packet::MAX_NODES, - }); - } - let nodes = check_nodes(packet, &mut out); - check_refs(packet, &nodes, &mut out); - check_repeats(packet, &mut out); - if inventory.has_revision(packet.inventory_rev) { - Inventoried { - inv: inventory, - rev: packet.inventory_rev, - out: &mut out, - } - .check(packet); - } else { - out.push(Refusal::V10UnknownRevision { - rev: packet.inventory_rev, - }); - } - if out.is_empty() { - Ok(LoweringInput::of(packet)) - } else { - Err(out) - } -} - -fn check_nodes(packet: &Packet, out: &mut Vec) -> BTreeSet { - let mut nodes = BTreeSet::new(); - let ids = packet - .referents - .iter() - .map(|r| &r.id) - .chain(packet.predications.iter().map(|p| &p.id)); - for id in ids { - if !nodes.insert(id.clone()) { - out.push(Refusal::V3DuplicateNode { id: id.clone() }); - } - } - nodes -} - -fn check_refs(packet: &Packet, nodes: &BTreeSet, out: &mut Vec) { - let mut need = |id: &NodeId, at: &'static str| { - if !nodes.contains(id) { - out.push(Refusal::V4DanglingRef { id: id.clone(), at }); - } - }; - for p in &packet.predications { - for r in &p.roles { - need(&r.filler, "role filler"); - } - } - for r in &packet.relations { - need(&r.from, "relation from"); - need(&r.to, "relation to"); - } - if let Some(a) = &packet.discourse.addressee { - need(a, "addressee"); - } - for (list, at) in node_lists(packet) { - for id in list { - need(id, at); - } - } -} - -fn node_lists(packet: &Packet) -> [(&Vec, &'static str); 3] { - [ - (&packet.context.participants, "participant"), - (&packet.context.audience, "audience"), - (&packet.constraints.preserve, "preserve"), - ] -} - -/// V11 for the lists that hold nodes and roles: each node id once per node list, -/// each role id once per predication. -fn check_repeats(packet: &Packet, out: &mut Vec) { - for (list, at) in node_lists(packet) { - let mut seen = BTreeSet::new(); - for id in list { - if !seen.insert(id) { - out.push(Refusal::V11DuplicateNodeRef { id: id.clone(), at }); - } - } - } - for p in &packet.predications { - let mut seen = BTreeSet::new(); - for r in &p.roles { - if !seen.insert(r.role) { - out.push(Refusal::V11DuplicateRole { - role: r.role, - predication: p.id.clone(), - }); - } - } - } -} - -/// The inventory-dependent rules V5..V8 and V11, read at the packet's revision. -struct Inventoried<'a> { - inv: &'a Inventory, - rev: u32, - out: &'a mut Vec, -} - -impl<'a> Inventoried<'a> { - fn check(&mut self, p: &Packet) { - let Packet { - context, - discourse, - referents, - predications, - relations, - extensions, - stance, - .. - } = p; - let Context { - setting, medium, .. - } = context; - for id in setting.iter().chain(medium) { - self.slot(id, Family::Concept, "context concept"); - } - let Discourse { - speech_act, - features, - .. - } = discourse; - if let Some(id) = speech_act { - self.slot(id, Family::SpeechAct, "speech act"); - } - self.features(features, Slot::Discourse); - self.features(&stance.features, Slot::Stance); - for Referent { - kind, - concept, - features, - .. - } in referents - { - self.slot(kind, Family::ReferentKind, "referent kind"); - self.slot(concept, Family::Concept, "referent concept"); - self.features(features, Slot::Referent); - } - for Predication { - concept, - frame, - roles, - features, - .. - } in predications - { - self.slot(concept, Family::Concept, "predication concept"); - self.slot(frame, Family::Frame, "frame"); - for r in roles { - self.slot(&r.role, Family::Role, "role"); - } - self.features(features, Slot::Predication); - } - for Relation { - relation, features, .. - } in relations - { - self.slot(relation, Family::Relation, "relation"); - self.features(features, Slot::Relation); - } - let mut seen = BTreeSet::new(); - for Extension { key, value } in extensions { - if !seen.insert(*key) { - self.out.push(Refusal::V11DuplicateFeature { - feature: *key, - slot: None, - }); - } - if let Some(entry) = self.slot(key, Family::Extension, "extension") - && !fits(entry.value_kind.as_ref(), value) - { - self.out.push(Refusal::V7ValueKind { - what: key.to_string(), - }); - } - } - } - - /// V5, then V6 when it resolved. - fn slot(&mut self, id: &InvId, want: Family, at: &'static str) -> Option<&'a InventoryEntry> { - let entry = self.resolve(id)?; - if entry.family() != want { - self.out.push(Refusal::V6WrongFamily { - id: *id, - expected: want, - at, - }); - return None; - } - Some(entry) - } - - fn resolve(&mut self, id: &InvId) -> Option<&'a InventoryEntry> { - let found = self.inv.resolve(id, self.rev); - if found.is_none() { - self.out.push(Refusal::V5Unresolved { id: *id }); - } - found - } - - fn features(&mut self, features: &[FeatureValue], slot: Slot) { - let mut seen = BTreeSet::new(); - for f in features { - if !seen.insert(f.feature) { - self.out.push(Refusal::V11DuplicateFeature { - feature: f.feature, - slot: Some(slot), - }); - } - let Some(entry) = self.slot(&f.feature, Family::Feature, "feature") else { - continue; - }; - if !entry.applies_to.contains(&slot) { - self.out.push(Refusal::V8NotApplicable { - feature: f.feature, - slot, - }); - } - if !fits(entry.value_kind.as_ref(), &f.value) { - self.out.push(Refusal::V7ValueKind { - what: f.feature.to_string(), - }); - } - } - } -} - -fn fits(kind: Option<&ValueKind>, value: &FeatValue) -> bool { - match (kind, value) { - (Some(ValueKind::Grade), FeatValue::Grade(_)) => true, - (Some(ValueKind::Options(labels)), FeatValue::Option(l)) => labels.contains(l), - (Some(ValueKind::Flag), FeatValue::Flag(_)) => true, - _ => false, - } -} - -/// What the lowerer sees: a validated packet's meaning, with no packet id, no -/// provenance (so no corpus item key, encoder or source) and no trace. Its -/// fields are private and the constructor is crate-private, called only by -/// `PacketStore::admit`, so a lowerer cannot be handed an unadmitted packet or -/// the key to an answer text. -/// -/// Each field has a doctest that reads it from outside the crate. A read -/// compiles exactly when the field is `pub`, so the doctest (which must fail to -/// compile) fails as soon as any one field is made public. A struct literal -/// cannot pin this: it fails to compile while any other field stays private, -/// and stable rustdoc does not check the error code. -/// -/// ```compile_fail -/// fn read(l: weksa::LoweringInput) { let _ = l.inventory_rev; } -/// ``` -/// -/// ```compile_fail -/// fn read(l: weksa::LoweringInput) { let _ = l.kind; } -/// ``` -/// -/// ```compile_fail -/// fn read(l: weksa::LoweringInput) { let _ = l.context; } -/// ``` -/// -/// ```compile_fail -/// fn read(l: weksa::LoweringInput) { let _ = l.discourse; } -/// ``` -/// -/// ```compile_fail -/// fn read(l: weksa::LoweringInput) { let _ = l.referents; } -/// ``` -/// -/// ```compile_fail -/// fn read(l: weksa::LoweringInput) { let _ = l.predications; } -/// ``` -/// -/// ```compile_fail -/// fn read(l: weksa::LoweringInput) { let _ = l.relations; } -/// ``` -/// -/// ```compile_fail -/// fn read(l: weksa::LoweringInput) { let _ = l.constraints; } -/// ``` -/// -/// ```compile_fail -/// fn read(l: weksa::LoweringInput) { let _ = l.extensions; } -/// ``` -/// -/// ```compile_fail -/// fn read(l: weksa::LoweringInput) { let _ = l.stance; } -/// ``` -/// -/// The constructor is private: a call from outside the crate compiles only if -/// `LoweringInput::of` is `pub`. The first block is the same code without the -/// call, so the second fails for that call and nothing else. -/// -/// ``` -/// fn build(p: &weksa::packet::Packet) { let _ = p; } -/// ``` -/// -/// ```compile_fail -/// fn build(p: &weksa::packet::Packet) { let _ = weksa::LoweringInput::of(p); } -/// ``` -/// -/// A lowering view is never read from bytes: `Deserialize` would let a caller -/// build one without `validate`. The first block proves the bound resolves for -/// a type that does implement it. -/// -/// ``` -/// fn de() {} -/// de::(); -/// ``` -/// -/// ```compile_fail -/// fn de() {} -/// de::(); -/// ``` -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct LoweringInput { - inventory_rev: u32, - kind: PacketKind, - context: Context, - discourse: Discourse, - referents: Vec, - predications: Vec, - relations: Vec, - constraints: Constraints, - extensions: Vec, - stance: Stance, -} - -impl LoweringInput { - fn of(p: &Packet) -> Self { - Self { - inventory_rev: p.inventory_rev, - kind: p.kind, - context: p.context.clone(), - discourse: p.discourse.clone(), - referents: p.referents.clone(), - predications: p.predications.clone(), - relations: p.relations.clone(), - constraints: p.constraints.clone(), - extensions: p.extensions.clone(), - stance: p.stance.clone(), - } - } - - pub fn inventory_rev(&self) -> u32 { - self.inventory_rev - } - pub fn kind(&self) -> PacketKind { - self.kind - } - pub fn context(&self) -> &Context { - &self.context - } - pub fn discourse(&self) -> &Discourse { - &self.discourse - } - pub fn referents(&self) -> &[Referent] { - &self.referents - } - pub fn predications(&self) -> &[Predication] { - &self.predications - } - pub fn relations(&self) -> &[Relation] { - &self.relations - } - pub fn constraints(&self) -> &Constraints { - &self.constraints - } - pub fn extensions(&self) -> &[Extension] { - &self.extensions - } - pub fn stance(&self) -> &Stance { - &self.stance - } -} diff --git a/crates/weksa/tests/cli.rs b/crates/weksa/tests/cli.rs deleted file mode 100644 index e05b24a..0000000 --- a/crates/weksa/tests/cli.rs +++ /dev/null @@ -1,869 +0,0 @@ -mod common; -use std::process::{Command, Output}; -use weksa::{CorpusStore, EntityStore, InventoryStore, PacketStore}; - -fn weksa(dir: &std::path::Path, args: &[&str]) -> Output { - Command::new(env!("CARGO_BIN_EXE_weksa")) - .current_dir(dir) - .args([ - "--corpus", - "./c.cc", - "--inventory", - "./i.cc", - "--entities", - "./e.cc", - ]) - .args(args) - .output() - .unwrap() -} - -/// Admits the cast entities the tests cite. -fn cast_entities(dir: &std::path::Path, slugs: &[&str]) { - for slug in slugs { - let id = format!("entity:{slug}"); - let form = format!("en-US={}", slug.to_uppercase()); - let put = weksa( - dir, - &[ - "entity", - "put", - &id, - "--form", - &form, - "--origin", - "cast", - "--persona", - slug, - "--fixture", - "f.yaml", - ], - ); - assert!(put.status.success(), "{}", out(&put)); - } -} - -fn out(o: &Output) -> String { - String::from_utf8_lossy(&o.stdout).into_owned() + &String::from_utf8_lossy(&o.stderr) -} - -const ADD: [&str; 14] = [ - "corpus", - "add", - "--slug", - "nibu", - "--text", - "Mind the hatch.", - "--speaker", - "entity:nibu", - "--group", - "scene-1", - "--admitted-on", - "2026-10-29", - "--source", - "cast-authored", -]; - -#[test] -fn seed_add_dump_split_withdraw() { - let d = tempfile::tempdir().unwrap(); - cast_entities(d.path(), &["nibu"]); - let seeded = weksa(d.path(), &["inventory", "seed"]); - assert!(seeded.status.success(), "{}", out(&seeded)); - assert!(out(&seeded).contains("seeded rev:1")); - let again = weksa(d.path(), &["inventory", "seed"]); - assert!( - !again.status.success(), - "a second seed is refused: {}", - out(&again) - ); - let dump = out(&weksa(d.path(), &["inventory", "dump"])); - assert!( - dump.contains("rev:1") && dump.contains("speech_act:5") && dump.contains("warn"), - "{dump}" - ); - - // A cast-authored item needs its persona, fixture and author. - let missing = weksa(d.path(), &ADD); - assert!( - !missing.status.success() && out(&missing).contains("--persona"), - "{}", - out(&missing) - ); - let mut add = ADD.to_vec(); - add.extend([ - "--persona", - "nibu", - "--fixture", - "f.yaml", - "--author", - "Ada Writer", - ]); - let added = weksa(d.path(), &add); - assert!( - added.status.success() && out(&added).contains("Added item:nibu:1"), - "{}", - out(&added) - ); - // The same text under a new key is refused (S6); another text is not. - let copy = weksa(d.path(), &add); - assert!( - !copy.status.success() && out(©).contains("S6TextHeld"), - "{}", - out(©) - ); - let mut other = add.clone(); - other[5] = "Mind the other hatch."; - let second = weksa(d.path(), &other); - assert!( - out(&second).contains("Added item:nibu:2"), - "{}", - out(&second) - ); - - let dump = out(&weksa(d.path(), &["corpus", "dump"])); - assert!( - dump.contains("item:nibu:1") - && dump.contains("Mind the hatch.") - && dump.contains("CastAuthored"), - "{dump}" - ); - let split = out(&weksa(d.path(), &["split", "item:nibu:1"])); - assert!( - ["Train", "Validation", "Test"] - .iter() - .any(|s| split.trim() == *s), - "{split}" - ); - assert!(!weksa(d.path(), &["split", "item:nibu:9"]).status.success()); - - let withdrawn = weksa(d.path(), &["corpus", "withdraw", "item:nibu:1"]); - assert!(withdrawn.status.success()); - assert!(out(&weksa(d.path(), &["corpus", "dump"])).contains("withdrawn")); -} - -#[test] -fn model_written_and_public_domain_sources_name_their_provenance() { - let d = tempfile::tempdir().unwrap(); - cast_entities(d.path(), &["s", "player"]); - let base = [ - "corpus", - "add", - "--slug", - "x", - "--speaker", - "entity:s", - "--group", - "g", - "--admitted-on", - "2026-10-29", - ]; - // S6 refuses one text under two keys, so each run writes its own. - let runs = std::cell::Cell::new(0); - let run = |extra: &[&str]| { - runs.set(runs.get() + 1); - let text = format!("line {}", runs.get()); - let mut args = base.to_vec(); - args.extend_from_slice(&["--text", &text]); - args.extend_from_slice(extra); - weksa(d.path(), &args) - }; - let no_family = run(&["--source", "model-written"]); - assert!( - !no_family.status.success() && out(&no_family).contains("--model-family"), - "{}", - out(&no_family) - ); - let not_an_enum_value = run(&["--source", "model-written", "--model-family", "m"]); - assert!( - !not_an_enum_value.status.success(), - "{}", - out(¬_an_enum_value) - ); - let model = run(&[ - "--source", - "model-written", - "--model-family", - "open-ai", - "--generator", - "g", - ]); - assert!(model.status.success(), "{}", out(&model)); - let dump = weksa(d.path(), &["corpus", "dump"]); - assert!( - out(&dump).contains("OpenAi") && out(&dump).contains("generator: Some(\"g\")"), - "{}", - out(&dump) - ); - let no_work = run(&["--source", "public-domain", "--author", "a"]); - assert!( - !no_work.status.success() && out(&no_work).contains("--work"), - "{}", - out(&no_work) - ); - let pd = run(&[ - "--source", - "public-domain", - "--work", - "w", - "--author", - "a", - "--edition-url", - "https://example.org/w", - ]); - assert!(pd.status.success(), "{}", out(&pd)); - let bad_turn = run(&[ - "--source", - "operator-written", - "--prior-turn", - "no equals sign", - ]); - assert!( - !bad_turn.status.success() && out(&bad_turn).contains("speaker=text"), - "{}", - out(&bad_turn) - ); - let turn = run(&[ - "--source", - "operator-written", - "--prior-turn", - "entity:player=hello", - ]); - assert!(turn.status.success(), "{}", out(&turn)); -} - -#[test] -fn validate_reports_each_packet() { - let d = tempfile::tempdir().unwrap(); - let w = common::world(); - let full = w.dir.path().join("i.cc"); - let packets = d.path().join("p.cc"); - let packet = common::good(); - PacketStore::open(&packets) - .unwrap() - .admit(packet.clone(), &w.inventory, &w.corpus, &w.entities) - .unwrap(); - - let validate = |inventory: &std::path::Path, extra: &[&str]| { - Command::new(env!("CARGO_BIN_EXE_weksa")) - .arg("--inventory") - .arg(inventory) - .arg("validate") - .arg(&packets) - .args(extra) - .output() - .unwrap() - }; - let ok = validate(&full, &[]); - assert!( - ok.status.success() && out(&ok).contains(&format!("ok {}", packet.id)), - "{}", - out(&ok) - ); - assert!(validate(&full, &[packet.id.as_str()]).status.success()); - assert!(!validate(&full, &["packet:missing"]).status.success()); - // Against an inventory that only reaches revision 1 the same packet is refused, by rule. - let short = d.path().join("short.cc"); - let (rev1, entries1) = weksa::inventory::seed(); - InventoryStore::open(&short) - .unwrap() - .admit_revision(rev1, entries1) - .unwrap(); - let refused = validate(&short, &[]); - assert!( - !refused.status.success() && out(&refused).contains("V10"), - "{}", - out(&refused) - ); -} - -#[test] -fn schema_writes_the_published_files() { - let d = tempfile::tempdir().unwrap(); - let o = weksa(d.path(), &["schema", "--out", "s"]); - assert!(o.status.success(), "{}", out(&o)); - let mut names: Vec = std::fs::read_dir(d.path().join("s")) - .unwrap() - .map(|e| e.unwrap().file_name().into_string().unwrap()) - .collect(); - names.sort(); - assert_eq!( - names, - [ - "weksa.corpus_item.schema.json", - "weksa.entity.schema.json", - "weksa.interlingua_packet.schema.json", - "weksa.inventory_entry.schema.json", - "weksa.inventory_revision.schema.json" - ] - ); -} - -#[test] -fn entity_put_and_name_spans() { - let d = tempfile::tempdir().unwrap(); - let put = |args: &[&str]| { - let mut all = vec![ - "entity", - "put", - "entity:kay", - "--origin", - "cast", - "--fixture", - "f.yaml", - ]; - all.extend_from_slice(args); - weksa(d.path(), &all) - }; - let no_persona = put(&["--form", "en-US=Kay"]); - assert!( - !no_persona.status.success() && out(&no_persona).contains("--persona"), - "{}", - out(&no_persona) - ); - let first = put(&["--form", "en-US=Kay", "--persona", "kay"]); - assert!( - first.status.success() && out(&first).contains("Added entity:kay"), - "{}", - out(&first) - ); - // Forms append; a language's forms keep the order given. - let grown = put(&[ - "--form", - "en-US=Kay", - "--form", - "en-US=Kay's", - "--persona", - "kay", - ]); - assert!(grown.status.success(), "{}", out(&grown)); - let shrunk = put(&["--form", "en-US=Kay", "--persona", "kay"]); - assert!( - !shrunk.status.success() && out(&shrunk).contains("EntityNotAppendOnly"), - "{}", - out(&shrunk) - ); - let dump = out(&weksa(d.path(), &["entity", "dump"])); - assert!( - dump.contains("entity:kay") && dump.contains("Kay's"), - "{dump}" - ); - - cast_entities(d.path(), &["nibu"]); - let add = |range: &str, text: &str| { - weksa( - d.path(), - &[ - "corpus", - "add", - "--slug", - "kay", - "--text", - text, - "--speaker", - "entity:nibu", - "--group", - "g", - "--admitted-on", - "2026-10-29", - "--source", - "operator-written", - "--name", - range, - ], - ) - }; - let named = add("0..3=entity:kay", "Kay left."); - assert!(named.status.success(), "{}", out(&named)); - let unequal = add("0..3=entity:kay", "Kai left."); - assert!( - !unequal.status.success() && out(&unequal).contains("NameSpanNotAForm"), - "{}", - out(&unequal) - ); - let malformed = add("0-3", "Kay again."); - assert!( - !malformed.status.success() && out(&malformed).contains("start..end=entity"), - "{}", - out(&malformed) - ); - let dump = out(&weksa(d.path(), &["corpus", "dump"])); - assert!( - dump.contains("names:") && dump.contains("entity:kay"), - "{dump}" - ); -} - -#[test] -fn entity_origin_flags_must_match_origin() { - let d = tempfile::tempdir().unwrap(); - let put = |args: &[&str]| { - let mut all = vec!["entity", "put", "entity:kay", "--form", "en-US=Kay"]; - all.extend_from_slice(args); - weksa(d.path(), &all) - }; - for mismatched in [ - &["--origin", "live"][..], - &["--origin", "live", "--persona", "kay", "--work", "edda"], - &[ - "--origin", - "live", - "--persona", - "kay", - "--fixture", - "f.yaml", - ], - &["--origin", "cast", "--persona", "kay"], - &["--origin", "cast", "--fixture", "f.yaml"], - &[ - "--origin", - "cast", - "--persona", - "kay", - "--fixture", - "f.yaml", - "--work", - "edda", - ], - &["--origin", "public-domain"], - &[ - "--origin", - "public-domain", - "--work", - "edda", - "--persona", - "kay", - ], - &[ - "--origin", - "public-domain", - "--work", - "edda", - "--fixture", - "f.yaml", - ], - &["--origin", "cast", "--persona", "", "--fixture", "f.yaml"], - &["--origin", "cast", "--persona", "kay", "--fixture", ""], - &["--origin", "live", "--persona", ""], - &["--origin", "public-domain", "--work", ""], - ] { - let o = put(mismatched); - assert!(!o.status.success(), "{mismatched:?}: {}", out(&o)); - } - assert!(!out(&weksa(d.path(), &["entity", "dump"])).contains("entity:kay")); - assert!( - put(&["--origin", "live", "--persona", "kay"]) - .status - .success() - ); - let work = weksa( - d.path(), - &[ - "entity", - "put", - "entity:brokkr", - "--form", - "en-US=Brokkr", - "--origin", - "public-domain", - "--work", - "edda", - ], - ); - assert!(work.status.success(), "{}", out(&work)); - assert!(out(&weksa(d.path(), &["entity", "dump"])).contains("PublicDomain")); -} - -#[test] -fn intake_normalizes_to_nfc() { - let d = tempfile::tempdir().unwrap(); - // A decomposed form and decomposed text meet as the same composed bytes. - let put = weksa( - d.path(), - &[ - "entity", - "put", - "entity:rene", - "--form", - "fr=Rene\u{301}", - "--origin", - "live", - "--persona", - "rene", - ], - ); - assert!(put.status.success(), "{}", out(&put)); - cast_entities(d.path(), &["nibu"]); - let add = weksa( - d.path(), - &[ - "corpus", - "add", - "--slug", - "rene", - "--text", - "Rene\u{301} part.", - "--language", - "fr", - "--speaker", - "entity:nibu", - "--group", - "g", - "--admitted-on", - "2026-10-29", - "--source", - "operator-written", - "--name", - "0..5=entity:rene", - ], - ); - assert!(add.status.success(), "{}", out(&add)); - let dump = out(&weksa(d.path(), &["corpus", "dump"])); - assert!(dump.contains("Ren\u{e9} part."), "{dump}"); -} - -/// `corpus add` of "Mind the hatch." by `entity:nibu` under `source`, with -/// `extra` flags. -fn add_with(dir: &std::path::Path, source: &str, extra: &[&str]) -> Output { - let mut args = vec![ - "corpus", - "add", - "--slug", - "nibu", - "--text", - "Mind the hatch.", - "--speaker", - "entity:nibu", - "--group", - "scene-1", - "--admitted-on", - "2026-10-29", - "--source", - source, - ]; - args.extend_from_slice(extra); - weksa(dir, &args) -} - -#[test] -fn source_flags_must_match_the_source() { - let d = tempfile::tempdir().unwrap(); - cast_entities(d.path(), &["nibu"]); - // A provenance flag that does not apply to --source refuses, naming the flag. - for (source, extra, flag) in [ - ("operator-written", &["--persona", "p"][..], "--persona"), - ( - "operator-approved", - &["--note", "n", "--work", "w"], - "--work", - ), - ( - "public-domain", - &[ - "--work", - "w", - "--author", - "a", - "--edition-url", - "u", - "--note", - "n", - ], - "--note", - ), - ( - "cast-authored", - &[ - "--persona", - "p", - "--fixture", - "f", - "--author", - "a", - "--generator", - "g", - ], - "--generator", - ), - ( - "model-written", - &["--model-family", "unrecorded", "--edition-url", "u"], - "--edition-url", - ), - ( - "operator-written", - &["--model-family", "unrecorded"], - "--model-family", - ), - ( - "model-written", - &[ - "--model-family", - "unrecorded", - "--prompt-digest", - "d", - "--fixture", - "f", - ], - "--fixture", - ), - // A source missing a flag it needs. - ( - "cast-authored", - &["--persona", "p", "--fixture", "f"], - "--author", - ), - ( - "public-domain", - &["--work", "w", "--edition-url", "u"], - "--author", - ), - // Empty provenance strings. - ( - "model-written", - &["--model-family", "unrecorded", "--generator", ""], - "", - ), - ( - "model-written", - &["--model-family", "unrecorded", "--prompt-digest", ""], - "", - ), - ( - "cast-authored", - &["--persona", "", "--fixture", "f", "--author", "a"], - "", - ), - ("operator-approved", &["--note", ""], ""), - ] { - let o = add_with(d.path(), source, extra); - assert!(!o.status.success(), "{source} {extra:?}: {}", out(&o)); - assert!(out(&o).contains(flag), "{source} {extra:?}: {}", out(&o)); - } - assert!(!out(&weksa(d.path(), &["corpus", "dump"])).contains("item:")); - // A source whose flags all apply is admitted; --author reaches a - // cast-authored item. - let cast = add_with( - d.path(), - "cast-authored", - &["--persona", "p", "--fixture", "f", "--author", "Ada Writer"], - ); - assert!(cast.status.success(), "{}", out(&cast)); - let dump = out(&weksa(d.path(), &["corpus", "dump"])); - assert!(dump.contains("author: \"Ada Writer\""), "{dump}"); -} - -#[test] -fn malformed_intake_never_echoes_its_values() { - const CANARY: &str = "CANARY-7f3a"; - let d = tempfile::tempdir().unwrap(); - cast_entities(d.path(), &["nibu"]); - let shape = |value: &str| format!("{CANARY}{value}"); - let puts = [ - (vec![shape("")], "--form"), - (vec![shape("_x=Kay")], ""), - (vec![format!("en-US={CANARY}\u{7}")], ""), - (vec![format!("{CANARY}_x=Kay")], ""), - ]; - for (forms, flag) in puts { - let mut args = vec![ - "entity", - "put", - "entity:kay", - "--origin", - "live", - "--persona", - "kay", - ]; - for f in &forms { - args.extend(["--form", f]); - } - let o = weksa(d.path(), &args); - assert!(!o.status.success(), "{forms:?}: {}", out(&o)); - assert!(!out(&o).contains(CANARY), "{forms:?}: {}", out(&o)); - assert!(out(&o).contains(flag), "{forms:?}: {}", out(&o)); - } - let bad_turn = [shape(""), format!("{CANARY}=hello")]; - let bad_name = [ - shape(""), - format!("0..1={CANARY}"), - format!("{CANARY}..1=entity:nibu"), - format!("0..{CANARY}=entity:nibu"), - ]; - let cases = bad_turn - .iter() - .map(|t| ("--prior-turn", t)) - .chain(bad_name.iter().map(|n| ("--name", n))); - for (flag, value) in cases { - let o = add_with(d.path(), "operator-written", &[flag, value]); - assert!(!o.status.success(), "{flag} {value}: {}", out(&o)); - assert!(!out(&o).contains(CANARY), "{flag} {value}: {}", out(&o)); - assert!(out(&o).contains(flag), "{flag} {value}: {}", out(&o)); - } -} - -#[test] -fn every_decomposed_form_and_turn_is_stored_composed() { - let d = tempfile::tempdir().unwrap(); - cast_entities(d.path(), &["nibu"]); - let put = weksa( - d.path(), - &[ - "entity", - "put", - "entity:rene", - "--form", - "fr=Rene\u{301}", - "--form", - "fr=Zoe\u{308}", - "--origin", - "live", - "--persona", - "rene", - ], - ); - assert!(put.status.success(), "{}", out(&put)); - let rene = EntityStore::open(&d.path().join("e.cc")) - .unwrap() - .get(&"entity:rene".parse().unwrap()) - .unwrap() - .unwrap(); - assert_eq!(rene.forms[0].forms, ["Ren\u{e9}", "Zo\u{eb}"]); - let add = add_with( - d.path(), - "operator-written", - &["--prior-turn", "entity:rene=Cafe\u{301} now"], - ); - assert!(add.status.success(), "{}", out(&add)); - let item = CorpusStore::open(&d.path().join("c.cc")) - .unwrap() - .get("item:nibu:1") - .unwrap() - .unwrap(); - assert_eq!(item.prior_turns[0].text, "Caf\u{e9} now"); -} - -/// An Ollama stand-in: every text embeds to `[1, its length]`, so identical -/// texts have cosine 1. -fn fake_embedder() -> String { - use std::io::{Read, Write}; - let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); - let url = format!("http://{}", listener.local_addr().unwrap()); - std::thread::spawn(move || { - for mut stream in listener.incoming().flatten() { - let mut buf = vec![0u8; 65536]; - let mut n = 0; - let body = loop { - n += stream.read(&mut buf[n..]).unwrap(); - let text = String::from_utf8_lossy(&buf[..n]).into_owned(); - if let Some((head, body)) = text.split_once("\r\n\r\n") { - let want: usize = head - .to_lowercase() - .split("content-length: ") - .nth(1) - .unwrap() - .split("\r\n") - .next() - .unwrap() - .parse() - .unwrap(); - if body.len() >= want { - break body.to_string(); - } - } - }; - let body: serde_json::Value = serde_json::from_str(&body).unwrap(); - let vectors: Vec> = body["input"] - .as_array() - .unwrap() - .iter() - .map(|t| vec![1.0, t.as_str().unwrap().len() as f64]) - .collect(); - let reply = serde_json::json!({ "embeddings": vectors }).to_string(); - write!( - stream, - "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\n\r\n{reply}", - reply.len() - ) - .unwrap(); - } - }); - url -} - -#[test] -fn proof_packet_lowers_and_the_lowering_scores_against_items() { - let d = tempfile::tempdir().unwrap(); - cast_entities(d.path(), &["nibu"]); - for args in [&["inventory", "seed"][..], &["inventory", "proof"]] { - let run = weksa(d.path(), args); - assert!(run.status.success(), "{}", out(&run)); - } - let item = weksa( - d.path(), - &[ - "corpus", - "add", - "--slug", - "nibu", - "--text", - "Warning: you may endanger yourself by touching the control. The ship mind will prevent that to protect you.", - "--speaker", - "entity:nibu", - "--group", - "scene-1", - "--admitted-on", - "2026-11-01", - "--source", - "operator-written", - ], - ); - assert!(item.status.success(), "{}", out(&item)); - let admitted = weksa(d.path(), &["packet", "proof", "./p.cc"]); - assert!( - admitted.status.success() && out(&admitted).contains("admitted packet:6e6962752d303031"), - "{}", - out(&admitted) - ); - let lowered = weksa(d.path(), &["lower", "./p.cc", "packet:6e6962752d303031"]); - assert!(lowered.status.success(), "{}", out(&lowered)); - let text = String::from_utf8(lowered.stdout).unwrap(); - assert!( - text.starts_with("Warning: you may endanger yourself"), - "{text}" - ); - - let missing = weksa(d.path(), &["lower", "./p.cc", "packet:0000000000000000"]); - assert!(!missing.status.success() && out(&missing).contains("no packet")); - - std::fs::write(d.path().join("out.txt"), &text).unwrap(); - let url = fake_embedder(); - let scored = weksa( - d.path(), - &[ - "score", - "item:nibu:1", - "--output", - "out.txt", - "--ollama-url", - &url, - ], - ); - assert!(scored.status.success(), "{}", out(&scored)); - assert_eq!( - out(&scored).trim(), - "item:nibu:1 chrf=100.00 meaning=1.0000" - ); - let unknown = weksa( - d.path(), - &[ - "score", - "item:nibu:9", - "--output", - "out.txt", - "--ollama-url", - &url, - ], - ); - assert!(!unknown.status.success() && out(&unknown).contains("no item item:nibu:9")); -} diff --git a/crates/weksa/tests/common/mod.rs b/crates/weksa/tests/common/mod.rs deleted file mode 100644 index 07e95ee..0000000 --- a/crates/weksa/tests/common/mod.rs +++ /dev/null @@ -1,331 +0,0 @@ -#![allow(dead_code)] - -use std::sync::OnceLock; - -use tempfile::TempDir; -use weksa::corpus::ModelFamily; -use weksa::inventory::{Change, seed}; -use weksa::packet::*; -use weksa::{ - CorpusItem, CorpusItemId, CorpusStore, Entity, EntityOrigin, EntityStore, InvId, Inventory, - InventoryEntry, InventoryRevision, InventoryStore, LanguageForms, NameSpan, OptionLabel, Slot, - Source, SplitScheme, ValueKind, -}; - -pub const PERSON: &str = "referent_kind:1"; -pub const AGENT: &str = "role:1"; -pub const PATIENT: &str = "role:2"; -pub const WARN: &str = "speech_act:5"; -pub const POLARITY: &str = "feature:1"; -pub const MODALITY: &str = "feature:4"; -pub const HEDGE: &str = "feature:6"; -pub const IRONY: &str = "feature:8"; -/// Stance features, slot Stance only (Grade). -pub const ANGER: &str = "feature:9"; -pub const AFFECTION: &str = "feature:16"; -pub const RESTRAINT: &str = "feature:18"; -/// Introduced at rev 2. -pub const CONCEPT_A: &str = "concept:1"; -pub const CONCEPT_B: &str = "concept:2"; -/// Introduced at rev 2, retired at rev 3. -pub const CONCEPT_RETIRED: &str = "concept:3"; -/// Introduced at rev 4. -pub const CONCEPT_LATE: &str = "concept:4"; -pub const FRAME: &str = "frame:1"; -/// An extension entry of value kind Grade, introduced at rev 2. -pub const EXT: &str = "extension:1"; -/// A Grade feature introduced at rev 2 that sits in every slot. -pub const FEAT_ALL: &str = "feature:50"; - -pub const PACKET_ID: &str = "packet:0123456789abcdef"; -pub const ENCODER_ID: &str = "encoder:fedcba9876543210"; - -pub fn iv(s: &str) -> InvId { - s.parse().unwrap() -} - -pub fn nd(s: &str) -> NodeId { - NodeId::parse(s).unwrap() -} - -pub fn lab(s: &str) -> OptionLabel { - OptionLabel::parse(s).unwrap() -} - -pub fn grade(v: u16) -> Grade { - Grade::new(v).unwrap() -} - -pub fn group_in_bucket(want: u64) -> String { - (0..100_000) - .map(|i| format!("g{i}")) - .find(|g| SplitScheme::V1.bucket(g) == want) - .unwrap_or_else(|| panic!("no group lands in bucket {want}")) -} - -pub fn cast() -> Source { - Source::CastAuthored { - persona: "nibu".into(), - fixture: "f.yaml".into(), - author: "Ada Writer".into(), - } -} - -pub fn model() -> Source { - Source::ModelWritten { - family: ModelFamily::Unrecorded, - generator: None, - prompt_digest: None, - } -} - -/// Every speaker the tests use, as entities with one en-US form. -const SPEAKERS: [&str; 6] = ["nibu", "brokkr", "player", "s", "someone", "hero"]; - -pub fn eid(slug: &str) -> EntityId { - EntityId::parse(&format!("entity:{slug}")).unwrap() -} - -/// A cast entity with `forms` in `language`, citation form first. -pub fn ent(slug: &str, language: &str, forms: &[&str]) -> Entity { - Entity { - id: eid(slug), - forms: vec![LanguageForms { - language: language.into(), - forms: forms.iter().map(|f| f.to_string()).collect(), - }], - origin: EntityOrigin::Cast { - persona: slug.into(), - fixture: "f.yaml".into(), - }, - } -} - -/// The entity store at `entities.cc` under `dir`, holding every test speaker -/// (en-US form: the slug capitalized). Reopening replays identically. -pub fn entities_in(dir: &std::path::Path) -> EntityStore { - let mut store = EntityStore::open(&dir.join("entities.cc")).unwrap(); - for slug in SPEAKERS { - let form = slug[..1].to_uppercase() + &slug[1..]; - store.put(ent(slug, "en-US", &[&form])).unwrap(); - } - store -} - -pub fn item_in(key: &str, text: &str, group: &str, source: Source) -> CorpusItem { - CorpusItem { - id: key.into(), - text: text.into(), - language: "en-US".into(), - speaker: eid("nibu"), - names: vec![], - group: group.into(), - prior_turns: vec![], - source, - admitted_on: "2026-10-29".into(), - supersedes: None, - withdrawn: false, - } -} - -pub fn entry(id: &str, introduced: u32) -> InventoryEntry { - InventoryEntry { - id: iv(id), - gloss: id.into(), - value_kind: None, - applies_to: vec![], - introduced_rev: introduced, - retired_rev: None, - } -} - -pub fn feature(id: &str, kind: ValueKind, slots: &[Slot]) -> InventoryEntry { - InventoryEntry { - value_kind: Some(kind), - applies_to: slots.to_vec(), - ..entry(id, 1) - } -} - -pub fn rev_with(changes: Vec, n: u32, parent: Option) -> InventoryRevision { - InventoryRevision { - n, - parent, - changes, - frozen: false, - } -} - -/// Admits `entries` as revision `n` and returns nothing; the stores are the only door. -pub fn admit(store: &mut InventoryStore, n: u32, entries: Vec) { - let changes = entries.iter().map(|e| Change::Add(e.id)).collect(); - let parent = n.checked_sub(1).filter(|p| *p > 0); - store - .admit_revision(rev_with(changes, n, parent), entries) - .unwrap(); -} - -/// Stores a test can admit through: a corpus holding `item:nibu:1` in a Train -/// group, and an inventory of revisions 1 (the seed), 2 (concepts A, B, -/// RETIRED, a frame, a Grade extension), 3 (RETIRED retired) and 4 (LATE). -pub struct World { - pub dir: TempDir, - pub inventory: InventoryStore, - pub corpus: CorpusStore, - pub entities: EntityStore, -} - -impl World { - pub fn inv(&self) -> Inventory { - self.inventory.inventory().unwrap() - } -} - -pub fn world() -> World { - let dir = tempfile::tempdir().unwrap(); - let mut corpus = CorpusStore::open(&dir.path().join("c.cc")).unwrap(); - let entities = entities_in(dir.path()); - let train = group_in_bucket(10); - let mut first = item_in("item:nibu:1", "Nibu says mind the hatch.", &train, cast()); - first.names = vec![NameSpan { - entity: eid("nibu"), - start: 0, - end: 4, - }]; - corpus.add(first, &entities).unwrap(); - let mut inventory = InventoryStore::open(&dir.path().join("i.cc")).unwrap(); - let (rev1, entries) = seed(); - inventory.admit_revision(rev1, entries).unwrap(); - let mut ext = entry(EXT, 2); - ext.value_kind = Some(ValueKind::Grade); - admit( - &mut inventory, - 2, - vec![ - entry(CONCEPT_A, 2), - entry(CONCEPT_B, 2), - entry(CONCEPT_RETIRED, 2), - entry(FRAME, 2), - ext, - InventoryEntry { - introduced_rev: 2, - ..feature( - FEAT_ALL, - ValueKind::Grade, - &[ - Slot::Discourse, - Slot::Referent, - Slot::Predication, - Slot::Relation, - Slot::Stance, - ], - ) - }, - ], - ); - inventory - .admit_revision( - rev_with(vec![Change::Retire(iv(CONCEPT_RETIRED))], 3, Some(2)), - vec![], - ) - .unwrap(); - admit(&mut inventory, 4, vec![entry(CONCEPT_LATE, 4)]); - World { - dir, - inventory, - corpus, - entities, - } -} - -/// The world's inventory, built once. -pub fn inventory() -> Inventory { - static INV: OnceLock = OnceLock::new(); - INV.get_or_init(|| world().inv()).clone() -} - -pub fn good() -> Packet { - Packet { - id: PacketId::parse(PACKET_ID).unwrap(), - schema_version: SCHEMA_VERSION, - inventory_rev: 2, - kind: PacketKind::DialogueLine, - provenance: Provenance { - source: ProvenanceSource::Authored, - corpus_item: CorpusItemId::parse("item:nibu:1").unwrap(), - encoder: EncoderId::parse(ENCODER_ID).unwrap(), - confidence: grade(900), - }, - context: Context { - setting: Some(iv(CONCEPT_A)), - medium: Some(iv(CONCEPT_B)), - participants: vec![nd("n1")], - audience: vec![nd("n1")], - }, - discourse: Discourse { - speech_act: Some(iv(WARN)), - addressee: Some(nd("n1")), - features: vec![FeatureValue { - feature: iv(HEDGE), - value: FeatValue::Grade(grade(300)), - }], - }, - referents: vec![Referent { - id: nd("n1"), - kind: iv(PERSON), - concept: iv(CONCEPT_A), - entity: Some(eid("nibu")), - features: vec![], - }], - predications: vec![Predication { - id: nd("n2"), - concept: iv(CONCEPT_B), - frame: iv(FRAME), - roles: vec![ - RoleFill { - role: iv(AGENT), - filler: nd("n1"), - }, - RoleFill { - role: iv(PATIENT), - filler: nd("n1"), - }, - ], - features: vec![FeatureValue { - feature: iv(MODALITY), - value: FeatValue::Option(lab("potential")), - }], - }], - relations: vec![Relation { - relation: iv("relation:8"), - from: nd("n2"), - to: nd("n1"), - features: vec![], - }], - constraints: Constraints { - preserve: vec![nd("n1")], - deterministic: true, - max_latency_ms: Some(50), - }, - extensions: vec![Extension { - key: iv(EXT), - value: FeatValue::Grade(grade(3)), - }], - trace: Trace { - notes: vec!["a note".into()], - rejected_readings: vec![], - }, - stance: Stance { - features: vec![ - FeatureValue { - feature: iv(ANGER), - value: FeatValue::Grade(grade(300)), - }, - FeatureValue { - feature: iv(RESTRAINT), - value: FeatValue::Grade(grade(700)), - }, - ], - }, - } -} diff --git a/crates/weksa/tests/entities.rs b/crates/weksa/tests/entities.rs deleted file mode 100644 index 46aeb71..0000000 --- a/crates/weksa/tests/entities.rs +++ /dev/null @@ -1,429 +0,0 @@ -mod common; -use common::*; -use weksa::corpus::{CorpusItem, NameSpan, Turn}; -use weksa::packet::{Packet, Referent}; -use weksa::{ - AddOutcome, CorpusItemId, CorpusStore, Entity, EntityId, EntityOrigin, EntityStore, PacketId, - PacketStore, Refusal, SCHEMA_VERSION, StoreError, StoreRefusal, validate, -}; - -fn refusal(r: Result) -> StoreRefusal { - match r { - Err(StoreError::Refused(r)) => r, - other => panic!("expected a store refusal, got {other:?}"), - } -} - -fn malformed(r: Result) -> &'static str { - match refusal(r) { - StoreRefusal::Malformed { why, .. } => why, - other => panic!("expected Malformed, got {other:?}"), - } -} - -fn span(slug: &str, start: u32, end: u32) -> NameSpan { - NameSpan { - entity: eid(slug), - start, - end, - } -} - -/// An item in `language` whose text and spans the test chooses. -fn named(n: u32, language: &str, text: &str, names: Vec) -> CorpusItem { - CorpusItem { - language: language.into(), - names, - ..item_in(&format!("item:t:{n}"), text, "g", cast()) - } -} - -#[test] -fn entity_id_form() { - for ok in ["entity:brokkr", "entity:a-1", "entity:7", "entity:a-b-c"] { - assert_eq!(EntityId::parse(ok).unwrap().as_str(), ok); - } - let long_ok = format!("entity:{}", "a".repeat(48)); - assert!(EntityId::parse(&long_ok).is_ok()); - let too_long = format!("entity:{}", "a".repeat(49)); - for bad in [ - "Entity:x", - "entity:-a", - "entity:a-", - "entity:", - "entity:a_b", - "entity:A", - "entity:a b", - "entity:a:b", - "brokkr", - "entity:\u{5c71}", - &too_long, - ] { - assert!(EntityId::parse(bad).is_err(), "{bad:?}"); - assert!( - serde_json::from_value::(bad.into()).is_err(), - "{bad:?} must not read back from a store" - ); - } - // A referent reads its entity through the same parse. - let mut referent = serde_json::to_value(good().referents[0].clone()).unwrap(); - assert!(serde_json::from_value::(referent.clone()).is_ok()); - referent["entity"] = "Nibu".into(); - assert!(serde_json::from_value::(referent).is_err()); -} - -#[test] -fn entity_forms_only_grow() { - let d = tempfile::tempdir().unwrap(); - let mut store = EntityStore::open(&d.path().join("e.cc")).unwrap(); - let kay = ent("kay", "en-US", &["Kay"]); - assert_eq!(store.put(kay.clone()).unwrap(), AddOutcome::Added); - assert_eq!(store.put(kay.clone()).unwrap(), AddOutcome::Unchanged); - - // A form appends; a language appends. - let mut grown = kay.clone(); - grown.forms[0].forms.push("Kay's".into()); - assert_eq!(store.put(grown.clone()).unwrap(), AddOutcome::Added); - let mut more = grown.clone(); - more.forms.push(weksa::LanguageForms { - language: "tr".into(), - forms: vec!["Kay".into()], - }); - assert_eq!(store.put(more.clone()).unwrap(), AddOutcome::Added); - // The store read back from disk holds the growth. - let mut store = EntityStore::open(&d.path().join("e.cc")).unwrap(); - assert_eq!(store.get(&eid("kay")).unwrap(), Some(more.clone())); - - // Dropping, reordering, editing and re-originating are refused, each leaving - // the stored entity as it was. - let not_append = StoreRefusal::EntityNotAppendOnly { id: eid("kay") }; - let edits: [fn(&mut Entity); 6] = [ - |e| e.forms[0].forms.reverse(), - |e| e.forms[0].forms[0] = "Kai".into(), - |e| { - e.forms.pop(); - }, - |e| e.forms.reverse(), - |e| e.forms[1].language = "fi".into(), - |e| { - e.origin = EntityOrigin::Live { - persona: "kay".into(), - } - }, - ]; - assert_eq!( - refusal(store.put(ent("kay", "en-US", &["Kay"]))), - not_append - ); - for edit in edits { - let mut v = more.clone(); - edit(&mut v); - assert_eq!(refusal(store.put(v.clone())), not_append, "{v:?}"); - } - assert_eq!(store.get(&eid("kay")).unwrap(), Some(more)); - - // Shape: no forms, a language twice, an empty language, an empty language - // entry, an empty, padded, control-bearing, oversize or repeated form. - let shape = |forms: Vec| weksa::Entity { - forms, - ..ent("shape", "en-US", &["X"]) - }; - let lf = |language: &str, forms: &[&str]| weksa::LanguageForms { - language: language.into(), - forms: forms.iter().map(|f| f.to_string()).collect(), - }; - for bad in [ - vec![], - vec![lf("en-US", &["A"]), lf("en-US", &["B"])], - vec![lf("en-US", &[])], - vec![lf("en-US", &[""])], - vec![lf("en-US", &[" A"])], - vec![lf("en-US", &["A\u{a0}"])], - vec![lf("en-US", &["A\nB"])], - vec![lf("en-US", &["A", "A"])], - ] { - malformed(store.put(shape(bad.clone()))); - assert!( - store.get(&eid("shape")).unwrap().is_none(), - "{bad:?} was stored" - ); - } - // An empty language is not a tag. - let e = shape(vec![lf("", &["A"])]); - assert!(matches!( - refusal(store.put(e)), - StoreRefusal::EntityBounds { - bound: weksa::store::EntityBound::LanguageTag, - .. - } - )); - // 64 bytes is the edge; a form may repeat across languages. - let edge = "a".repeat(64); - store - .put(shape(vec![lf("en-US", &[&edge]), lf("tr", &[&edge])])) - .unwrap(); -} - -#[test] -fn corpus_spans_checked() { - let d = tempfile::tempdir().unwrap(); - let mut ents = entities_in(d.path()); - ents.put(ent("kay", "en-US", &["Kay"])).unwrap(); - ents.put(ent("ab", "en-US", &["abab", "bab"])).unwrap(); - let mut corpus = CorpusStore::open(&d.path().join("c.cc")).unwrap(); - - assert_eq!( - corpus - .add( - named(1, "en-US", "Kay left.", vec![span("kay", 0, 3)]), - &ents - ) - .unwrap(), - AddOutcome::Added - ); - - let range = |n: u32, start: u32, end: u32| StoreRefusal::NameSpanRange { - key: format!("item:t:{n}"), - start, - end, - }; - let not_a_form = |n: u32, slug: &str, start: u32, end: u32| StoreRefusal::NameSpanNotAForm { - key: format!("item:t:{n}"), - entity: eid(slug), - start, - end, - }; - let mut refused = |item: CorpusItem| refusal(corpus.add(item, &ents)); - // Empty, backwards, past the end: a range refusal. - for (start, end) in [(3, 3), (4, 3), (0, 99)] { - let item = named(2, "en-US", "Kay left.", vec![span("kay", start, end)]); - assert_eq!(refused(item), range(2, start, end)); - } - // Inside a multibyte character: start and end off the boundary. - let text = "\u{e9}Kay"; - assert_eq!(text.len(), 5); - let item = named(2, "en-US", text, vec![span("kay", 1, 4)]); - assert_eq!(refused(item), range(2, 1, 4)); - let item = named(2, "en-US", "Kay\u{e9}", vec![span("kay", 0, 4)]); - assert_eq!(refused(item), range(2, 0, 4)); - // An unknown entity. - let item = named(2, "en-US", "Kay left.", vec![span("ghost", 0, 3)]); - assert_eq!( - refused(item), - StoreRefusal::UnknownEntity { - entity: eid("ghost") - } - ); - // Not byte-equal: other text, other case, a missing language. - let item = named(2, "en-US", "Kai left.", vec![span("kay", 0, 3)]); - assert_eq!(refused(item), not_a_form(2, "kay", 0, 3)); - let item = named(2, "en-US", "kay left.", vec![span("kay", 0, 3)]); - assert_eq!(refused(item), not_a_form(2, "kay", 0, 3)); - let item = named(2, "tr", "Kay left.", vec![span("kay", 0, 3)]); - assert_eq!(refused(item), not_a_form(2, "kay", 0, 3)); - // Overlap: both spans are forms, they share bytes. - let item = named(2, "en-US", "abab", vec![span("ab", 0, 4), span("ab", 1, 4)]); - assert_eq!( - refused(item), - StoreRefusal::NameSpanOverlap { - key: "item:t:2".into() - } - ); - // Adjacent spans do not overlap, whatever order they are listed in. - let adjacent = named( - 90, - "en-US", - "KayKay", - vec![span("kay", 3, 6), span("kay", 0, 3)], - ); - assert_eq!(corpus.add(adjacent, &ents).unwrap(), AddOutcome::Added); - - // Speakers: unknown, and unknown in a prior turn. - let ghost = EntityId::parse("entity:ghost").unwrap(); - let unknown = StoreRefusal::UnknownEntity { - entity: ghost.clone(), - }; - let mut item = named(91, "en-US", "Hello there.", vec![]); - item.speaker = ghost.clone(); - assert_eq!(refusal(corpus.add(item, &ents)), unknown); - let mut item = named(92, "en-US", "Hello again.", vec![]); - item.prior_turns = vec![ - Turn { - speaker: eid("nibu"), - text: "hi".into(), - }, - Turn { - speaker: ghost, - text: "hi".into(), - }, - ]; - assert_eq!(refusal(corpus.add(item, &ents)), unknown); - let mut item = named(93, "en-US", "Hello at last.", vec![]); - item.prior_turns = vec![Turn { - speaker: eid("nibu"), - text: "hi".into(), - }]; - assert_eq!(corpus.add(item, &ents).unwrap(), AddOutcome::Added); - // Nothing refused was stored. - let keys: Vec = corpus.all().unwrap().into_iter().map(|i| i.id).collect(); - assert_eq!(keys, ["item:t:1", "item:t:90", "item:t:93"]); -} - -#[test] -fn corpus_spans_every_script() { - let d = tempfile::tempdir().unwrap(); - let mut ents = entities_in(d.path()); - ents.put(ent( - "minsu", - "ko", - &["\u{BBFC}\u{C218}", "\u{BBFC}\u{C218}\u{AC00}"], - )) - .unwrap(); - ents.put(ent("brokkr-tr", "tr", &["Brokkr", "Brokkr'\u{131}n"])) - .unwrap(); - ents.put(ent( - "somchai", - "th", - &["\u{E2A}\u{E21}\u{E0A}\u{E32}\u{E22}"], - )) - .unwrap(); - let mut corpus = CorpusStore::open(&d.path().join("c.cc")).unwrap(); - - // The particle-bound Korean form: byte equality with the stored form. - let ko = "\u{BBFC}\u{C218}\u{AC00} \u{C654}\u{B2E4}"; - let with_particle = "\u{BBFC}\u{C218}\u{AC00}".len(); - assert_eq!(with_particle, 9); - let item = named(1, "ko", ko, vec![span("minsu", 0, with_particle as u32)]); - assert_eq!(corpus.add(item, &ents).unwrap(), AddOutcome::Added); - // The bare stem is a stored form too; a span cutting a syllable is not. - let item = named( - 2, - "ko", - "\u{BBFC}\u{C218} \u{C654}\u{B2E4}", - vec![span("minsu", 0, 6)], - ); - assert_eq!(corpus.add(item, &ents).unwrap(), AddOutcome::Added); - let item = named(3, "ko", ko, vec![span("minsu", 0, 7)]); - assert!(matches!( - refusal(corpus.add(item, &ents)), - StoreRefusal::NameSpanRange { .. } - )); - // Turkish: the apostrophe suffix. - let tr = "Brokkr'\u{131}n geldi."; - let len = "Brokkr'\u{131}n".len() as u32; - let item = named(4, "tr", tr, vec![span("brokkr-tr", 0, len)]); - assert_eq!(corpus.add(item, &ents).unwrap(), AddOutcome::Added); - // Thai: no spaces, no segmentation, a span is a byte range. - let th = "\u{E2A}\u{E21}\u{E0A}\u{E32}\u{E22}\u{E21}\u{E32}\u{E41}\u{E25}\u{E49}\u{E27}"; - let len = "\u{E2A}\u{E21}\u{E0A}\u{E32}\u{E22}".len() as u32; - let item = named(5, "th", th, vec![span("somchai", 0, len)]); - assert_eq!(corpus.add(item, &ents).unwrap(), AddOutcome::Added); - // The same bytes under another language tag are refused. - let item = named(6, "ko", th, vec![span("somchai", 0, len)]); - assert!(matches!( - refusal(corpus.add(item, &ents)), - StoreRefusal::NameSpanNotAForm { .. } - )); -} - -#[test] -fn packet_entities_resolve() { - let mut w = world(); - w.entities.put(ent("kay", "en-US", &["Kay"])).unwrap(); - let mut talk = named(2, "en-US", "Kay and Hero talk.", vec![span("kay", 0, 3)]); - talk.id = "item:nibu:2".into(); - talk.prior_turns = vec![Turn { - speaker: eid("hero"), - text: "hi".into(), - }]; - w.corpus.add(talk, &w.entities).unwrap(); - - let mut packets = PacketStore::open(&w.dir.path().join("p.cc")).unwrap(); - let packet = |n: u8, item: &str, entity: &str| { - let mut p = good(); - p.id = PacketId::parse(&format!("packet:{n:016x}")).unwrap(); - p.provenance.corpus_item = CorpusItemId::parse(item).unwrap(); - p.referents[0].entity = Some(eid(entity)); - p - }; - let mut admit = |p: Packet| packets.admit(p, &w.inventory, &w.corpus, &w.entities); - - // An entity the store lacks. - let unknown = StoreRefusal::UnknownEntity { - entity: eid("ghost"), - }; - assert_eq!(refusal(admit(packet(1, "item:nibu:1", "ghost"))), unknown); - assert_eq!(refusal(admit(packet(2, "item:nibu:2", "ghost"))), unknown); - // A stored entity that is not on the item. - let off_item = |item: &str, entity: &str| StoreRefusal::EntityNotOnItem { - entity: eid(entity), - item: item.into(), - }; - assert_eq!( - refusal(admit(packet(3, "item:nibu:2", "brokkr"))), - off_item("item:nibu:2", "brokkr") - ); - // nibu:1 names nibu (and is spoken by nibu); kay is only on nibu:2. - assert_eq!( - refusal(admit(packet(4, "item:nibu:1", "kay"))), - off_item("item:nibu:1", "kay") - ); - // On the item: a name entity, the speaker, a prior-turn speaker. - admit(packet(5, "item:nibu:2", "kay")).unwrap(); - admit(packet(6, "item:nibu:2", "nibu")).unwrap(); - admit(packet(7, "item:nibu:2", "hero")).unwrap(); - // No packet escapes the item check: a stored entity that no item names is - // refused on every item, and a packet cannot omit its item. - assert_eq!( - refusal(admit(packet(8, "item:nibu:1", "brokkr"))), - off_item("item:nibu:1", "brokkr") - ); - // A referent with no entity needs none; only a packet that cites checks. - let mut anonymous = packet(9, "item:nibu:2", "kay"); - anonymous.referents[0].entity = None; - admit(anonymous).unwrap(); - // Refused packets were not stored. - let stored: Vec = PacketStore::open(&w.dir.path().join("p.cc")) - .unwrap() - .all() - .unwrap() - .iter() - .map(|p| p.id.to_string()) - .collect(); - assert_eq!(stored.len(), 4, "{stored:?}"); -} - -#[test] -fn packet_schema_v2() { - assert_eq!(SCHEMA_VERSION, 4); - let mut p = good(); - p.schema_version = 2; - let Err(refused) = validate(&p, &inventory()) else { - panic!("a v1 packet must not validate") - }; - assert_eq!(refused, vec![Refusal::V1SchemaVersion { found: 2 }]); - - let published: std::collections::BTreeMap<_, _> = - weksa::schema::published().into_iter().collect(); - let packet: serde_json::Value = - serde_json::from_str(&published["weksa.interlingua_packet.schema.json"]).unwrap(); - let referent = &packet["$defs"]["Referent"]["properties"]; - assert!(referent.get("entity").is_some(), "{referent}"); - assert!(referent.get("name").is_none(), "{referent}"); - let defs = packet["$defs"].as_object().unwrap(); - assert!(!defs.keys().any(|k| k.ends_with("Name")), "{defs:?}"); - assert_eq!( - packet["$defs"]["EntityId"]["pattern"], - "^entity:[a-z0-9]([a-z0-9-]{0,46}[a-z0-9])?$" - ); - let item: serde_json::Value = - serde_json::from_str(&published["weksa.corpus_item.schema.json"]).unwrap(); - assert!(item["properties"].get("names").is_some()); - assert!(published.contains_key("weksa.entity.schema.json")); - // v4: the packet carries the speaker's stance, a bare feature list. - assert!(packet["properties"].get("stance").is_some(), "{packet}"); - assert_eq!( - packet["$defs"]["Stance"]["properties"]["features"]["type"], - "array" - ); -} diff --git a/crates/weksa/tests/entity_door.rs b/crates/weksa/tests/entity_door.rs deleted file mode 100644 index bcd53e8..0000000 --- a/crates/weksa/tests/entity_door.rs +++ /dev/null @@ -1,638 +0,0 @@ -//! Behavioural pins from Soul's entity-store s1 probes: the entity door does not -//! trust the writer (ruling entity-door-bounds). -mod common; -use common::*; -use unicode_normalization::UnicodeNormalization; -use weksa::corpus::{CorpusItem, NameSpan}; -use weksa::entity::{Entity, EntityOrigin, LanguageForms}; -use weksa::store::EntityBound; -use weksa::{EntityStore, StoreError, StoreRefusal}; - -fn refusal(r: Result) -> StoreRefusal { - match r { - Err(StoreError::Refused(r)) => r, - other => panic!("expected a store refusal, got {other:?}"), - } -} - -fn item( - n: impl std::fmt::Display, - lang: &str, - text: &str, - names: Vec, - group: &str, -) -> CorpusItem { - CorpusItem { - language: lang.into(), - names, - ..item_in(&format!("item:probe:{n}"), text, group, cast()) - } -} - -fn span(slug: &str, start: usize, end: usize) -> NameSpan { - NameSpan { - entity: eid(slug), - start: start as u32, - end: end as u32, - } -} - -fn bounds(entity: &Entity, bound: EntityBound) -> StoreRefusal { - StoreRefusal::EntityBounds { - id: entity.id.clone(), - bound, - } -} - -#[test] -fn span_never_splits_a_grapheme_cluster() { - let mut w = world(); - let g = group_in_bucket(10); - let splits = |n: u32, start: usize, end: usize| StoreRefusal::NameSpanSplitsCluster { - key: format!("item:probe:{n}"), - start: start as u32, - end: end as u32, - }; - - // Devanagari: the word is a base, a spacing vowel sign and a base; the form - // "र" ends inside the cluster of the first two. - w.entities.put(ent("ra", "hi", &["र"])).unwrap(); - let text = "राम आया"; - assert_eq!( - refusal( - w.corpus - .add(item(1, "hi", text, vec![span("ra", 0, 3)], &g), &w.entities) - ), - splits(1, 0, 3) - ); - // The whole words are legal names, the second starting after a space. - w.entities.put(ent("ram", "hi", &["राम", "आया"])).unwrap(); - let both = vec![span("ram", 0, 9), span("ram", 10, text.len())]; - w.corpus - .add(item(2, "hi", text, both, &g), &w.entities) - .unwrap(); - - // A start inside a cluster is refused too. - w.entities.put(ent("aa", "hi", &["ा"])).unwrap(); - assert_eq!( - refusal(w.corpus.add( - item(3, "hi", "राम x", vec![span("aa", 3, 6)], &g), - &w.entities - )), - splits(3, 3, 6) - ); - - // Latin base + combining mark with no precomposed form, so NFC keeps two - // code points: the form "q" leaves the accent outside. - let q = "q\u{301} arrive."; - w.entities.put(ent("q", "fr", &["q"])).unwrap(); - assert_eq!( - refusal( - w.corpus - .add(item(4, "fr", q, vec![span("q", 0, 1)], &g), &w.entities) - ), - splits(4, 0, 1) - ); - w.entities.put(ent("qa", "fr", &["q\u{301}"])).unwrap(); - w.corpus - .add(item(5, "fr", q, vec![span("qa", 0, 3)], &g), &w.entities) - .unwrap(); -} - -#[test] -fn text_and_forms_are_stored_nfc() { - let mut w = world(); - let g = group_in_bucket(10); - let not_nfc = |key: &str| StoreRefusal::NotNfc { key: key.into() }; - // Decomposed Hangul (conjoining jamo) text is refused, not segmented. - let nfd = "\u{1106}\u{1175}\u{11AB}\u{1109}\u{116E} \u{110B}\u{116A}\u{11BB}\u{1103}\u{1161}"; - assert!(!unicode_normalization::is_nfc(nfd)); - assert_eq!( - refusal(w.corpus.add(item(1, "ko", nfd, vec![], &g), &w.entities)), - not_nfc("item:probe:1") - ); - // The same words composed are admitted, and a span on a syllable pair is exact. - let nfc: String = nfd.nfc().collect(); - let name: String = "\u{1106}\u{1175}\u{11AB}\u{1109}\u{116E}".nfc().collect(); - w.entities.put(ent("minsu", "ko", &[&name])).unwrap(); - w.corpus - .add( - item(2, "ko", &nfc, vec![span("minsu", 0, name.len())], &g), - &w.entities, - ) - .unwrap(); - // Latin NFD text, NFD forms and NFD prior turns are refused. - assert_eq!( - refusal(w.corpus.add( - item(3, "fr", "Rene\u{301} arrive.", vec![], &g), - &w.entities - )), - not_nfc("item:probe:3") - ); - assert_eq!( - refusal(w.entities.put(ent("rene", "fr", &["Rene\u{301}"]))), - not_nfc("entity:rene") - ); - let mut turns = item(4, "fr", "Salut.", vec![], &g); - turns.prior_turns = vec![weksa::Turn { - speaker: eid("nibu"), - text: "Rene\u{301}?".into(), - }]; - assert_eq!( - refusal(w.corpus.add(turns, &w.entities)), - not_nfc("item:probe:4") - ); - // NFC text and an NFC form of the same name agree byte for byte. - w.entities.put(ent("rene", "fr", &["Ren\u{e9}"])).unwrap(); - w.corpus - .add( - item(5, "fr", "Ren\u{e9} part.", vec![span("rene", 0, 5)], &g), - &w.entities, - ) - .unwrap(); -} - -#[test] -fn entity_capacity_is_bounded() { - let mut w = world(); - let lf = |language: &str, forms: Vec| LanguageForms { - language: language.into(), - forms, - }; - let one = |f: &str| vec![f.to_string()]; - let make = |forms: Vec| Entity { - id: eid("big"), - forms, - origin: EntityOrigin::Live { - persona: "p".into(), - }, - }; - - // A form is bounded by bytes, not words: a spaced name of any word count and - // names in scripts that write no spaces are admitted (the Thai name counts - // five UAX #29 words, a Han name one per character). - for (i, name) in [ - "I will never leave you, not ever.", - "Mary of the Isles too", - "de la Cruz y Garcia", - "\u{e2a}\u{e21}\u{e0a}\u{e32}\u{e22} \u{e43}\u{e08}\u{e14}\u{e35}", - "\u{5c71}\u{7530}\u{592a}\u{90ce}", - ] - .into_iter() - .enumerate() - { - w.entities - .put(Entity { - id: eid(&format!("name-{i}")), - ..make(vec![lf("en-US", one(name))]) - }) - .unwrap(); - } - // 64 bytes is the edge; 65 is over, counted in bytes (21 Han characters are - // 63 bytes, 22 are 66). - let wide = make(vec![lf("en-US", one(&"a".repeat(65)))]); - assert_eq!( - refusal(w.entities.put(wide.clone())), - bounds(&wide, EntityBound::FormBytes) - ); - w.entities - .put(Entity { - id: eid("edge"), - ..make(vec![lf("en-US", one(&"a".repeat(64)))]) - }) - .unwrap(); - w.entities - .put(Entity { - id: eid("han-edge"), - ..make(vec![lf("ja", one(&"\u{5c71}".repeat(21)))]) - }) - .unwrap(); - let han = make(vec![lf("ja", one(&"\u{5c71}".repeat(22)))]); - assert_eq!( - refusal(w.entities.put(han.clone())), - bounds(&han, EntityBound::FormBytes) - ); - - // 8 forms per language is the edge. - let forms = |n: usize| (0..n).map(|i| format!("form{i}")).collect::>(); - w.entities - .put(Entity { - id: eid("eight"), - ..make(vec![lf("en-US", forms(8))]) - }) - .unwrap(); - let nine = make(vec![lf("en-US", forms(9))]); - assert_eq!( - refusal(w.entities.put(nine.clone())), - bounds(&nine, EntityBound::Forms) - ); - - // 8 languages is the edge. - let langs = |n: usize| { - (0..n) - .map(|i| lf(&format!("x-l{i}"), one("Form"))) - .collect::>() - }; - w.entities - .put(Entity { - id: eid("langs"), - ..make(langs(8)) - }) - .unwrap(); - let many = make(langs(9)); - assert_eq!( - refusal(w.entities.put(many.clone())), - bounds(&many, EntityBound::Languages) - ); - - // Soul's probe: 200 junk languages of 5000 long forms each. - let long: Vec = (0..5000) - .map(|i| format!("form number {i} of a long free text channel")) - .collect(); - let probe = make( - (0..200) - .map(|l| lf(&format!("x-{l}\n\u{7}"), long.clone())) - .collect(), - ); - assert_eq!( - refusal(w.entities.put(probe.clone())), - bounds(&probe, EntityBound::Languages) - ); - assert!(w.entities.get(&eid("big")).unwrap().is_none()); - - // Language tags are BCP 47 in form. - for bad in [ - "", - "e", - "en_US", - "x-0\n\u{7}", - "-en", - "en-", - "en--US", - "en-US-abcdefghi", - "1n", - "\u{65e5}\u{672c}", - "en-US-aaaaaaaa-bbbbbbbb-cccccccc-dddddddd-eeeeeeee", - "en-aaaaaaaa-aaaaaaaa-aaaaaaaa-bbbbbb", - ] { - let e = make(vec![lf(bad, one("Form"))]); - assert_eq!( - refusal(w.entities.put(e.clone())), - bounds(&e, EntityBound::LanguageTag), - "{bad:?}" - ); - } - for (i, ok) in [ - "en", - "en-US", - "zh-Hant-TW", - "x-klingon", - "de-CH-1996", - "en-aaaaaaaa-aaaaaaaa-aaaaaaaa-bbbbb", - ] - .into_iter() - .enumerate() - { - w.entities - .put(Entity { - id: eid(&format!("tag-{i}")), - ..make(vec![lf(ok, one("Form"))]) - }) - .unwrap(); - } - assert!(w.entities.get(&eid("big")).unwrap().is_none()); -} - -#[test] -fn item_language_is_a_tag() { - let mut w = world(); - let g = group_in_bucket(10); - for (n, bad) in ["", "en_US", "x-0\n\u{7}"].into_iter().enumerate() { - assert!(matches!( - refusal( - w.corpus - .add(item(n + 1, bad, "Nibu left.", vec![], &g), &w.entities) - ), - StoreRefusal::Malformed { .. } - )); - } -} - -#[test] -fn entities_are_append_only_and_replay_needs_them() { - let mut w = world(); - let base = ent("z", "en-US", &["Zed"]); - w.entities.put(base.clone()).unwrap(); - let not_append = StoreRefusal::EntityNotAppendOnly { id: eid("z") }; - - let mut prepend = base.clone(); - prepend.forms.insert( - 0, - LanguageForms { - language: "fr".into(), - forms: vec!["Z\u{e8}de".into()], - }, - ); - assert_eq!(refusal(w.entities.put(prepend)), not_append); - let mut persona = base.clone(); - persona.origin = EntityOrigin::Cast { - persona: "someone-else".into(), - fixture: "f.yaml".into(), - }; - assert_eq!(refusal(w.entities.put(persona)), not_append); - let mut duplicate = base.clone(); - duplicate.forms.push(LanguageForms { - language: "en-US".into(), - forms: vec!["Zeddy".into()], - }); - assert!(matches!( - refusal(w.entities.put(duplicate)), - StoreRefusal::Malformed { .. } - )); - - // A corpus replay against an entity store that lacks the speaker is refused. - let empty = EntityStore::open(&w.dir.path().join("empty.cc")).unwrap(); - let again = CorpusItem { - id: "item:nibu:2".into(), - ..item(1, "en-US", "Nibu says hello.", vec![], &group_in_bucket(10)) - }; - assert_eq!( - refusal(w.corpus.add(again, &empty)), - StoreRefusal::UnknownEntity { - entity: eid("nibu") - } - ); -} - -/// One entity holding `form` as its only en-US form. -fn with_form(slug: &str, form: &str) -> Entity { - Entity { - id: eid(slug), - forms: vec![LanguageForms { - language: "en-US".into(), - forms: vec![form.into()], - }], - origin: EntityOrigin::Live { - persona: "p".into(), - }, - } -} - -#[test] -fn invisible_format_characters_are_refused_in_forms() { - let mut w = world(); - let hidden = |slug: &str| StoreRefusal::Malformed { - key: format!("entity:{slug}"), - why: "a form holds an invisible format character", - }; - let at = |code: u32| char::from_u32(code).unwrap(); - // Every Cf code point except the joiners, plus U+2028 and U+2029, listed - // from Python's unicodedata. The code-point sweep in form.rs pins the edges - // of the rule; this checks the door applies it, in each position. - let listed: Vec = include_str!("format_refused.txt") - .lines() - .filter(|l| !l.starts_with('#')) - .map(|l| u32::from_str_radix(l, 16).unwrap()) - .collect(); - assert_eq!(listed.len(), 170); - for (n, code) in listed.iter().enumerate() { - let c = at(*code); - let mut forms = vec![format!("Ka{c}y")]; - if !c.is_whitespace() { - forms.extend([format!("{c}Kay"), format!("Kay{c}")]); - } - for form in forms { - assert_eq!( - refusal(w.entities.put(with_form("fmt", &form))), - hidden("fmt"), - "U+{code:X} (#{n}) in {form:?}" - ); - } - } - // Default_Ignorable fillers that are not Cf: Hangul and Khmer. - for code in [0x115f, 0x1160, 0x17b4, 0x17b5, 0x3164, 0xffa0] { - assert_eq!( - refusal( - w.entities - .put(with_form("fmt", &format!("Ka{}y", at(code)))) - ), - hidden("fmt"), - "U+{code:X}" - ); - } - assert!(w.entities.get(&eid("fmt")).unwrap().is_none()); - - // The neighbours of the refused ranges, ZWNJ, ZWJ, CGJ, variation - // selectors and the Mongolian selectors (all Mn) are admitted in a form. - let admitted: [u32; 33] = [ - 0xac, - 0xae, - 0x34f, - 0x5ff, - 0x606, - 0x61b, - 0x61d, - 0x6dc, - 0x6de, - 0x70e, - 0x710, - 0x88f, - 0x892, - 0x8e1, - 0x8e3, - 0x180b, - 0x180c, - 0x180d, - 0x180f, - 0x200a, - 0x200c, - 0x200d, - 0x2010, - 0x202f, - 0x2065, - 0x2070, - 0xfe00, - 0xfe0f, - 0xfeff - 1, - 0xfffc, - 0x110be, - 0x1d172, - 0xe0100, - ]; - for (n, code) in admitted.into_iter().enumerate() { - let form = format!("Ka{}y", at(code)); - w.entities - .put(with_form(&format!("ok-{n}"), &form)) - .unwrap_or_else(|e| panic!("U+{code:X}: {e}")); - } - // ZWNJ in a Persian form, ZWJ in an emoji sequence, VS16 and a Mongolian - // free variation selector beside a letter. - for (n, form) in [ - "\u{645}\u{6cc}\u{200c}\u{62e}\u{648}\u{627}\u{647}\u{645}", - "\u{1f469}\u{200d}\u{1f4bb}", - "\u{2764}\u{fe0f}", - "\u{1820}\u{180b}", - "\u{1f600}\u{e0100}", - ] - .into_iter() - .enumerate() - { - w.entities - .put(with_form(&format!("script-{n}"), form)) - .unwrap(); - } - // A form of only joiners, variation selectors, CGJ or Mongolian selectors - // shows nothing and is refused. - let empty_looking = |slug: &str| StoreRefusal::Malformed { - key: format!("entity:{slug}"), - why: "a form is more than joiners and selectors", - }; - for form in [ - "\u{200d}", - "\u{200c}", - "\u{200c}\u{200d}", - "\u{fe0f}", - "\u{fe00}\u{fe0f}", - "\u{34f}", - "\u{180b}\u{180c}\u{180d}\u{180f}", - "\u{e0100}", - "\u{e01ef}\u{34f}\u{200d}", - ] { - assert_eq!( - refusal(w.entities.put(with_form("blank", form))), - empty_looking("blank"), - "{form:?}" - ); - } - assert!(w.entities.get(&eid("blank")).unwrap().is_none()); -} - -#[test] -fn an_origin_attests_where_the_entity_came_from() { - let mut w = world(); - let bad = |slug: &str, why: &'static str| StoreRefusal::Malformed { - key: format!("entity:{slug}"), - why, - }; - let with = |slug: &str, origin: EntityOrigin| Entity { - origin, - ..with_form(slug, "Kay") - }; - let cast = |persona: &str, fixture: &str| EntityOrigin::Cast { - persona: persona.into(), - fixture: fixture.into(), - }; - assert_eq!( - refusal(w.entities.put(with("a", cast("", "f.yaml")))), - bad("a", "an origin names its persona") - ); - assert_eq!( - refusal(w.entities.put(with("b", cast("kay", "")))), - bad("b", "a cast origin names its fixture") - ); - assert_eq!( - refusal(w.entities.put(with( - "c", - EntityOrigin::Live { - persona: String::new() - } - ))), - bad("c", "an origin names its persona") - ); - assert_eq!( - refusal(w.entities.put(with( - "d", - EntityOrigin::PublicDomain { - work: String::new() - } - ))), - bad("d", "a public-domain origin names its work") - ); - for slug in ["a", "b", "c", "d"] { - assert!(w.entities.get(&eid(slug)).unwrap().is_none()); - } - w.entities.put(with("e", cast("kay", "f.yaml"))).unwrap(); - w.entities - .put(with( - "f", - EntityOrigin::PublicDomain { - work: "edda".into(), - }, - )) - .unwrap(); -} - -#[test] -fn an_origin_attestation_holds_something_readable() { - let mut w = world(); - let with = |slug: &str, origin: EntityOrigin| Entity { - origin, - ..with_form(slug, "Kay") - }; - type Build = fn(String) -> EntityOrigin; - let origins: [(Build, &str); 4] = [ - ( - |v| EntityOrigin::Cast { - persona: v, - fixture: "f.yaml".into(), - }, - "an origin names its persona", - ), - ( - |v| EntityOrigin::Cast { - persona: "kay".into(), - fixture: v, - }, - "a cast origin names its fixture", - ), - ( - |v| EntityOrigin::Live { persona: v }, - "an origin names its persona", - ), - ( - |v| EntityOrigin::PublicDomain { work: v }, - "a public-domain origin names its work", - ), - ]; - let unreadable = [ - " ".to_string(), - "\t".into(), - "\n".into(), - "\u{200b}".into(), - "\u{200d}".into(), - "\0".into(), - "--".into(), - "ok\u{200b}ok".into(), - "ok\u{2028}ok".into(), - "ok\0ok".into(), - "x".repeat(201), - "x".repeat(2 * 1024 * 1024), - ]; - let mut n = 0; - for (build, why) in &origins { - for bad in &unreadable { - n += 1; - let slug = format!("o-{n}"); - assert_eq!( - refusal(w.entities.put(with(&slug, build(bad.clone())))), - StoreRefusal::Malformed { - key: format!("entity:{slug}"), - why - }, - "{why}: {:?}", - bad.chars().take(12).collect::() - ); - assert!(w.entities.get(&eid(&slug)).unwrap().is_none()); - } - } - for (m, (build, _)) in origins.iter().enumerate() { - for (k, good) in ["x".repeat(200), "two words, 2 more".to_string()] - .into_iter() - .enumerate() - { - w.entities - .put(with(&format!("g-{}", 2 * m + k), build(good))) - .unwrap(); - } - } -} diff --git a/crates/weksa/tests/format_refused.txt b/crates/weksa/tests/format_refused.txt deleted file mode 100644 index b994f95..0000000 --- a/crates/weksa/tests/format_refused.txt +++ /dev/null @@ -1,171 +0,0 @@ -# Cf minus ZWNJ/ZWJ, plus U+2028 and U+2029; Python 3.13 unicodedata 15.1.0 over 0..10FFFF -AD -600 -601 -602 -603 -604 -605 -61C -6DD -70F -890 -891 -8E2 -180E -200B -200E -200F -2028 -2029 -202A -202B -202C -202D -202E -2060 -2061 -2062 -2063 -2064 -2066 -2067 -2068 -2069 -206A -206B -206C -206D -206E -206F -FEFF -FFF9 -FFFA -FFFB -110BD -110CD -13430 -13431 -13432 -13433 -13434 -13435 -13436 -13437 -13438 -13439 -1343A -1343B -1343C -1343D -1343E -1343F -1BCA0 -1BCA1 -1BCA2 -1BCA3 -1D173 -1D174 -1D175 -1D176 -1D177 -1D178 -1D179 -1D17A -E0001 -E0020 -E0021 -E0022 -E0023 -E0024 -E0025 -E0026 -E0027 -E0028 -E0029 -E002A -E002B -E002C -E002D -E002E -E002F -E0030 -E0031 -E0032 -E0033 -E0034 -E0035 -E0036 -E0037 -E0038 -E0039 -E003A -E003B -E003C -E003D -E003E -E003F -E0040 -E0041 -E0042 -E0043 -E0044 -E0045 -E0046 -E0047 -E0048 -E0049 -E004A -E004B -E004C -E004D -E004E -E004F -E0050 -E0051 -E0052 -E0053 -E0054 -E0055 -E0056 -E0057 -E0058 -E0059 -E005A -E005B -E005C -E005D -E005E -E005F -E0060 -E0061 -E0062 -E0063 -E0064 -E0065 -E0066 -E0067 -E0068 -E0069 -E006A -E006B -E006C -E006D -E006E -E006F -E0070 -E0071 -E0072 -E0073 -E0074 -E0075 -E0076 -E0077 -E0078 -E0079 -E007A -E007B -E007C -E007D -E007E -E007F diff --git a/crates/weksa/tests/forms.rs b/crates/weksa/tests/forms.rs deleted file mode 100644 index 61b62e1..0000000 --- a/crates/weksa/tests/forms.rs +++ /dev/null @@ -1,152 +0,0 @@ -mod common; -use common::*; -use weksa::packet::{EncoderId, Grade, NodeId, PacketId, Referent}; -use weksa::{Family, InvId, OptionLabel, OptionSet}; - -/// Soul's probe: a whole corpus line, and the snake_case form that would -/// pass for an identifier. -const LINE: &str = "Take your hand off that control unless you want your nervous system to become my least useful circuit diagram."; -const SNAKE: &str = "take_your_hand_off_that_control_unless_you_want_your_nervous_system_to_become_my_least_useful_circuit_diagram"; - -#[test] -fn forms_refuse_surface_text() { - for text in [LINE, SNAKE, "", " "] { - assert!(NodeId::parse(text).is_err(), "NodeId {text:?}"); - assert!(PacketId::parse(text).is_err(), "PacketId {text:?}"); - assert!(EncoderId::parse(text).is_err(), "EncoderId {text:?}"); - assert!(InvId::parse(text).is_err(), "InvId {text:?}"); - assert!(OptionLabel::parse(text).is_err(), "OptionLabel {text:?}"); - } - // Soul's extension key `nibu.` is not an inventory id either. - assert!(InvId::parse(&format!("extension:{SNAKE}")).is_err()); - assert!(InvId::parse(&format!("nibu.{SNAKE}")).is_err()); - // NodeId is n, k canonical 1..=9999. - for ok in ["n1", "n10", "n9999"] { - assert_eq!(NodeId::parse(ok).unwrap().as_str(), ok); - } - for bad in [ - "n0", "n01", "n+1", "n-1", "n10000", "n", "1", "N1", "n1 ", "r1", "n1x", - ] { - assert!(NodeId::parse(bad).is_err(), "{bad:?}"); - } - // PacketId and EncoderId are a prefix and exactly 16 lowercase hex digits. - assert!(PacketId::parse(PACKET_ID).is_ok()); - assert!(EncoderId::parse(ENCODER_ID).is_ok()); - for bad in [ - "packet:0123456789abcde", - "packet:0123456789abcdef0", - "packet:0123456789ABCDEF", - "packet:0123456789abcdeg", - "encoder:0123456789abcdef", - "0123456789abcdef", - "packet:item:nibu:1:r2:abc", - ] { - assert!(PacketId::parse(bad).is_err(), "{bad:?}"); - } - assert!(EncoderId::parse("packet:0123456789abcdef").is_err()); - assert!(EncoderId::parse("encoder:0123456789abcdeg").is_err()); - assert!(EncoderId::parse("encoder:0123456789ABCDEF").is_err()); - assert!(EncoderId::parse("encoder:0123456789abcde").is_err()); - assert!(EncoderId::parse("encoder:0123456789abcdef0").is_err()); -} - -#[test] -fn a_refusal_names_the_form_and_the_value_and_a_form_reads_back_as_written() { - let e = NodeId::parse("n0").unwrap_err(); - assert_eq!(e.form, "NodeId"); - assert_eq!(e.got, "n0"); - assert_eq!(e.to_string(), "\"n0\" is not a NodeId"); - assert_eq!( - Grade::new(1001).unwrap_err().to_string(), - "\"1001\" is not a Grade" - ); - assert_eq!(EncoderId::parse(ENCODER_ID).unwrap().as_str(), ENCODER_ID); - assert_eq!(PacketId::parse(PACKET_ID).unwrap().as_str(), PACKET_ID); - assert_eq!( - EncoderId::parse(ENCODER_ID).unwrap().to_string(), - ENCODER_ID - ); - assert_eq!(lab("potential").as_str(), "potential"); -} - -#[test] -fn a_malformed_value_does_not_deserialize() { - // A node id that is the snake line fails where a store would read it. - let mut referent = serde_json::to_value(good().referents[0].clone()).unwrap(); - assert!(serde_json::from_value::(referent.clone()).is_ok()); - referent["id"] = SNAKE.into(); - assert!(serde_json::from_value::(referent.clone()).is_err()); - referent["id"] = "n1".into(); - referent["concept"] = "concept:01".into(); - assert!(serde_json::from_value::(referent).is_err()); - assert!(serde_json::from_str::("1001").is_err()); - assert_eq!(serde_json::from_str::("1000").unwrap().get(), 1000); - assert!(serde_json::from_str::("[]").is_err()); -} - -#[test] -fn inv_id_canonical() { - for bad in [ - "concept:01", - "concept:+1", - "concept:0", - "concept:-1", - "concept:", - "concept:1x", - "concept: 1", - "nofamily:1", - "Concept:1", - "concept", - "concept:99999999999", - "concept:1:2", - ] { - assert!(InvId::parse(bad).is_err(), "{bad:?}"); - } - let id = InvId::parse("concept:1").unwrap(); - assert_eq!(id.family(), Family::Concept); - assert_eq!(id.to_string(), "concept:1"); - for f in Family::ALL { - let id = InvId::parse(&format!("{}:12", f.prefix())).unwrap(); - assert_eq!(id.family(), f); - assert_eq!(id.to_string(), format!("{}:12", f.prefix())); - } - assert_eq!(iv("extension:1").family(), Family::Extension); -} - -#[test] -fn options_bounded() { - let labels = |n: usize| (0..n).map(|i| lab(&format!("l{i}"))).collect::>(); - assert!(OptionSet::new(labels(0)).is_err()); - assert!(OptionSet::new(labels(1)).is_ok()); - assert!(OptionSet::new(labels(16)).is_ok()); - assert!(OptionSet::new(labels(17)).is_err()); - assert!(OptionSet::new(vec![lab("a"), lab("a")]).is_err()); - assert!(OptionSet::new(vec![lab("a"), lab("b"), lab("a")]).is_err()); - // Soul's probe: the snake line is not a label; labels are short snake words. - assert!(OptionLabel::parse(SNAKE).is_err()); - assert!(OptionLabel::parse(&"a".repeat(24)).is_ok()); - assert!(OptionLabel::parse(&"a".repeat(25)).is_err()); - for bad in ["A", "1a", "_a", "a-b", "a b", "a.b", "\u{e9}"] { - assert!(OptionLabel::parse(bad).is_err(), "{bad:?}"); - } - for ok in ["a", "a1", "a_b", "potential"] { - assert!(OptionLabel::parse(ok).is_ok(), "{ok:?}"); - } -} - -#[test] -fn grade_bound_is_the_type() { - assert!(Grade::new(1001).is_err()); - assert_eq!(Grade::new(1000).unwrap().get(), 1000); - assert_eq!(Grade::new(0).unwrap().get(), 0); - // The published schema carries the same bound. - let (_, json) = weksa::schema::published() - .into_iter() - .find(|(n, _)| n == "weksa.interlingua_packet.schema.json") - .unwrap(); - let schema: serde_json::Value = serde_json::from_str(&json).unwrap(); - let grade = &schema["$defs"]["Grade"]; - assert_eq!(grade["maximum"], 1000, "{grade}"); - assert_eq!(grade["minimum"], 0, "{grade}"); - assert_eq!(schema["$defs"]["NodeId"]["pattern"], "^n[1-9][0-9]{0,3}$"); -} diff --git a/crates/weksa/tests/lower.rs b/crates/weksa/tests/lower.rs deleted file mode 100644 index f50f096..0000000 --- a/crates/weksa/tests/lower.rs +++ /dev/null @@ -1,212 +0,0 @@ -//! The English lowerer against the committed inventory (revision 2) and the -//! `nibu-proof-001` packet, and the rules that keep a table row from speaking -//! for a meaning it does not carry. - -mod common; -use std::path::Path; - -use common::{iv, nd}; -use weksa::inventory::proof_revision; -use weksa::packet::{FeatValue, FeatureValue, Packet, proof_packet}; -use weksa::{ - CorpusStore, EntityStore, Inventory, InventoryStore, LowerError, LoweringInput, PacketStore, - lower, validate, -}; - -const PROOF: &str = "Warning: you may endanger yourself by touching the control. \ - The ship mind will prevent that to protect you."; - -fn committed() -> (Inventory, tempfile::TempDir) { - let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); - let inv = InventoryStore::open(&root.join("data/interlingua/inventory.cc")).unwrap(); - (inv.inventory().unwrap(), tempfile::tempdir().unwrap()) -} - -/// A `LoweringInput` has one source, `PacketStore::admit`, so every packet -/// under test goes through a throwaway store backed by the committed stores. -fn view(packet: &Packet, _inv: &Inventory) -> LoweringInput { - let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); - let inventory = InventoryStore::open(&root.join("data/interlingua/inventory.cc")).unwrap(); - admitted(packet, &inventory) -} - -/// The same door against an inventory store the test built. -fn admitted(packet: &Packet, inventory: &InventoryStore) -> LoweringInput { - let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); - let corpus = CorpusStore::open(&root.join("data/corpus/corpus.cc")).unwrap(); - let entities = EntityStore::open(&root.join("data/entity/entity.cc")).unwrap(); - let dir = tempfile::tempdir().unwrap(); - PacketStore::open(&dir.path().join("p.cc")) - .unwrap() - .admit(packet.clone(), inventory, &corpus, &entities) - .unwrap() -} - -fn lowered(packet: &Packet, inv: &Inventory) -> Result { - lower(&view(packet, inv), inv) -} - -#[test] -fn the_proof_packet_is_admitted_by_the_store_and_lowers_to_its_sentences() { - let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); - let (inv, dir) = committed(); - let inventory = InventoryStore::open(&root.join("data/interlingua/inventory.cc")).unwrap(); - let corpus = CorpusStore::open(&root.join("data/corpus/corpus.cc")).unwrap(); - let entities = EntityStore::open(&root.join("data/entity/entity.cc")).unwrap(); - let mut packets = PacketStore::open(&dir.path().join("p.cc")).unwrap(); - packets - .admit(proof_packet(), &inventory, &corpus, &entities) - .unwrap(); - let stored = packets.get("packet:6e6962752d303031").unwrap().unwrap(); - assert_eq!(lowered(&stored, &inv).unwrap(), PROOF); -} - -#[test] -fn only_the_addressee_is_you_and_is_reflexive() { - let (inv, _dir) = committed(); - let mut p = proof_packet(); - p.discourse.addressee = None; - assert_eq!( - lowered(&p, &inv).unwrap(), - "Warning: the player may endanger the player by touching the control. \ - The ship mind will prevent that to protect the player." - ); -} - -#[test] -fn no_speech_act_means_no_opener_and_a_capital() { - let (inv, _dir) = committed(); - let mut p = proof_packet(); - p.discourse.speech_act = None; - assert!(lowered(&p, &inv).unwrap().starts_with("You may endanger")); - p.discourse.speech_act = Some(iv("speech_act:1")); - assert!(lowered(&p, &inv).unwrap().starts_with("You may endanger")); -} - -#[test] -fn a_packet_carrying_what_the_lowerer_does_not_render_is_refused() { - let dir = tempfile::tempdir().unwrap(); - let mut store = InventoryStore::open(&dir.path().join("i.cc")).unwrap(); - let (rev1, entries) = weksa::inventory::seed(); - store.admit_revision(rev1, entries).unwrap(); - let (rev2, entries) = proof_revision(); - store.admit_revision(rev2, entries).unwrap(); - let mut ext = common::entry("extension:1", 3); - ext.value_kind = Some(weksa::ValueKind::Grade); - common::admit(&mut store, 3, vec![ext]); - let inv = store.inventory().unwrap(); - let discourse = |p: &mut Packet| { - p.discourse.features.push(FeatureValue { - feature: iv("feature:6"), - value: FeatValue::Grade(common::grade(300)), - }) - }; - let predication = |p: &mut Packet| { - p.predications[0].features.push(FeatureValue { - feature: iv("feature:1"), - value: FeatValue::Option(common::lab("negative")), - }) - }; - let relation = |p: &mut Packet| { - p.relations.push(weksa::packet::Relation { - relation: iv("relation:8"), - from: nd("n4"), - to: nd("n5"), - features: vec![], - }) - }; - let extension = |p: &mut Packet| { - p.inventory_rev = 3; - p.extensions.push(weksa::packet::Extension { - key: iv("extension:1"), - value: FeatValue::Grade(common::grade(3)), - }) - }; - let referent_feature = |p: &mut Packet| { - p.referents[0].features.push(FeatureValue { - feature: iv("feature:6"), - value: FeatValue::Grade(common::grade(1)), - }) - }; - let no_predication = |p: &mut Packet| p.predications.clear(); - let extra_role = |p: &mut Packet| { - let filler = p.predications[0].roles[0].filler.clone(); - p.predications[0].roles.push(weksa::packet::RoleFill { - role: iv("role:3"), - filler, - }) - }; - let lone_referent = |p: &mut Packet| { - let mut r = p.referents[0].clone(); - r.id = nd("n99"); - p.referents.push(r) - }; - for (what, edit) in [ - ("features", &discourse as &dyn Fn(&mut Packet)), - ("features", &predication), - ("relations", &relation), - ("extensions", &extension), - ("no predication", &no_predication), - ("a role the frame does not name", &extra_role), - ("a referent no role mentions", &lone_referent), - ] { - let mut p = proof_packet(); - edit(&mut p); - assert_eq!( - lower(&admitted(&p, &store), &inv), - Err(LowerError::Unrendered { what }), - "{what}" - ); - } - // The seed lets no feature sit on a referent, so that arm is refused by validate. - let mut p = proof_packet(); - referent_feature(&mut p); - assert!(validate(&p, &inv).is_err()); -} - -#[test] -fn an_unknown_row_and_an_unfilled_role_are_refused_by_name() { - let (inv, _dir) = committed(); - let mut p = proof_packet(); - p.discourse.speech_act = Some(iv("speech_act:3")); - assert_eq!( - lowered(&p, &inv), - Err(LowerError::NoRow { - id: iv("speech_act:3") - }) - ); - let mut p = proof_packet(); - p.predications[0].roles.pop(); - assert_eq!( - lowered(&p, &inv), - Err(LowerError::MissingRole { - predication: nd("n4"), - role: "patient".into() - }) - ); -} - -#[test] -fn a_row_cannot_speak_for_an_id_whose_gloss_differs() { - // Revision 2 with concept:4 glossed "hatch" in place of "control": the row - // for concept:4 says "control" and must not render it. - let dir = tempfile::tempdir().unwrap(); - let mut store = InventoryStore::open(&dir.path().join("i.cc")).unwrap(); - let (rev1, entries) = weksa::inventory::seed(); - store.admit_revision(rev1, entries).unwrap(); - let (rev2, mut entries) = proof_revision(); - for e in &mut entries { - if e.id == iv("concept:4") { - e.gloss = "hatch".into(); - } - } - store.admit_revision(rev2, entries).unwrap(); - let inv = store.inventory().unwrap(); - assert_eq!( - lower(&admitted(&proof_packet(), &store), &inv), - Err(LowerError::GlossMismatch { - id: iv("concept:4"), - row: "control" - }) - ); -} diff --git a/crates/weksa/tests/score.rs b/crates/weksa/tests/score.rs deleted file mode 100644 index 6920b93..0000000 --- a/crates/weksa/tests/score.rs +++ /dev/null @@ -1,692 +0,0 @@ -mod common; - -use std::cell::RefCell; -use std::io::{BufRead, BufReader, Read, Write}; -use std::net::TcpListener; -use std::sync::{Arc, Mutex}; - -use anyhow::Result; -use common::{cast, group_in_bucket, item_in, model}; -use serde_json::Value; -use weksa::CorpusItem; -use weksa::score::{ - Embedder, ItemScore, OllamaEmbedder, ScoreInput, Scorer, aggregate, chrf, cosine, - leak_diagnostic, -}; - -/// (reference, hypothesis, sacrebleu 2.4.3 sentence_chrf), map 491536c. -const VECTORS: [(&str, &str, f64); 6] = [ - ( - "Take your hand off that control unless you want your nervous system to become my least useful circuit diagram.", - "Take your hand off that control, unless you'd like your nervous system to be my least useful wiring diagram.", - 74.4498, - ), - ( - "The command has no receipt. It is not truth.", - "The order carries no receipt, so it is not true.", - 45.7164, - ), - ("I am not angry.", "I am not angry.", 100.0), - ("Forgive me, sir.", "Get out.", 3.9683), - ("ãã®åˆ¶å¾¡ã«è§¦ã‚‹ãªã€‚", "ãã®æ“作盤ã«è§¦ã‚‰ãªã„ã§ã€‚", 14.2931), - ("Well, aren't you a sight for sore eyes, love.", "", 0.0), -]; - -#[test] -fn chrf_matches_sacrebleu_vectors() { - for (reference, hypothesis, want) in VECTORS { - let got = chrf(hypothesis, reference); - assert!( - (got - want).abs() < 1e-4, - "{hypothesis:?} against {reference:?}: {got} != {want}" - ); - } -} - -#[test] -fn chrf_argument_order() { - let short = "mind the hatch"; - let long = "mind the hatch before the storm arrives tonight"; - let (as_hypothesis, as_reference) = (chrf(short, long), chrf(long, short)); - assert!( - as_hypothesis < as_reference - 1.0, - "recall is weighted: a short hypothesis of a long reference ({as_hypothesis}) must score below the reverse ({as_reference})" - ); -} - -#[derive(Default)] -struct Recorder { - seen: RefCell>, -} - -impl Embedder for Recorder { - fn model(&self) -> &str { - "fake" - } - fn embed(&self, texts: &[&str]) -> Result>> { - self.seen - .borrow_mut() - .extend(texts.iter().map(|t| t.to_string())); - Ok(texts.iter().map(|t| vec![t.len() as f32, 1.0]).collect()) - } -} - -/// A deterministic bag-of-letters embedder, so cosine differs with the text. -struct Bag; - -impl Embedder for Bag { - fn model(&self) -> &str { - "bag" - } - fn embed(&self, texts: &[&str]) -> Result>> { - Ok(texts - .iter() - .map(|t| { - let mut v = vec![0.0; 27]; - v[26] = 1.0; - for c in t.chars().filter(char::is_ascii_alphabetic) { - v[(c.to_ascii_lowercase() as u8 - b'a') as usize] += 1.0; - } - v - }) - .collect()) - } -} - -fn score_one(scorer: &Scorer, reference: &str, output: &str) -> ItemScore { - scorer - .score(&[ScoreInput { reference, output }]) - .unwrap() - .remove(0) -} - -fn close(got: f64, want: f64) { - assert!((got - want).abs() < 1e-4, "{got} != {want}"); -} - -#[test] -fn chrf_whitespace_parity() { - // sacrebleu 2.4.3 strips str.split's whitespace; every vector scores 100 - let vectors = [ - ("a\tb c", "ab c"), - ("a\u{a0}b", "ab"), - ("a\u{1c}b", "ab"), - ("a\u{85}b", "ab"), - ("a\u{2028}b c", "ab c"), - ("line one\nline two", "line one line two"), - ("ãã®\u{3000}制御", "ãã®åˆ¶å¾¡"), - ]; - for (hypothesis, reference) in vectors { - assert_eq!(chrf(hypothesis, reference), 100.0, "{hypothesis:?}"); - assert_eq!(chrf(reference, hypothesis), 100.0, "{hypothesis:?}"); - } - // a zero-width space is not whitespace - close(chrf("a\u{200b}b", "ab"), 45.4545); -} - -#[test] -fn scorer_argument_order() { - let short = "mind the hatch"; - let long = "mind the hatch before the storm arrives tonight"; - let scorer = Scorer::new(Bag); - let got = score_one(&scorer, long, short); - assert_eq!(got.chrf, chrf(short, long)); - assert_ne!(got.chrf, chrf(long, short)); -} - -#[test] -fn meaning_none_on_empty_text() { - let scorer = Scorer::new(Recorder::default()); - let inputs = [ - ScoreInput { - reference: "Nibu says hi", - output: " ", - }, - ScoreInput { - reference: " ", - output: "says hi", - }, - ScoreInput { - reference: "Nibu says hi", - output: "says hi", - }, - ]; - let scores = scorer.score(&inputs).unwrap(); - assert_eq!(scores[0].meaning, None); - assert_eq!(scores[1].meaning, None); - assert!(scores[2].meaning.is_some()); - assert_eq!( - *scorer.embedder().seen.borrow(), - ["Nibu says hi", "says hi"] - ); - assert_eq!(scores[0].chrf, 0.0); -} - -/// Answers every embed call with the same vectors. -struct Fixed(Vec>); - -impl Embedder for Fixed { - fn model(&self) -> &str { - "fixed" - } - fn embed(&self, _texts: &[&str]) -> Result>> { - Ok(self.0.clone()) - } -} - -#[test] -fn scorer_refuses_bad_vectors() { - let run = |vectors: Vec>| { - Scorer::new(Fixed(vectors)).score(&[ScoreInput { - reference: "mind the hatch", - output: "mind the door", - }]) - }; - assert!(run(vec![vec![1.0, 2.0], vec![2.0, 1.0]]).is_ok()); - for bad in [ - vec![vec![1.0, 2.0], vec![1.0]], - vec![vec![1.0], vec![1.0, 2.0]], - vec![vec![], vec![]], - vec![vec![0.0, 0.0], vec![0.0, 0.0]], - vec![vec![1.0, 2.0], vec![0.0, 0.0]], - vec![vec![0.0, 0.0], vec![1.0, 2.0]], - vec![vec![1.0, f32::NAN], vec![1.0, 2.0]], - vec![vec![1.0, 2.0]], - ] { - assert!(run(bad.clone()).is_err(), "{bad:?}"); - } -} - -#[test] -fn cosine_precondition() { - assert!((cosine(&[1.0, 2.0], &[2.0, 4.0]) - 1.0).abs() < 1e-9); - assert!(cosine(&[1.0, 0.0], &[0.0, 1.0]).abs() < 1e-9); - for (a, b) in [ - (vec![1.0], vec![1.0, 1.0]), - (vec![], vec![]), - (vec![0.0, 0.0], vec![1.0, 1.0]), - ] { - let refused = std::panic::catch_unwind(|| cosine(&a, &b)); - assert!(refused.is_err(), "{a:?} {b:?}"); - } -} - -/// A plain-HTTP fake of `/api/embed`. Records each request body; answers with -/// `dim` floats per input, one fewer vector than asked when `drop_one`, and a -/// short last vector when `short_last`. -fn fake_ollama(drop_one: bool, short_last: bool) -> (String, Arc>>) { - let listener = TcpListener::bind("127.0.0.1:0").unwrap(); - let url = format!("http://{}", listener.local_addr().unwrap()); - let bodies = Arc::new(Mutex::new(Vec::new())); - let seen = Arc::clone(&bodies); - std::thread::spawn(move || { - for stream in listener.incoming() { - let mut stream = stream.unwrap(); - let mut reader = BufReader::new(stream.try_clone().unwrap()); - 'conn: loop { - let mut length = 0; - let mut line = String::new(); - let mut first = true; - loop { - line.clear(); - if reader.read_line(&mut line).unwrap_or(0) == 0 { - break 'conn; - } - if first { - assert!(line.starts_with("POST /api/embed "), "{line}"); - first = false; - } - let lower = line.to_ascii_lowercase(); - if let Some(v) = lower.strip_prefix("content-length:") { - length = v.trim().parse().unwrap(); - } - if line == "\r\n" { - break; - } - } - let mut body = vec![0u8; length]; - reader.read_exact(&mut body).unwrap(); - let body: Value = serde_json::from_slice(&body).unwrap(); - let mut count = body["input"].as_array().unwrap().len(); - seen.lock().unwrap().push(body); - if drop_one { - count -= 1; - } - let vectors: Vec> = (0..count) - .map(|i| vec![1.0; if short_last && i + 1 == count { 2 } else { 3 }]) - .collect(); - let reply = serde_json::json!({ "embeddings": vectors }).to_string(); - write!( - stream, - "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\n\r\n{reply}", - reply.len() - ) - .unwrap(); - } - } - }); - (url, bodies) -} - -fn lengths(bodies: &Mutex>) -> Vec { - let bodies = bodies.lock().unwrap(); - bodies - .iter() - .map(|b| b["input"].as_array().unwrap().len()) - .collect() -} - -#[test] -fn ollama_request_shape_and_batching() { - let (url, bodies) = fake_ollama(false, false); - let embedder = OllamaEmbedder::new(&url, "qwen3-embedding:0.6b"); - let texts: Vec = (0..70).map(|i| format!("text {i}")).collect(); - let refs: Vec<&str> = texts.iter().map(String::as_str).collect(); - let vectors = embedder.embed(&refs).unwrap(); - assert_eq!(vectors.len(), 70); - assert_eq!(lengths(&bodies), [32, 32, 6]); - for body in bodies.lock().unwrap().iter() { - assert_eq!(body["model"], "qwen3-embedding:0.6b"); - assert_eq!(body["options"]["num_thread"], 8); - } - assert_eq!(bodies.lock().unwrap()[2]["input"][5], "text 69"); - - let (url, _) = fake_ollama(true, false); - let err = OllamaEmbedder::new(&url, "m") - .embed(&["a", "b"]) - .unwrap_err(); - assert!(err.to_string().contains("got 1"), "{err}"); - - let (url, _) = fake_ollama(false, true); - let err = OllamaEmbedder::new(&url, "m") - .embed(&["a", "b"]) - .unwrap_err(); - assert!(err.to_string().contains("dimension"), "{err}"); -} - -#[test] -fn ollama_clamps_batch_and_omits_unset_num_thread() { - let (url, bodies) = fake_ollama(false, false); - let mut embedder = OllamaEmbedder::new(&url, "m"); - embedder.batch = 500; - embedder.num_thread = None; - let texts: Vec = (0..40).map(|i| i.to_string()).collect(); - let refs: Vec<&str> = texts.iter().map(String::as_str).collect(); - embedder.embed(&refs).unwrap(); - assert_eq!(lengths(&bodies), [32, 8]); - assert!(bodies.lock().unwrap()[0].get("options").is_none()); - assert!(embedder.embed(&[]).unwrap().is_empty()); - assert_eq!(lengths(&bodies).len(), 2, "no request for no texts"); -} - -#[test] -#[ignore = "needs WEKSA_OLLAMA_URL, Yggdrasil's ollama-cpu"] -fn ollama_on_yggdrasil() { - let url = std::env::var("WEKSA_OLLAMA_URL").unwrap(); - let embedder = OllamaEmbedder::new(url, "qwen3-embedding:0.6b"); - let text = "Take your hand off that control."; - let vectors = embedder.embed(&[text, text]).unwrap(); - assert_eq!(vectors.len(), 2); - assert!(vectors.iter().all(|v| v.len() == 1024)); - assert!((cosine(&vectors[0], &vectors[1]) - 1.0).abs() < 1e-4); -} - -fn score(chrf: f64, meaning: Option) -> ItemScore { - ItemScore { chrf, meaning } -} - -#[test] -fn aggregate_by_derived_split() { - let train = item_in("item:a:1", "t", &group_in_bucket(10), cast()); - let train2 = item_in("item:a:2", "t", &group_in_bucket(10), cast()); - let validation = item_in("item:b:1", "v", &group_in_bucket(50), cast()); - let test = item_in("item:c:1", "x", &group_in_bucket(90), cast()); - let test_model = item_in("item:c:2", "x", &group_in_bucket(90), model()); - let agg = aggregate(&[ - (&train, score(10.0, Some(0.5))), - (&train2, score(20.0, None)), - (&validation, score(40.0, Some(0.8))), - (&test, score(60.0, Some(0.6))), - (&test_model, score(100.0, Some(1.0))), - ]); - assert_eq!(agg.train.count, 2); - assert_eq!(agg.train.chrf, Some(15.0)); - assert_eq!(agg.train.meaning, Some(0.5), "None is not averaged as zero"); - assert_eq!((agg.validation.count, agg.validation.chrf), (1, Some(40.0))); - assert_eq!((agg.test.count, agg.test.chrf), (1, Some(60.0))); - assert_eq!( - agg.test.meaning, - Some(0.6), - "the model-written test item is in no split" - ); - assert_eq!(aggregate(&[]).validation.chrf, None); -} - -const HELD_OUT: &str = "extraordinary lighthouse keepers whispered unforgettable stories throughout midnight thunderstorms before surrendering peacefully"; -const UNRELATED: &str = "harbor ledgers record ordinary tides"; - -fn chunks(size: usize, joiner: &str) -> Vec { - HELD_OUT - .split(' ') - .collect::>() - .chunks(size) - .map(|c| c.join(joiner)) - .collect() -} - -fn validation(key: &str, text: &str) -> CorpusItem { - item_in(key, text, &group_in_bucket(50), cast()) -} - -fn train(key: &str, text: &str) -> CorpusItem { - item_in(key, text, &group_in_bucket(10), cast()) -} - -/// (novel, covered) per order for the Validation split. -fn read(minted: &[&str], items: &[CorpusItem]) -> Vec<(usize, usize)> { - leak_diagnostic(minted, items) - .validation - .coverage_by_n - .iter() - .map(|c| (c.novel, c.covered)) - .collect() -} - -fn refs(strings: &[String]) -> Vec<&str> { - strings.iter().map(String::as_str).collect() -} - -#[test] -fn leak_diagnostic_catches_chunked_glosses() { - let minted = chunks(3, " "); - let minted = refs(&minted); - let held = validation("item:v:1", HELD_OUT); - let unrelated = validation("item:v:2", "gardening is quiet work"); - let train_item = train("item:t:1", UNRELATED); - let test_model = item_in("item:x:1", HELD_OUT, &group_in_bucket(90), model()); - - let d = leak_diagnostic( - &minted, - &[ - held.clone(), - unrelated.clone(), - train_item.clone(), - test_model, - ], - ); - let pair = d.validation.max_chrf.as_ref().unwrap(); - assert!( - pair.chrf < 50.0, - "each chunk alone scores low: {}", - pair.chrf - ); - assert_eq!(pair.item, "item:v:1"); - // pooled over both items: 12+4 words, 11+3 bigrams, 10+2 trigrams - assert_eq!( - read(&minted, &[held.clone(), unrelated, train_item.clone()]), - [(16, 12), (14, 8), (12, 4)] - ); - let worst = d.validation.worst.as_ref().unwrap(); - assert_eq!( - (worst.item.as_str(), worst.n, worst.covered, worst.novel), - ("item:v:1", 2, 8, 11) - ); - assert_eq!(d.validation.items, 2); - assert_eq!(d.test.items, 0, "a model-written test item is in no split"); - assert_eq!(d.test.max_chrf, None); - assert_eq!(d.test.worst, None); - - let only_train = leak_diagnostic(&minted, &[train_item]); - assert_eq!(only_train.validation.items, 0); - assert_eq!(only_train.validation.max_chrf, None); - assert_eq!(only_train.validation.worst, None); - - let full = leak_diagnostic(&[HELD_OUT], &[held]); - assert_eq!(full.validation.coverage_by_n[2].share, Some(1.0)); - assert!(full.validation.max_chrf.unwrap().chrf > 99.0); -} - -#[test] -fn coverage_reads_joined_labels() { - let items = [ - validation("item:v:1", HELD_OUT), - train("item:t:1", UNRELATED), - ]; - for size in [1, 2, 3] { - let plain = chunks(size, " "); - let want = read(&refs(&plain), &items); - for joiner in ["_", "."] { - let joined = chunks(size, joiner); - assert_eq!(read(&refs(&joined), &items), want, "{size} {joiner:?}"); - } - // camelCase: each word after the first capitalised within a chunk - let camel: Vec = chunks(size, " ") - .iter() - .map(|c| { - c.split(' ') - .enumerate() - .map(|(i, w)| match i { - 0 => w.to_string(), - _ => w[..1].to_uppercase() + &w[1..], - }) - .collect::() - }) - .collect(); - assert_eq!(read(&refs(&camel), &items), want, "{size} camel"); - } - let items = [ - validation("item:v:1", "He will kick the bucket"), - train("item:t:1", "he will go"), - ]; - for label in [ - "kick_the_bucket", - "kick.the.bucket", - "kickTheBucket", - "kick the bucket", - ] { - let d = leak_diagnostic(&[label], &items); - let by_n = &d.validation.coverage_by_n; - assert_eq!((by_n[2].novel, by_n[2].covered), (3, 1), "{label}"); - let worst = d.validation.worst.unwrap(); - assert_eq!((worst.n, worst.covered), (2, 2), "{label}"); - } -} - -#[test] -fn coverage_is_case_insensitive() { - let items = [ - validation("item:v:1", HELD_OUT), - train("item:t:1", UNRELATED), - ]; - let minted = chunks(2, " "); - let shouted: Vec = minted.iter().map(|m| m.to_uppercase()).collect(); - assert_eq!(read(&refs(&shouted), &items), read(&refs(&minted), &items)); - assert_eq!(read(&refs(&minted), &items)[0], (12, 12)); -} - -#[test] -fn coverage_pooled_per_order() { - // honest single-word glosses saturate n=1 and read nothing above it - let items = [ - train( - "item:t:1", - "They left before the storm and found the harbor.", - ), - validation("item:v:1", "They left the group before the storm."), - validation("item:v:2", "They found the place after the storm."), - validation("item:v:3", "Warn them before they leave the harbor."), - ]; - let glosses = [ - "left", "group", "found", "place", "storm", "after", "warn", "leave", "harbor", - ]; - let d = leak_diagnostic(&glosses, &items); - let by_n = &d.validation.coverage_by_n; - assert_eq!(by_n.iter().map(|c| c.n).collect::>(), [1, 2, 3]); - assert_eq!((by_n[0].novel, by_n[0].covered), (6, 5)); - assert!(by_n[0].share.unwrap() < 1.0); - for c in &by_n[1..] { - assert!(c.novel > 0); - assert_eq!((c.covered, c.share), (0, Some(0.0))); - } - assert_eq!(d.validation.worst, None); - - // no novel n-gram reads None, never NaN; an empty split reads the same - let items = [ - train("item:t:1", "tides harbors guard the quiet shore"), - validation("item:v:1", "harbors guard the quiet"), - ]; - let d = leak_diagnostic(&["anything at all"], &items); - for split in [&d.validation, &d.test] { - assert_eq!(split.coverage_by_n.len(), 3); - assert!( - split - .coverage_by_n - .iter() - .all(|c| (c.novel, c.share) == (0, None)) - ); - assert_eq!(split.worst, None); - } -} - -#[test] -fn coverage_keeps_digits_and_strings() { - let items = [ - validation("item:v:1", "code 0400 open"), - train("item:t:1", "code open"), - ]; - assert_eq!(read(&["0400"], &items)[0], (1, 1)); - // a bigram spanning two minted strings is not covered - let items = [validation("item:v:1", "zeta eta")]; - assert_eq!(read(&["zeta", "eta"], &items), [(2, 2), (1, 0), (0, 0)]); - assert_eq!(read(&["zeta eta"], &items), [(2, 2), (1, 1), (0, 0)]); -} - -#[test] -fn worst_item_rules() { - // equal covered counts: the larger share wins over an earlier, smaller one - let items = [ - validation("item:v:1", "alpha beta gamma"), - validation("item:v:2", "delta epsilon"), - ]; - let w = leak_diagnostic(&["alpha beta", "delta epsilon"], &items) - .validation - .worst - .unwrap(); - assert_eq!( - (w.item.as_str(), w.n, w.covered, w.novel), - ("item:v:2", 2, 1, 1) - ); - // a later, smaller share does not displace an earlier larger one - let items = [ - validation("item:v:1", "delta epsilon"), - validation("item:v:2", "alpha beta gamma"), - ]; - let w = leak_diagnostic(&["alpha beta", "delta epsilon"], &items) - .validation - .worst - .unwrap(); - assert_eq!(w.item, "item:v:1"); - // a larger covered count beats a larger share - let items = [ - validation("item:v:1", "delta epsilon"), - validation("item:v:2", "alpha beta gamma delta"), - ]; - let w = leak_diagnostic(&["alpha beta gamma", "delta epsilon"], &items) - .validation - .worst - .unwrap(); - assert_eq!((w.item.as_str(), w.covered, w.novel), ("item:v:2", 2, 3)); - // full ties go to the first item, then the smaller n - let items = [ - validation("item:v:1", "delta epsilon"), - validation("item:v:2", "delta epsilon"), - ]; - let w = leak_diagnostic(&["delta epsilon"], &items) - .validation - .worst - .unwrap(); - assert_eq!(w.item, "item:v:1"); - let items = [ - validation("item:v:1", "alpha beta gamma"), - train("item:t:1", "alpha beta"), - ]; - let w = leak_diagnostic(&["alpha beta gamma"], &items) - .validation - .worst - .unwrap(); - assert_eq!((w.n, w.covered, w.novel), (2, 1, 1)); -} - -#[test] -fn ollama_reports_its_model() { - assert_eq!( - OllamaEmbedder::new("http://x", "qwen3-embedding:0.6b").model(), - "qwen3-embedding:0.6b" - ); -} - -#[test] -fn leak_diagnostic_reports_the_best_pair_and_the_first_of_ties() { - let a = validation("item:v:1", HELD_OUT); - let b = validation("item:v:2", HELD_OUT); - // the better minted string comes second; equal items tie, the first is kept - let d = leak_diagnostic(&["zzzzzz qqqqqq", HELD_OUT], &[a, b]); - let pair = d.validation.max_chrf.unwrap(); - assert_eq!( - (pair.minted.as_str(), pair.item.as_str()), - (HELD_OUT, "item:v:1") - ); -} - -#[test] -fn scores_whole_texts() { - let reference = "He is going to kick the bucket tomorrow."; - let output = "He is going to die tomorrow."; - // exhaustive: a scorer input that grows a field breaks this line - let ScoreInput { reference, output } = ScoreInput { reference, output }; - let scorer = Scorer::new(Recorder::default()); - let got = score_one(&scorer, reference, output); - assert_eq!(*scorer.embedder().seen.borrow(), [reference, output]); - assert_eq!(got.chrf, chrf(output, reference)); -} - -#[test] -fn filler_costs() { - // naming filler cannot delete it: the scorer has no input but the two texts - let reference = "I am not angry."; - let filler = "Darling, I am truly not angry, darling."; - let scorer = Scorer::new(Bag); - close(score_one(&scorer, reference, filler).chrf, 46.8378); - let copy_reference = "He is going to kick the bucket tomorrow."; - let padded = "Well, he is going to kick the bucket tomorrow, honestly."; - let unnamed = score_one(&scorer, copy_reference, padded); - close(unnamed.chrf, chrf(padded, copy_reference)); - assert!(unnamed.chrf < 100.0); - let honest = score_one(&scorer, "Never again.", "Never again."); - assert_eq!(honest.chrf, 100.0); -} - -#[test] -fn honest_name_earns_credit() { - let scorer = Scorer::new(Bag); - let reference = "Brokkr came back."; - assert_eq!(score_one(&scorer, reference, reference).chrf, 100.0); - let wrong = score_one(&scorer, reference, "Kestrel came back."); - assert!(wrong.chrf < 100.0); - assert!(wrong.meaning.unwrap() < 1.0); -} - -#[test] -fn every_script_alike() { - let scorer = Scorer::new(Bag); - for text in [ - "민수가 왔다", - "Brokkr'ın gemisi geldi", - "Brokkrin laiva tuli", - "สมชายมาà¹à¸¥à¹‰à¸§", - "ブロッカーãŒæ¥ãŸã€‚", - ] { - let got = score_one(&scorer, text, text); - assert_eq!(got.chrf, 100.0, "{text}"); - } -} diff --git a/crates/weksa/tests/seeds.rs b/crates/weksa/tests/seeds.rs deleted file mode 100644 index 3cec02d..0000000 --- a/crates/weksa/tests/seeds.rs +++ /dev/null @@ -1,38 +0,0 @@ -//! The committed seed stores replay through today's doors, so a rule that -//! tightens cannot leave a seed the stores would now refuse. -mod common; -use std::path::Path; -use weksa::{CorpusStore, EntityStore}; - -#[test] -fn committed_seeds_replay_through_the_doors() { - let data = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../data"); - let seeded_entities = EntityStore::open(&data.join("entity/entity.cc")).unwrap(); - let seeded_corpus = CorpusStore::open(&data.join("corpus/corpus.cc")).unwrap(); - let dir = tempfile::tempdir().unwrap(); - let mut entities = EntityStore::open(&dir.path().join("e.cc")).unwrap(); - let all = seeded_entities.all().unwrap(); - assert!(!all.is_empty()); - for e in all { - entities.put(e).unwrap(); - } - let mut corpus = CorpusStore::open(&dir.path().join("c.cc")).unwrap(); - let items = seeded_corpus.all().unwrap(); - assert!(!items.is_empty()); - for item in items { - let withdrawn = item.withdrawn; - let id = item.id.clone(); - corpus - .add( - weksa::CorpusItem { - withdrawn: false, - ..item - }, - &entities, - ) - .unwrap(); - if withdrawn { - corpus.withdraw(&id).unwrap(); - } - } -} diff --git a/crates/weksa/tests/split_schema.rs b/crates/weksa/tests/split_schema.rs deleted file mode 100644 index d1e638a..0000000 --- a/crates/weksa/tests/split_schema.rs +++ /dev/null @@ -1,153 +0,0 @@ -use weksa::corpus::{CorpusItem, ModelFamily, Source}; -use weksa::split::fnv1a64; -use weksa::{Split, SplitScheme}; - -fn item(group: &str, source: Source) -> CorpusItem { - CorpusItem { - id: "item:t:1".into(), - text: "t".into(), - language: "en-US".into(), - speaker: weksa::EntityId::parse("entity:s").unwrap(), - names: vec![], - group: group.into(), - prior_turns: vec![], - source, - admitted_on: "2026-10-29".into(), - supersedes: None, - withdrawn: false, - } -} - -fn cast() -> Source { - Source::CastAuthored { - persona: "p".into(), - fixture: "f".into(), - author: "a".into(), - } -} - -fn model() -> Source { - Source::ModelWritten { - family: ModelFamily::Unrecorded, - generator: None, - prompt_digest: None, - } -} - -fn group_in_bucket(want: u64) -> String { - (0..100_000) - .map(|i| format!("g{i}")) - .find(|g| SplitScheme::V1.bucket(g) == want) - .unwrap_or_else(|| panic!("no group lands in bucket {want}")) -} - -#[test] -fn fnv1a64_matches_the_reference_vectors() { - assert_eq!(fnv1a64(b""), 0xcbf2_9ce4_8422_2325); - assert_eq!(fnv1a64(b"a"), 0xaf63_dc4c_8601_ec8c); - assert_eq!(fnv1a64(b"foobar"), 0x8594_4171_f739_67e8); -} - -#[test] -fn p1_split_deterministic() { - let a = SplitScheme::V1.split(&item("scene-9", cast())); - for _ in 0..3 { - assert_eq!(SplitScheme::V1.split(&item("scene-9", cast())), a); - } - // The split reads the group and nothing else: speaker, text and id do not move it. - let mut other = item("scene-9", cast()); - other.speaker = weksa::EntityId::parse("entity:someone").unwrap(); - other.text = "different".into(); - other.id = "item:other:7".into(); - assert_eq!(SplitScheme::V1.split(&other), a); - // The bucket is the pinned hash of the prefixed group. - assert_eq!( - SplitScheme::V1.bucket("x"), - fnv1a64(b"weksa-split-v1:x") % 100 - ); -} - -#[test] -fn split_boundaries() { - let s = SplitScheme::V1; - assert_eq!(s.of_bucket(0), Split::Train); - assert_eq!(s.of_bucket(39), Split::Train); - assert_eq!(s.of_bucket(40), Split::Validation); - assert_eq!(s.of_bucket(79), Split::Validation); - assert_eq!(s.of_bucket(80), Split::Test); - assert_eq!(s.of_bucket(99), Split::Test); - for (bucket, want) in [ - (39, Split::Train), - (40, Split::Validation), - (79, Split::Validation), - (80, Split::Test), - ] { - assert_eq!( - s.split(&item(&group_in_bucket(bucket), cast())), - Some(want), - "bucket {bucket}" - ); - } -} - -#[test] -fn p2_model_written_in_a_test_group_is_excluded() { - let test_group = group_in_bucket(95); - assert_eq!( - SplitScheme::V1.split(&item(&test_group, cast())), - Some(Split::Test) - ); - // A model line of the test meaning is in no split; it is never moved to train. - assert_eq!(SplitScheme::V1.split(&item(&test_group, model())), None); - // Model-written items in the other splits keep them. - assert_eq!( - SplitScheme::V1.split(&item(&group_in_bucket(10), model())), - Some(Split::Train) - ); - assert_eq!( - SplitScheme::V1.split(&item(&group_in_bucket(60), model())), - Some(Split::Validation) - ); -} - -#[test] -fn p3_split_ratio() { - let mut counts = [0usize; 3]; - for i in 0..10_000 { - match SplitScheme::V1.split(&item(&format!("scene-{i}"), cast())) { - Some(Split::Train) => counts[0] += 1, - Some(Split::Validation) => counts[1] += 1, - Some(Split::Test) => counts[2] += 1, - None => panic!("a cast item is always in a split"), - } - } - let near = |got: usize, want_percent: usize| got.abs_diff(want_percent * 100) <= 200; - assert!( - near(counts[0], 40) && near(counts[1], 40) && near(counts[2], 20), - "{counts:?}" - ); -} - -#[test] -fn published_schemas_match() { - let dir = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../schemas/cultnet"); - let published = weksa::schema::published(); - let mut names: Vec = published.iter().map(|(n, _)| n.clone()).collect(); - names.sort(); - let mut on_disk: Vec = std::fs::read_dir(&dir) - .unwrap() - .map(|e| e.unwrap().file_name().into_string().unwrap()) - .filter(|n| n.ends_with(".schema.json")) - .collect(); - on_disk.sort(); - assert_eq!( - on_disk, names, - "schemas/cultnet holds exactly the generated files" - ); - for (name, json) in published { - let held = std::fs::read_to_string(dir.join(&name)) - .unwrap() - .replace("\r\n", "\n"); - assert_eq!(held, json, "{name} drifted; run `weksa schema`"); - } -} diff --git a/crates/weksa/tests/stores.rs b/crates/weksa/tests/stores.rs deleted file mode 100644 index 07cbe0c..0000000 --- a/crates/weksa/tests/stores.rs +++ /dev/null @@ -1,1294 +0,0 @@ -mod common; -use common::*; -use tempfile::TempDir; -use weksa::corpus::{CorpusItem, ModelFamily, Source, Turn}; -use weksa::inventory::{Change, seed}; -use weksa::packet::PacketId; -use weksa::{ - AddOutcome, CorpusStore, EntityStore, Family, InventoryEntry, InventoryRevision, - InventoryStore, PacketStore, Refusal, Slot, SplitScheme, StoreError, StoreRefusal, ValueKind, -}; - -fn item(key: &str, text: &str) -> CorpusItem { - CorpusItem { - prior_turns: vec![Turn { - speaker: eid("player"), - text: "hello".into(), - }], - ..item_in(key, text, "scene-1", cast()) - } -} - -fn refusal(r: Result) -> StoreRefusal { - match r { - Err(StoreError::Refused(r)) => r, - other => panic!("expected a store refusal, got {other:?}"), - } -} - -fn ents(d: &TempDir) -> EntityStore { - entities_in(d.path()) -} - -fn dir() -> TempDir { - tempfile::tempdir().unwrap() -} - -fn empty_corpus(d: &TempDir) -> CorpusStore { - CorpusStore::open(&d.path().join("empty-corpus.cc")).unwrap() -} - -fn malformed(key: &str, why: &'static str) -> StoreRefusal { - StoreRefusal::Malformed { - key: key.into(), - why, - } -} - -#[test] -fn s1_item_immutable() { - let d = dir(); - let path = d.path().join("c.cc"); - let mut store = CorpusStore::open(&path).unwrap(); - assert_eq!( - store - .add(item("item:nibu:1", "Mind the hatch."), &ents(&d)) - .unwrap(), - AddOutcome::Added - ); - // An equal item is idempotent, even from a reopened store. - let mut store = CorpusStore::open(&path).unwrap(); - assert_eq!( - store - .add(item("item:nibu:1", "Mind the hatch."), &ents(&d)) - .unwrap(), - AddOutcome::Unchanged - ); - let refused = || StoreRefusal::S1ItemImmutable { - key: "item:nibu:1".into(), - }; - // Changed text, and every other field, is refused; the stored item stays. - let base = item("item:nibu:1", "Mind the hatch."); - let mut variants = vec![item("item:nibu:1", "Mind the hatch!")]; - let mut v = base.clone(); - v.source = model(); - variants.push(v); - let mut v = base.clone(); - v.source = Source::OperatorWritten; - variants.push(v); - let mut v = base.clone(); - v.language = "en-GB".into(); - variants.push(v); - let mut v = base.clone(); - v.speaker = eid("brokkr"); - variants.push(v); - let mut v = base.clone(); - v.group = "scene-2".into(); - variants.push(v); - let mut v = base.clone(); - v.prior_turns.clear(); - variants.push(v); - let mut v = base.clone(); - v.admitted_on = "2026-10-30".into(); - variants.push(v); - let mut v = base.clone(); - v.supersedes = Some("item:nibu:9".into()); - variants.push(v); - for v in variants { - assert_eq!(refusal(store.add(v.clone(), &ents(&d))), refused(), "{v:?}"); - } - assert_eq!(store.get("item:nibu:1").unwrap(), Some(base.clone())); - // A withdrawn item is a different item: re-adding it live is refused too. - store.withdraw("item:nibu:1").unwrap(); - assert_eq!(refusal(store.add(base, &ents(&d))), refused()); -} - -#[test] -fn corpus_keys_are_canonical() { - let d = dir(); - let mut store = CorpusStore::open(&d.path().join("c.cc")).unwrap(); - let long_slug = format!("item:a{}:1", "b".repeat(48)); - let not_canonical = [ - "item:a:01", - "item:a:+1", - "item:a:0", - "item:A:1", - "item:a-:1", - "item:-a:1", - "item:a:b:1", - "item::1", - "item:a:", - "item:a:\u{661}", - "item:a:4294967296", - &long_slug, - ]; - for (i, key) in not_canonical.into_iter().enumerate() { - assert_eq!( - refusal(store.add(item(key, &format!("line {i}")), &ents(&d))), - malformed(key, "item key is not item::"), - "{key}" - ); - } - assert!(store.all().unwrap().is_empty()); - // The edges of the canonical form are admitted. - let longest = format!("item:{}:4294967295", "a".repeat(48)); - let canonical = ["item:a:1", "item:a-b:10", "item:7:2", "item:ab:9", &longest]; - for (i, key) in canonical.into_iter().enumerate() { - assert_eq!( - store - .add(item(key, &format!("fine {i}")), &ents(&d)) - .unwrap(), - AddOutcome::Added, - "{key}" - ); - } - // An alias of a stored key is not a corpus item id at all. - assert!(weksa::CorpusItemId::parse("item:nibu:01").is_err()); - // The last free number is handed out; past u32::MAX the slug has no free - // key, and asking neither wraps nor panics. - store - .add(item("item:b:4294967294", "one before the last"), &ents(&d)) - .unwrap(); - assert_eq!(store.next_key("b").unwrap(), "item:b:4294967295"); - let full = "a".repeat(48); - assert_eq!( - refusal(store.next_key(&full)), - malformed(&full, "slug has no free key") - ); -} - -#[test] -fn s6_text_held_by_another_key() { - let d = dir(); - let mut store = CorpusStore::open(&d.path().join("c.cc")).unwrap(); - store - .add(item("item:nibu:1", "Mind the hatch."), &ents(&d)) - .unwrap(); - let elsewhere = |key: &str, text: &str| CorpusItem { - group: "scene-2".into(), - ..item(key, text) - }; - let held = |key: &str, by: &str| StoreRefusal::S6TextHeld { - key: key.into(), - held_by: by.into(), - }; - // The same text in another group, and in the same group, is refused. - assert_eq!( - refusal(store.add(elsewhere("item:nibu:2", "Mind the hatch."), &ents(&d))), - held("item:nibu:2", "item:nibu:1") - ); - assert_eq!( - refusal(store.add(item("item:nibu:2", "Mind the hatch."), &ents(&d))), - held("item:nibu:2", "item:nibu:1") - ); - // Case, spacing and punctuation variants are the same text. - for text in [ - "MIND THE HATCH", - "mind the\thatch...", - " Mind-the-hatch! ", - "Mind the hatch.\n", - ] { - assert_eq!( - refusal(store.add(elsewhere("item:nibu:2", text), &ents(&d))), - held("item:nibu:2", "item:nibu:1"), - "{text:?}" - ); - } - store - .add( - item("item:nibu:3", "\u{c4}rger \u{fc}ber \u{d6}l"), - &ents(&d), - ) - .unwrap(); - assert_eq!( - refusal(store.add( - elsewhere("item:nibu:4", "\u{e4}rger \u{dc}BER \u{f6}l"), - &ents(&d) - )), - held("item:nibu:4", "item:nibu:3") - ); - // Text that differs in a letter or digit is a different text. - store - .add(item("item:nibu:5", "Mind the hatch 2"), &ents(&d)) - .unwrap(); - store - .add(item("item:nibu:6", "Mind the hatches."), &ents(&d)) - .unwrap(); - // A withdrawn holder still holds. - store.withdraw("item:nibu:1").unwrap(); - assert_eq!( - refusal(store.add(elsewhere("item:nibu:2", "mind the hatch"), &ents(&d))), - held("item:nibu:2", "item:nibu:1") - ); - // Superseding a different key does not excuse a copy of another item. - let mut copy = item("item:nibu:8", "Mind the hatches!"); - copy.supersedes = Some("item:nibu:5".into()); - assert_eq!( - refusal(store.add(copy, &ents(&d))), - held("item:nibu:8", "item:nibu:6") - ); - // A superseder of the holder, in the holder's group, is Added. - let mut fix = item("item:nibu:2", "mind the hatch"); - fix.supersedes = Some("item:nibu:1".into()); - assert_eq!(store.add(fix, &ents(&d)).unwrap(), AddOutcome::Added); - // The superseded holder's text stays held against a third key. - assert_eq!( - refusal(store.add(elsewhere("item:nibu:7", "Mind the hatch."), &ents(&d))), - held("item:nibu:7", "item:nibu:1") - ); -} - -#[test] -fn corpus_admission_rules() { - let d = dir(); - let mut store = CorpusStore::open(&d.path().join("c.cc")).unwrap(); - assert_eq!( - refusal(store.add(item("nibu:1", "x"), &ents(&d))), - malformed("nibu:1", "item key is not item::") - ); - assert_eq!( - refusal(store.add(item("item:nibu:x", "x"), &ents(&d))), - malformed("item:nibu:x", "item key is not item::") - ); - let mut born_withdrawn = item("item:nibu:1", "x"); - born_withdrawn.withdrawn = true; - assert_eq!( - refusal(store.add(born_withdrawn, &ents(&d))), - malformed("item:nibu:1", "an item is admitted live, then withdrawn") - ); - let mut orphan = item("item:nibu:1", "x"); - orphan.supersedes = Some("item:nibu:9".into()); - assert_eq!( - refusal(store.add(orphan, &ents(&d))), - malformed("item:nibu:1", "supersedes an item that does not exist") - ); - // Nothing was written by the refusals. - assert!(store.all().unwrap().is_empty()); - // A correction is a new item that supersedes the old one. - store.add(item("item:nibu:1", "old"), &ents(&d)).unwrap(); - let mut fix = item("item:nibu:2", "new"); - fix.supersedes = Some("item:nibu:1".into()); - assert_eq!(store.add(fix, &ents(&d)).unwrap(), AddOutcome::Added); - assert_eq!( - refusal(store.withdraw("item:nibu:7")), - malformed("item:nibu:7", "no such item") - ); -} - -#[test] -fn s6_reads_text_under_compatibility_case_folding() { - let d = dir(); - let mut store = CorpusStore::open(&d.path().join("c.cc")).unwrap(); - // Each pair is one text to the key: fullwidth, ligature, superscript and - // circled-digit compatibility forms and full case folds (straße, STRASSE). - let pairs = [ - ("Brokkr", "\u{ff22}\u{ff52}\u{ff4f}\u{ff4b}\u{ff4b}\u{ff52}"), - ("fine", "\u{fb01}ne"), - ("room 2", "room \u{b2}"), - ("gate 1", "gate \u{2460}"), - ("stra\u{df}e", "STRASSE"), - ( - "\u{3a3}\u{3af}\u{3c3}\u{3c5}\u{3c6}\u{3bf}\u{3c2}", - "\u{3c3}\u{3af}\u{3c3}\u{3c5}\u{3c6}\u{3bf}\u{3c3}", - ), - // The first NFKC step: a compatibility letter folds only after it is - // mapped (math-bold S has no case fold of its own; black-letter H is - // not a case variant of h until NFKC reads it as H). - ( - "sigrun", - "\u{1d412}\u{1d422}\u{1d420}\u{1d42b}\u{1d42e}\u{1d427}", - ), - ("h", "\u{210c}"), - // The second NFKC step: a full case fold can leave a decomposed or - // reordered sequence that only a second NFKC composes to the same key - // (dialytika tonos, caron under line, macron below). - ("\u{390}", "\u{3aa}\u{301}"), - ("\u{3b0}", "\u{3ab}\u{301}"), - ("\u{1e96}\u{30c}", "\u{21e}\u{331}"), - ]; - for (n, (a, b)) in pairs.into_iter().enumerate() { - let (first, second) = ( - format!("item:s6:{}", 2 * n + 1), - format!("item:s6:{}", 2 * n + 2), - ); - store.add(item(&first, a), &ents(&d)).unwrap(); - assert_eq!( - refusal(store.add(item(&second, b), &ents(&d))), - StoreRefusal::S6TextHeld { - key: second.clone(), - held_by: first.clone() - }, - "{a:?} vs {b:?}" - ); - assert!(store.get(&second).unwrap().is_none()); - } - // A decomposed text never reaches the key: the door refuses it as not NFC. - assert_eq!( - refusal(store.add(item("item:s6:90", "Bro\u{308}kkr"), &ents(&d))), - StoreRefusal::NotNfc { - key: "item:s6:90".into() - } - ); - // Cyrillic a is not Latin a: homoglyphs across scripts stay apart. - store.add(item("item:s6:91", "cat"), &ents(&d)).unwrap(); - store - .add(item("item:s6:92", "c\u{430}t"), &ents(&d)) - .unwrap(); -} - -#[test] -fn s6_keeps_tone_marks_and_viramas_apart() { - let d = dir(); - let mut store = CorpusStore::open(&d.path().join("c.cc")).unwrap(); - // Minimal pairs that differ only by a combining mark: Thai mai tho and mai - // ek on one word, Thai tone on a short vowel, a Devanagari virama. - let pairs = [ - ("\u{e19}\u{e49}\u{e33}", "\u{e19}\u{e33}"), - ("\u{e44}\u{e21}\u{e49}", "\u{e44}\u{e21}\u{e48}"), - ("\u{915}\u{94d}\u{92f}", "\u{915}\u{92f}"), - ]; - for (n, (a, b)) in pairs.into_iter().enumerate() { - store - .add(item(&format!("item:mk:{}", 2 * n + 1), a), &ents(&d)) - .unwrap_or_else(|e| panic!("{a:?}: {e}")); - store - .add(item(&format!("item:mk:{}", 2 * n + 2), b), &ents(&d)) - .unwrap_or_else(|e| panic!("{b:?} held by {a:?}: {e}")); - } - // A mark alone is still no text. - for (n, text) in ["\u{e49}", "\u{94d}"].into_iter().enumerate() { - let key = format!("item:mk:{}", 90 + n); - assert_eq!( - refusal(store.add(item(&key, text), &ents(&d))), - malformed(&key, "text has no alphanumeric character") - ); - } -} - -#[test] -fn an_attestation_holds_something_readable() { - let d = dir(); - let mut store = CorpusStore::open(&d.path().join("c.cc")).unwrap(); - let s = |x: &str| x.to_string(); - type Build = fn(String) -> Source; - let builders: [(Build, &'static str); 9] = [ - ( - |v| Source::CastAuthored { - persona: v, - fixture: "f".into(), - author: "a".into(), - }, - "a cast-authored source names its persona", - ), - ( - |v| Source::CastAuthored { - persona: "p".into(), - fixture: v, - author: "a".into(), - }, - "a cast-authored source names its fixture", - ), - ( - |v| Source::CastAuthored { - persona: "p".into(), - fixture: "f".into(), - author: v, - }, - "a cast-authored source names its author", - ), - ( - |v| Source::OperatorApproved { note: v }, - "an operator-approved source carries its note", - ), - ( - |v| Source::PublicDomain { - work: v, - author: "a".into(), - edition_url: "u".into(), - }, - "a public-domain source names its work", - ), - ( - |v| Source::PublicDomain { - work: "w".into(), - author: v, - edition_url: "u".into(), - }, - "a public-domain source names its author", - ), - ( - |v| Source::PublicDomain { - work: "w".into(), - author: "a".into(), - edition_url: v, - }, - "a public-domain source names its edition url", - ), - ( - |v| Source::ModelWritten { - family: ModelFamily::Unrecorded, - generator: Some(v), - prompt_digest: None, - }, - "a model-written source names a non-empty generator", - ), - ( - |v| Source::ModelWritten { - family: ModelFamily::Unrecorded, - generator: None, - prompt_digest: Some(v), - }, - "a model-written source carries a non-empty prompt digest", - ), - ]; - let unreadable = [ - s(" "), - s("\t"), - s("\n"), - s("\u{200b}"), - s("\u{200d}"), - s("\0"), - s("--"), - s("\u{e49}"), - s("ok\u{200b}ok"), - s("ok\u{200d}ok"), - s("ok\u{2028}ok"), - s("ok\nok"), - s("ok\0ok"), - "x".repeat(201), - "x".repeat(2 * 1024 * 1024), - ]; - let mut n = 0; - for (build, why) in &builders { - for bad in &unreadable { - n += 1; - let key = format!("item:att:{n}"); - let refused = item_in(&key, &format!("line {n}"), "scene-1", build(bad.clone())); - assert_eq!( - refusal(store.add(refused, &ents(&d))), - malformed(&key, why), - "{why}: {:?}", - bad.chars().take(12).collect::() - ); - } - } - assert!(store.all().unwrap().is_empty()); - // 200 bytes and plain words with spaces attest; 201 does not. - for (m, (build, _)) in builders.iter().enumerate() { - for (k, good) in ["x".repeat(200), s("two words, 2 more")] - .into_iter() - .enumerate() - { - let key = format!("item:att-ok:{}", 2 * m + k + 1); - store - .add( - item_in(&key, &format!("fine {m} {k}"), "scene-1", build(good)), - &ents(&d), - ) - .unwrap_or_else(|e| panic!("{key}: {e}")); - } - } -} - -#[test] -fn a_text_with_no_alphanumeric_character_is_refused() { - let d = dir(); - let mut store = CorpusStore::open(&d.path().join("c.cc")).unwrap(); - for (n, text) in ["...", "?!", " ", "\u{2014}\u{2014}", "\u{ff01}"] - .into_iter() - .enumerate() - { - let key = format!("item:p:{}", n + 1); - assert_eq!( - refusal(store.add(item(&key, text), &ents(&d))), - malformed(&key, "text has no alphanumeric character"), - "{text:?}" - ); - } - assert!(store.all().unwrap().is_empty()); - store.add(item("item:p:1", "a..."), &ents(&d)).unwrap(); -} - -#[test] -fn an_item_source_attests_with_non_empty_strings() { - let d = dir(); - let mut store = CorpusStore::open(&d.path().join("c.cc")).unwrap(); - let s = |x: &str| x.to_string(); - let cast = |persona: &str, fixture: &str, author: &str| Source::CastAuthored { - persona: s(persona), - fixture: s(fixture), - author: s(author), - }; - let model = |generator: Option<&str>, digest: Option<&str>| Source::ModelWritten { - family: ModelFamily::Unrecorded, - generator: generator.map(s), - prompt_digest: digest.map(s), - }; - let public = |work: &str, author: &str, url: &str| Source::PublicDomain { - work: s(work), - author: s(author), - edition_url: s(url), - }; - let cases: [(Source, &'static str); 9] = [ - ( - cast("", "f", "a"), - "a cast-authored source names its persona", - ), - ( - cast("p", "", "a"), - "a cast-authored source names its fixture", - ), - ( - cast("p", "f", ""), - "a cast-authored source names its author", - ), - ( - Source::OperatorApproved { note: s("") }, - "an operator-approved source carries its note", - ), - ( - public("", "a", "u"), - "a public-domain source names its work", - ), - ( - public("w", "", "u"), - "a public-domain source names its author", - ), - ( - public("w", "a", ""), - "a public-domain source names its edition url", - ), - ( - model(Some(""), None), - "a model-written source names a non-empty generator", - ), - ( - model(None, Some("")), - "a model-written source carries a non-empty prompt digest", - ), - ]; - for (n, (source, why)) in cases.into_iter().enumerate() { - let key = format!("item:prov:{}", n + 1); - let refused = item_in(&key, &format!("line {n}"), "scene-1", source); - assert_eq!(refusal(store.add(refused, &ents(&d))), malformed(&key, why)); - } - assert!(store.all().unwrap().is_empty()); - // The same sources with every string present are admitted. - for (n, source) in [ - cast("p", "f", "a"), - Source::OperatorApproved { note: s("n") }, - Source::OperatorWritten, - public("w", "a", "u"), - model(Some("g"), Some("d")), - model(None, None), - ] - .into_iter() - .enumerate() - { - let key = format!("item:prov:{}", n + 1); - let item = item_in(&key, &format!("line {n}"), "scene-1", source); - assert_eq!(store.add(item, &ents(&d)).unwrap(), AddOutcome::Added); - } -} - -#[test] -fn s5_supersede_keeps_the_group() { - let d = dir(); - let mut store = CorpusStore::open(&d.path().join("c.cc")).unwrap(); - store.add(item("item:nibu:1", "old"), &ents(&d)).unwrap(); - let mut elsewhere = item("item:nibu:2", "new"); - elsewhere.supersedes = Some("item:nibu:1".into()); - elsewhere.group = "scene-2".into(); - assert_eq!( - refusal(store.add(elsewhere, &ents(&d))), - StoreRefusal::S5SupersedeGroup { - key: "item:nibu:2".into(), - expected: "scene-1".into(), - got: "scene-2".into() - } - ); - assert!(store.get("item:nibu:2").unwrap().is_none()); - let mut same = item("item:nibu:2", "new"); - same.supersedes = Some("item:nibu:1".into()); - assert_eq!(store.add(same, &ents(&d)).unwrap(), AddOutcome::Added); -} - -#[test] -fn withdrawing_keeps_the_text_and_the_number() { - let d = dir(); - let path = d.path().join("c.cc"); - let mut store = CorpusStore::open(&path).unwrap(); - store - .add(item("item:nibu:1", "keep me"), &ents(&d)) - .unwrap(); - assert_eq!(store.next_key("nibu").unwrap(), "item:nibu:2"); - assert_eq!(store.next_key("other").unwrap(), "item:other:1"); - store.withdraw("item:nibu:1").unwrap(); - store.withdraw("item:nibu:1").unwrap(); - let store = CorpusStore::open(&path).unwrap(); - let got = store.get("item:nibu:1").unwrap().unwrap(); - assert!(got.withdrawn); - assert_eq!(got.text, "keep me"); - assert_eq!(store.next_key("nibu").unwrap(), "item:nibu:2"); -} - -fn entries_for(rev: u32, ids: &[&str]) -> (InventoryRevision, Vec) { - let entries = ids.iter().map(|id| entry(id, rev)).collect(); - let changes = ids.iter().map(|id| Change::Add(iv(id))).collect(); - ( - rev_with(changes, rev, if rev == 1 { None } else { Some(rev - 1) }), - entries, - ) -} - -fn seeded(path: &std::path::Path) -> InventoryStore { - let mut store = InventoryStore::open(path).unwrap(); - let (rev, entries) = seed(); - store.admit_revision(rev, entries).unwrap(); - store -} - -#[test] -fn s2_revision_chain() { - let d = dir(); - let mut store = InventoryStore::open(&d.path().join("i.cc")).unwrap(); - // The first revision is 1 with no parent. - let (rev, entries) = entries_for(2, &["concept:1"]); - assert_eq!( - refusal(store.admit_revision(rev, entries)), - StoreRefusal::S2RevisionChain { - expected: 1, - got: 2 - } - ); - let (mut rev, entries) = entries_for(1, &["concept:1"]); - rev.parent = Some(0); - assert_eq!( - refusal(store.admit_revision(rev, entries)), - StoreRefusal::S2RevisionChain { - expected: 1, - got: 1 - } - ); - let (rev, entries) = entries_for(1, &["concept:1"]); - store.admit_revision(rev, entries).unwrap(); - // Replaying n, skipping n, or naming the wrong parent is refused. - let (rev, entries) = entries_for(1, &["concept:2"]); - assert_eq!( - refusal(store.admit_revision(rev, entries)), - StoreRefusal::S2RevisionChain { - expected: 2, - got: 1 - } - ); - let (rev, entries) = entries_for(3, &["concept:2"]); - assert_eq!( - refusal(store.admit_revision(rev, entries)), - StoreRefusal::S2RevisionChain { - expected: 2, - got: 3 - } - ); - let (mut rev, entries) = entries_for(2, &["concept:2"]); - rev.parent = None; - assert_eq!( - refusal(store.admit_revision(rev, entries)), - StoreRefusal::S2RevisionChain { - expected: 2, - got: 2 - } - ); - let (rev, entries) = entries_for(2, &["concept:2"]); - store.admit_revision(rev, entries).unwrap(); - assert_eq!(store.inventory().unwrap().latest().unwrap().n, 2); -} - -#[test] -fn s3_frozen_is_final() { - let d = dir(); - let mut store = InventoryStore::open(&d.path().join("i.cc")).unwrap(); - let (mut rev, entries) = entries_for(1, &["concept:1"]); - rev.frozen = true; - store.admit_revision(rev, entries).unwrap(); - let (rev, entries) = entries_for(2, &["concept:2"]); - assert_eq!( - refusal(store.admit_revision(rev, entries)), - StoreRefusal::S3Frozen { frozen: 1 } - ); - assert_eq!(store.inventory().unwrap().latest().unwrap().n, 1); -} - -#[test] -fn s4_entry_introduced_by_its_revision() { - let d = dir(); - let mut store = InventoryStore::open(&d.path().join("i.cc")).unwrap(); - let (rev, mut entries) = entries_for(1, &["concept:1"]); - entries[0].introduced_rev = 2; - assert_eq!( - refusal(store.admit_revision(rev, entries)), - StoreRefusal::S4IntroducedRev { - id: iv("concept:1"), - introduced: 2, - revision: 1 - } - ); - assert!(store.inventory().unwrap().latest().is_none()); -} - -#[test] -fn revision_change_rules() { - let d = dir(); - let mut store = seeded(&d.path().join("i.cc")); - // An added id with no entry. - let rev = rev_with(vec![Change::Add(iv("concept:1"))], 2, Some(1)); - assert_eq!( - refusal(store.admit_revision(rev, vec![])), - malformed("concept:1", "added id has no entry") - ); - // An entry the changes do not name. - let (rev, mut entries) = entries_for(2, &["concept:1"]); - entries.push(entry("concept:2", 2)); - assert_eq!( - refusal(store.admit_revision(rev, entries)), - malformed("concept:2", "entry is not in the revision's changes") - ); - // An id already in the inventory, and one added twice. - let (rev, entries) = entries_for(2, &["role:1"]); - assert_eq!( - refusal(store.admit_revision(rev, entries)), - malformed("role:1", "id is already in the inventory") - ); - let (mut rev, entries) = entries_for(2, &["concept:1"]); - rev.changes.push(Change::Add(iv("concept:1"))); - assert_eq!( - refusal(store.admit_revision(rev, entries)), - malformed("concept:1", "id is already in the inventory") - ); - // A malformed entry: a feature with no value kind, a non-feature with one, - // an extension with no value kind or with slots, an entry born retired. - let shape = "entry shape does not match its family"; - let (rev, mut entries) = entries_for(2, &["feature:50"]); - entries[0].applies_to = vec![Slot::Discourse]; - assert_eq!( - refusal(store.admit_revision(rev, entries)), - malformed("feature:50", shape) - ); - let (rev, mut entries) = entries_for(2, &["feature:50"]); - entries[0].value_kind = Some(ValueKind::Flag); - assert_eq!( - refusal(store.admit_revision(rev, entries)), - malformed("feature:50", shape) - ); - let (rev, mut entries) = entries_for(2, &["concept:1"]); - entries[0].value_kind = Some(ValueKind::Flag); - assert_eq!( - refusal(store.admit_revision(rev, entries)), - malformed("concept:1", shape) - ); - let (rev, entries) = entries_for(2, &["extension:1"]); - assert_eq!( - refusal(store.admit_revision(rev, entries)), - malformed("extension:1", shape) - ); - let (rev, mut entries) = entries_for(2, &["extension:1"]); - entries[0].value_kind = Some(ValueKind::Flag); - entries[0].applies_to = vec![Slot::Discourse]; - assert_eq!( - refusal(store.admit_revision(rev, entries)), - malformed("extension:1", shape) - ); - let (rev, mut entries) = entries_for(2, &["concept:1"]); - entries[0].retired_rev = Some(2); - assert_eq!( - refusal(store.admit_revision(rev, entries)), - malformed("concept:1", shape) - ); - // Retire targets must exist and be live; an id changes once per revision. - let rev = rev_with(vec![Change::Retire(iv("concept:9"))], 2, Some(1)); - assert_eq!( - refusal(store.admit_revision(rev, vec![])), - malformed("concept:9", "retire target is missing or already retired") - ); - let rev = rev_with( - vec![Change::Retire(iv("role:1")), Change::Retire(iv("role:1"))], - 2, - Some(1), - ); - assert_eq!( - refusal(store.admit_revision(rev, vec![])), - malformed("role:1", "id changes twice in one revision") - ); - // Nothing above was written. - let inv = store.inventory().unwrap(); - assert_eq!(inv.latest().unwrap().n, 1); - assert!(inv.entry(&iv("concept:1")).is_none()); - assert!(inv.entry(&iv("role:1")).unwrap().retired_rev.is_none()); -} - -#[test] -fn a_well_formed_extension_entry_is_admitted() { - let d = dir(); - let mut store = seeded(&d.path().join("i.cc")); - let (rev, mut entries) = entries_for(2, &["extension:1"]); - entries[0].value_kind = Some(ValueKind::Grade); - store.admit_revision(rev, entries).unwrap(); - assert_eq!( - store - .inventory() - .unwrap() - .entry(&iv("extension:1")) - .unwrap() - .family(), - Family::Extension - ); -} - -#[test] -fn retiring_keeps_the_entry_for_earlier_revisions() { - let d = dir(); - let path = d.path().join("i.cc"); - let mut store = seeded(&path); - let (rev, entries) = entries_for(2, &["concept:1"]); - store.admit_revision(rev, entries).unwrap(); - let retire = rev_with(vec![Change::Retire(iv("concept:1"))], 3, Some(2)); - store.admit_revision(retire, vec![]).unwrap(); - let inv = InventoryStore::open(&path).unwrap().inventory().unwrap(); - let id = iv("concept:1"); - assert_eq!(inv.entry(&id).unwrap().retired_rev, Some(3)); - assert!(inv.resolve(&id, 2).is_some()); - assert!(inv.resolve(&id, 3).is_none()); - let again = rev_with(vec![Change::Retire(id)], 4, Some(3)); - assert!(matches!( - refusal(store.admit_revision(again, vec![])), - StoreRefusal::Malformed { .. } - )); -} - -#[test] -fn revision_one_survives_revision_two() { - let inv = inventory(); - let revisions: Vec<(u32, usize)> = inv.revisions().map(|r| (r.n, r.changes.len())).collect(); - assert_eq!(revisions, vec![(1, 61), (2, 6), (3, 1), (4, 1)]); - // A packet of revision 1 is judged by the inventory of revision 1: the - // revision is known (no V10), and it names nothing that came later. - let mut p = good(); - p.inventory_rev = 1; - let refusals = weksa::validate(&p, &inv).unwrap_err(); - assert!( - refusals - .iter() - .all(|r| matches!(r, Refusal::V5Unresolved { .. })), - "{refusals:?}" - ); -} - -#[test] -fn the_seed_is_the_standards_enumerations_and_the_nuance_features() { - let d = dir(); - let store = seeded(&d.path().join("i.cc")); - let inv = store.inventory().unwrap(); - let count = |f: Family| inv.entries().filter(|e| e.family() == f).count(); - assert_eq!(count(Family::ReferentKind), 11); - assert_eq!(count(Family::Relation), 16); - assert_eq!(count(Family::Role), 8); - assert_eq!(count(Family::SpeechAct), 8); - assert_eq!(count(Family::Feature), 18); - assert_eq!( - count(Family::Concept) + count(Family::Frame) + count(Family::Extension), - 0 - ); - let gloss = |id: &str| inv.entry(&iv(id)).unwrap().gloss.clone(); - assert_eq!(gloss(PERSON), "person"); - assert_eq!(gloss(AGENT), "agent"); - assert_eq!(gloss(WARN), "warn"); - assert_eq!(gloss("relation:8"), "causes"); - assert_eq!(gloss(POLARITY), "polarity"); - assert_eq!(gloss(MODALITY), "modality"); - assert_eq!(gloss(HEDGE), "hedge"); - assert_eq!(gloss(IRONY), "irony"); - assert_eq!( - inv.entry(&iv(IRONY)).unwrap().value_kind, - Some(ValueKind::Flag) - ); - // Felt affect and restraint: Grade features in slot Stance alone. - for (id, name) in [ - (ANGER, "anger"), - (AFFECTION, "affection"), - (RESTRAINT, "restraint"), - ] { - let e = inv.entry(&iv(id)).unwrap(); - assert_eq!(e.gloss, name); - assert_eq!(e.value_kind, Some(ValueKind::Grade)); - assert_eq!(e.applies_to, vec![Slot::Stance]); - } - let stance_names: Vec<&str> = inv - .entries() - .filter(|e| e.applies_to == [Slot::Stance]) - .map(|e| e.gloss.as_str()) - .collect(); - assert_eq!( - stance_names, - [ - "anger", - "fear", - "sadness", - "joy", - "disgust", - "surprise", - "contempt", - "affection", - "shame", - "restraint" - ] - ); - // Hedge, certainty and irony stay on content; displayed formality, warmth - // and intensity are each language's ontology's, not the seed's. - for e in inv.entries() { - assert!( - !["warmth", "formality", "intensity"].contains(&e.gloss.as_str()), - "{}", - e.gloss - ); - } - assert!( - inv.entries() - .all(|e| e.well_formed() && e.introduced_rev == 1 && e.retired_rev.is_none()) - ); - assert!(inv.has_revision(1)); -} - -fn admit_one(d: &TempDir, entry: InventoryEntry) -> Result<(), StoreError> { - let mut store = seeded(&d.path().join("i.cc")); - let rev = rev_with(vec![Change::Add(entry.id)], 2, Some(1)); - store.admit_revision(rev, vec![entry]) -} - -#[test] -fn gloss_is_bounded_text() { - let why = "gloss is 1 to 120 bytes with no control character"; - let with = |gloss: &str| { - let d = dir(); - let mut e = entry("concept:1", 2); - e.gloss = gloss.into(); - admit_one(&d, e) - }; - with(&"g".repeat(120)).unwrap(); - with("two words").unwrap(); - for bad in [ - String::new(), - "g".repeat(121), - "a\nb".into(), - "a\u{7}b".into(), - ] { - assert_eq!(refusal(with(&bad)), malformed("concept:1", why), "{bad:?}"); - } -} - -#[test] -fn packet_admission_goes_through_the_validator() { - let mut w = world(); - let mut store = PacketStore::open(&w.dir.path().join("p.cc")).unwrap(); - let mut bad = good(); - bad.schema_version = 5; - match store.admit(bad, &w.inventory, &w.corpus, &w.entities) { - Err(e @ StoreError::Invalid(_)) => { - assert!(e.to_string().contains("V1"), "{e}"); - let StoreError::Invalid(r) = e else { - unreachable!() - }; - assert_eq!(r, vec![Refusal::V1SchemaVersion { found: 5 }]); - } - other => panic!("{other:?}"), - } - assert!(store.all().unwrap().is_empty()); - // The door yields the lowering view, stance included. - let view = store - .admit(good(), &w.inventory, &w.corpus, &w.entities) - .unwrap(); - assert_eq!(view.stance(), &good().stance); - assert!(!view.stance().features.is_empty()); - // The same packet is idempotent and yields the same view; a different one - // under the same key is refused. - let replay = store - .admit(good(), &w.inventory, &w.corpus, &w.entities) - .unwrap(); - assert_eq!(replay, view); - let mut other = good(); - other.provenance.confidence = grade(1); - assert_eq!( - refusal(store.admit(other, &w.inventory, &w.corpus, &w.entities)), - StoreRefusal::PacketImmutable { - key: PACKET_ID.into() - } - ); - assert_eq!(store.all().unwrap().len(), 1); - assert_eq!(store.get(PACKET_ID).unwrap(), Some(good())); - // A withdrawn item admits no packet, a replay included; the packet admitted - // before the withdrawal stays stored and readable. - w.corpus.withdraw("item:nibu:1").unwrap(); - let mut fresh = good(); - fresh.id = PacketId::parse("packet:00000000000000aa").unwrap(); - let withdrawn = StoreRefusal::WithdrawnCorpusItem { - key: "item:nibu:1".into(), - }; - assert_eq!( - refusal(store.admit(fresh, &w.inventory, &w.corpus, &w.entities)), - withdrawn - ); - assert_eq!( - refusal(store.admit(good(), &w.inventory, &w.corpus, &w.entities)), - withdrawn - ); - assert_eq!(store.all().unwrap().len(), 1); - assert_eq!(store.get(PACKET_ID).unwrap(), Some(good())); -} - -#[test] -fn admission_reads_the_store_inventory() { - // The store holds only revision 1: a revision-2 packet has no inventory to - // stand on, whatever the caller believes about revision 2. - let d = dir(); - let mut corpus = empty_corpus(&d); - corpus - .add( - item_in("item:nibu:1", "Nibu says mind the hatch.", "g", cast()), - &ents(&d), - ) - .unwrap(); - let mut inventory = seeded(&d.path().join("i.cc")); - let mut packets = PacketStore::open(&d.path().join("p.cc")).unwrap(); - assert_eq!( - match packets.admit(good(), &inventory, &corpus, &ents(&d)) { - Err(StoreError::Invalid(r)) => r, - other => panic!("{other:?}"), - }, - vec![Refusal::V10UnknownRevision { rev: 2 }] - ); - assert!(packets.all().unwrap().is_empty()); - // The same packet admits once revision 2 is admitted. - let mut ext = entry(EXT, 2); - ext.value_kind = Some(ValueKind::Grade); - admit( - &mut inventory, - 2, - vec![ - entry(CONCEPT_A, 2), - entry(CONCEPT_B, 2), - entry(FRAME, 2), - ext, - ], - ); - packets - .admit(good(), &inventory, &corpus, &ents(&d)) - .unwrap(); -} - -#[test] -fn corpus_item_must_resolve() { - let w = world(); - let mut packets = PacketStore::open(&w.dir.path().join("p.cc")).unwrap(); - let mut p = good(); - p.provenance.corpus_item = weksa::CorpusItemId::parse("item:nibu:9").unwrap(); - assert_eq!( - refusal(packets.admit(p, &w.inventory, &w.corpus, &w.entities)), - StoreRefusal::UnknownCorpusItem { - key: "item:nibu:9".into() - } - ); - assert!(packets.all().unwrap().is_empty()); - // Free text is not an item id, in Rust or on the wire; a packet with no item - // cannot be built or read back. - for bad in ["banana", "", "the reference sentence goes right here"] { - assert!(weksa::CorpusItemId::parse(bad).is_err(), "{bad:?}"); - } - let mut wire = serde_json::to_value(good().provenance).unwrap(); - for bad in [serde_json::Value::Null, "banana".into()] { - wire["corpus_item"] = bad; - assert!(serde_json::from_value::(wire.clone()).is_err()); - } - wire.as_object_mut().unwrap().remove("corpus_item"); - assert!(serde_json::from_value::(wire).is_err()); -} - -#[test] -fn store_roundtrip() { - let w = world(); - let d = dir(); - // A packet, an entry, a revision and an item written to temp stores read back equal. - let packets = d.path().join("p.cc"); - let p = good(); - PacketStore::open(&packets) - .unwrap() - .admit(p.clone(), &w.inventory, &w.corpus, &w.entities) - .unwrap(); - assert_eq!( - PacketStore::open(&packets).unwrap().get(PACKET_ID).unwrap(), - Some(p) - ); - - let inventories = d.path().join("i.cc"); - let declared = feature( - "feature:1", - ValueKind::Options(weksa::OptionSet::new(vec![lab("a"), lab("b")]).unwrap()), - &[Slot::Relation, Slot::Referent], - ); - let mut concept = entry("concept:1", 1); - concept.gloss = "a gloss".into(); - let rev = rev_with( - vec![Change::Add(iv("concept:1")), Change::Add(iv("feature:1"))], - 1, - None, - ); - InventoryStore::open(&inventories) - .unwrap() - .admit_revision(rev.clone(), vec![concept.clone(), declared.clone()]) - .unwrap(); - let back = InventoryStore::open(&inventories) - .unwrap() - .inventory() - .unwrap(); - assert_eq!(back.entry(&iv("concept:1")), Some(&concept)); - assert_eq!(back.entry(&iv("feature:1")), Some(&declared)); - assert_eq!(back.latest(), Some(&rev)); - - let corpus = d.path().join("c.cc"); - let it = item("item:nibu:1", "round trip"); - CorpusStore::open(&corpus) - .unwrap() - .add(it.clone(), &ents(&d)) - .unwrap(); - assert_eq!( - CorpusStore::open(&corpus) - .unwrap() - .get("item:nibu:1") - .unwrap(), - Some(it) - ); -} - -#[test] -fn the_committed_stores_load_and_hold_clean_data() { - let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); - let inv = InventoryStore::open(&root.join("data/interlingua/inventory.cc")) - .unwrap() - .inventory() - .unwrap(); - assert!(inv.has_revision(1)); - assert!(inv.entry(&iv(PERSON)).is_some()); - let items = CorpusStore::open(&root.join("data/corpus/corpus.cc")) - .unwrap() - .all() - .unwrap(); - assert!(items.len() >= 3, "{} items", items.len()); - for i in &items { - assert!( - !i.text.trim().is_empty() && i.language == "en-US" && !i.withdrawn, - "{i:?}" - ); - if matches!(i.source, Source::ModelWritten { .. }) { - assert_ne!( - SplitScheme::V1.split(i), - Some(weksa::Split::Test), - "{}", - i.id - ); - } - } - let by_id = |id: &str| { - items - .iter() - .find(|i| i.id == id) - .unwrap_or_else(|| panic!("{id}")) - }; - // brokkr:1 carries the generator string its fixture wrote, verbatim. - let fixture = std::fs::read_to_string( - root.join("examples/persona-lowering/brokkr-edda-voice/lowered-output.yaml"), - ) - .unwrap(); - let written = fixture - .lines() - .find_map(|l| l.strip_prefix("generator: ")) - .expect("the fixture names its generator"); - assert_eq!(written, "codex-local-source-aware-draft"); - assert_eq!( - by_id("item:brokkr:1").source, - Source::ModelWritten { - family: ModelFamily::OpenAi, - generator: Some(written.into()), - prompt_digest: None - } - ); - // Fixtures that record no author are Unrecorded, never CastAuthored. - let unrecorded = Source::ModelWritten { - family: ModelFamily::Unrecorded, - generator: None, - prompt_digest: None, - }; - assert_eq!(by_id("item:nibu:1").source, unrecorded); - assert_eq!(by_id("item:nibu:2").source, unrecorded); - assert!( - !items - .iter() - .any(|i| matches!(i.source, Source::CastAuthored { .. })), - "no item claims a recorded human author" - ); - assert_eq!(items.len(), 3); - // Every committed entity attests, with non-empty strings, where it came - // from, and a cast entity's fixture is a file of this repository. - let entities = EntityStore::open(&root.join("data/entity/entity.cc")) - .unwrap() - .all() - .unwrap(); - assert!(!entities.is_empty()); - for e in &entities { - match &e.origin { - weksa::entity::EntityOrigin::Cast { persona, fixture } => { - assert!(!persona.is_empty(), "{}", e.id); - assert!(root.join(fixture).is_file(), "{}: {fixture}", e.id); - } - weksa::entity::EntityOrigin::Live { persona } => assert!(!persona.is_empty()), - weksa::entity::EntityOrigin::PublicDomain { work } => assert!(!work.is_empty()), - } - } -} - -#[test] -fn seed_reseeded() { - let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); - let entities = EntityStore::open(&root.join("data/entity/entity.cc")).unwrap(); - let ids: Vec = entities - .all() - .unwrap() - .iter() - .map(|e| e.id.to_string()) - .collect(); - assert_eq!(ids, ["entity:brokkr", "entity:nibu"]); - // The committed corpus replays through the door with its entities: every - // speaker resolves and every span is a stored form. - let items = CorpusStore::open(&root.join("data/corpus/corpus.cc")) - .unwrap() - .all() - .unwrap(); - assert_eq!(items.len(), 3); - // The committed inventory is the seed, reseeded (stance features in, the - // displayed-affect features out), then the nibu-proof revision. - let committed = InventoryStore::open(&root.join("data/interlingua/inventory.cc")) - .unwrap() - .inventory() - .unwrap(); - let (rev, mut entries) = seed(); - let (proof, proof_entries) = weksa::inventory::proof_revision(); - assert!(committed.has_revision(rev.n) && committed.has_revision(proof.n)); - entries.extend(proof_entries); - let mut have: Vec<_> = committed.entries().cloned().collect(); - let mut want = entries; - have.sort_by_key(|e| e.id); - want.sort_by_key(|e| e.id); - assert_eq!(have, want); - let d = dir(); - let mut replay = CorpusStore::open(&d.path().join("c.cc")).unwrap(); - for i in items { - assert!(entities.get(&i.speaker).unwrap().is_some(), "{}", i.id); - replay.add(i, &entities).unwrap(); - } -} diff --git a/crates/weksa/tests/validate.rs b/crates/weksa/tests/validate.rs deleted file mode 100644 index 3453376..0000000 --- a/crates/weksa/tests/validate.rs +++ /dev/null @@ -1,685 +0,0 @@ -mod common; -use common::*; -use weksa::packet::*; -use weksa::{Family, PacketStore, Refusal, Slot, validate}; - -fn refusals(p: &Packet) -> Vec { - validate(p, &inventory()).err().unwrap_or_default() -} - -fn one(p: &Packet) -> Refusal { - let mut r = refusals(p); - assert_eq!(r.len(), 1, "{r:?}"); - r.remove(0) -} - -fn unresolved(id: &str) -> Refusal { - Refusal::V5Unresolved { id: iv(id) } -} - -fn empty_payload() -> Packet { - let mut p = good(); - p.referents.clear(); - p.predications.clear(); - p.relations.clear(); - p.discourse.addressee = None; - p.context.participants.clear(); - p.context.audience.clear(); - p.constraints.preserve.clear(); - p -} - -fn feat(id: &str, value: FeatValue) -> FeatureValue { - FeatureValue { - feature: iv(id), - value, - } -} - -#[test] -fn a_good_packet_validates() { - assert!(validate(&good(), &inventory()).is_ok()); -} - -#[test] -fn v1_schema_version() { - let mut p = good(); - p.schema_version = SCHEMA_VERSION + 1; - assert_eq!( - refusals(&p), - vec![Refusal::V1SchemaVersion { - found: SCHEMA_VERSION + 1 - }] - ); -} - -#[test] -fn v2_payload_required() { - assert_eq!(refusals(&empty_payload()), vec![Refusal::V2PayloadRequired]); - // Any one family is payload enough. - let mut only_relations = empty_payload(); - only_relations.relations = good().relations; - only_relations.relations[0].from = nd("n8"); - only_relations.relations[0].to = nd("n9"); - assert!(!refusals(&only_relations).contains(&Refusal::V2PayloadRequired)); - let mut only_predications = empty_payload(); - only_predications.predications = good().predications; - only_predications.predications[0].roles.clear(); - assert_eq!(refusals(&only_predications), vec![]); - let mut only_referents = empty_payload(); - only_referents.referents = good().referents; - assert_eq!(refusals(&only_referents), vec![]); -} - -#[test] -fn v3_unique_node_ids() { - let mut p = good(); - p.predications[0].id = nd("n1"); - p.relations[0].from = nd("n1"); - assert_eq!( - refusals(&p), - vec![Refusal::V3DuplicateNode { id: nd("n1") }] - ); - let mut twice = good(); - twice.referents.push(twice.referents[0].clone()); - assert_eq!( - refusals(&twice), - vec![Refusal::V3DuplicateNode { id: nd("n1") }] - ); -} - -#[test] -fn v4_refs_resolve() { - let dangling = |edit: fn(&mut Packet), at: &'static str| { - let mut p = good(); - edit(&mut p); - assert_eq!( - refusals(&p), - vec![Refusal::V4DanglingRef { id: nd("n99"), at }], - "{at}" - ); - }; - dangling( - |p| p.predications[0].roles[0].filler = nd("n99"), - "role filler", - ); - dangling(|p| p.relations[0].from = nd("n99"), "relation from"); - dangling(|p| p.relations[0].to = nd("n99"), "relation to"); - dangling(|p| p.discourse.addressee = Some(nd("n99")), "addressee"); - dangling(|p| p.context.participants.push(nd("n99")), "participant"); - dangling(|p| p.context.audience.push(nd("n99")), "audience"); - dangling(|p| p.constraints.preserve.push(nd("n99")), "preserve"); -} - -#[test] -fn v5_inventory_resolves_at_rev() { - let with_concept = |rev: u32, id: &str| { - let mut p = good(); - p.inventory_rev = rev; - p.referents[0].concept = iv(id); - p - }; - // Introduced after the packet's rev. - assert_eq!( - one(&with_concept(2, CONCEPT_LATE)), - unresolved(CONCEPT_LATE) - ); - // Introduced at the packet's rev. - assert_eq!(refusals(&with_concept(4, CONCEPT_LATE)), vec![]); - // Retired later than the packet's rev still resolves. - assert_eq!(refusals(&with_concept(2, CONCEPT_RETIRED)), vec![]); - // Retired at or before the packet's rev does not. - assert_eq!( - one(&with_concept(3, CONCEPT_RETIRED)), - unresolved(CONCEPT_RETIRED) - ); - assert_eq!( - one(&with_concept(4, CONCEPT_RETIRED)), - unresolved(CONCEPT_RETIRED) - ); - // An id no revision holds. - assert_eq!( - one(&with_concept(2, "concept:99")), - unresolved("concept:99") - ); - // An extension is resolved like any inventory id. - let mut ext = good(); - ext.extensions[0].key = iv("extension:99"); - assert_eq!(one(&ext), unresolved("extension:99")); -} - -#[test] -fn v6_family_matches_slot() { - let wrong = |edit: fn(&mut Packet), id: &str, expected: Family, at: &'static str| { - let mut p = good(); - edit(&mut p); - assert_eq!( - refusals(&p), - vec![Refusal::V6WrongFamily { - id: iv(id), - expected, - at - }], - "{at}" - ); - }; - wrong( - |p| p.discourse.features[0].feature = iv(AGENT), - AGENT, - Family::Feature, - "feature", - ); - wrong( - |p| p.discourse.speech_act = Some(iv("relation:1")), - "relation:1", - Family::SpeechAct, - "speech act", - ); - wrong( - |p| p.referents[0].kind = iv(CONCEPT_A), - CONCEPT_A, - Family::ReferentKind, - "referent kind", - ); - wrong( - |p| p.referents[0].concept = iv(PERSON), - PERSON, - Family::Concept, - "referent concept", - ); - wrong( - |p| p.predications[0].concept = iv(FRAME), - FRAME, - Family::Concept, - "predication concept", - ); - wrong( - |p| p.predications[0].frame = iv(CONCEPT_A), - CONCEPT_A, - Family::Frame, - "frame", - ); - wrong( - |p| p.predications[0].roles[0].role = iv(HEDGE), - HEDGE, - Family::Role, - "role", - ); - wrong( - |p| p.relations[0].relation = iv(AGENT), - AGENT, - Family::Relation, - "relation", - ); - wrong( - |p| p.context.setting = Some(iv(FRAME)), - FRAME, - Family::Concept, - "context concept", - ); - wrong( - |p| p.context.medium = Some(iv(FRAME)), - FRAME, - Family::Concept, - "context concept", - ); - wrong( - |p| p.extensions[0].key = iv(CONCEPT_A), - CONCEPT_A, - Family::Extension, - "extension", - ); -} - -#[test] -fn v7_value_kind() { - let hedge = |v: FeatValue| { - let mut p = good(); - p.discourse.features[0].value = v; - refusals(&p) - }; - let bad = vec![Refusal::V7ValueKind { what: HEDGE.into() }]; - assert_eq!(hedge(FeatValue::Grade(grade(1000))), vec![]); - assert_eq!(hedge(FeatValue::Flag(true)), bad); - assert_eq!(hedge(FeatValue::Option(lab("x"))), bad); - let modality = |v: FeatValue| { - let mut p = good(); - p.predications[0].features[0].value = v; - refusals(&p) - }; - let bad_modality = vec![Refusal::V7ValueKind { - what: MODALITY.into(), - }]; - assert_eq!(modality(FeatValue::Option(lab("necessary"))), vec![]); - assert_eq!(modality(FeatValue::Option(lab("banana"))), bad_modality); - assert_eq!(modality(FeatValue::Grade(grade(1))), bad_modality); - let mut flag = good(); - flag.discourse.features = vec![feat(IRONY, FeatValue::Flag(false))]; - assert_eq!(refusals(&flag), vec![]); - flag.discourse.features[0].value = FeatValue::Grade(grade(5)); - assert_eq!( - refusals(&flag), - vec![Refusal::V7ValueKind { what: IRONY.into() }] - ); -} - -#[test] -fn extension_key_is_an_extension_entry() { - // A concept id is not an extension; an extension entry fits its value kind. - let mut by_concept = good(); - by_concept.extensions[0].key = iv(CONCEPT_A); - assert_eq!( - one(&by_concept), - Refusal::V6WrongFamily { - id: iv(CONCEPT_A), - expected: Family::Extension, - at: "extension" - } - ); - let mut flagged = good(); - flagged.extensions[0].value = FeatValue::Flag(true); - assert_eq!(one(&flagged), Refusal::V7ValueKind { what: EXT.into() }); - let mut optioned = good(); - optioned.extensions[0].value = FeatValue::Option(lab("x")); - assert_eq!(one(&optioned), Refusal::V7ValueKind { what: EXT.into() }); - let mut unknown = good(); - unknown.extensions[0].key = iv("extension:2"); - assert_eq!(one(&unknown), unresolved("extension:2")); - // No extension means none to check. - let mut none = good(); - none.extensions.clear(); - assert_eq!(refusals(&none), vec![]); -} - -#[test] -fn v8_feature_applies_to() { - let mut p = good(); - // polarity applies to predications only. - p.discourse - .features - .push(feat(POLARITY, FeatValue::Option(lab("negative")))); - assert_eq!( - refusals(&p), - vec![Refusal::V8NotApplicable { - feature: iv(POLARITY), - slot: Slot::Discourse - }] - ); - let mut referent = good(); - referent.referents[0] - .features - .push(feat(HEDGE, FeatValue::Grade(grade(1)))); - assert_eq!( - refusals(&referent), - vec![Refusal::V8NotApplicable { - feature: iv(HEDGE), - slot: Slot::Referent - }] - ); - let mut relation = good(); - relation.relations[0] - .features - .push(feat(HEDGE, FeatValue::Grade(grade(1)))); - assert_eq!( - refusals(&relation), - vec![Refusal::V8NotApplicable { - feature: iv(HEDGE), - slot: Slot::Relation - }] - ); - let mut pred = good(); - pred.predications[0] - .features - .push(feat(HEDGE, FeatValue::Grade(grade(1)))); - assert_eq!(refusals(&pred), vec![]); -} - -#[test] -fn stance_features_apply_to_the_stance_only() { - // Affect and restraint sit in slot Stance and nowhere else. - assert_eq!(refusals(&good()), vec![]); - for id in [ANGER, RESTRAINT] { - let mut p = good(); - p.discourse - .features - .push(feat(id, FeatValue::Grade(grade(1)))); - assert_eq!( - refusals(&p), - vec![Refusal::V8NotApplicable { - feature: iv(id), - slot: Slot::Discourse - }] - ); - } - let mut referent = good(); - referent.referents[0] - .features - .push(feat(AFFECTION, FeatValue::Grade(grade(1)))); - assert_eq!( - refusals(&referent), - vec![Refusal::V8NotApplicable { - feature: iv(AFFECTION), - slot: Slot::Referent - }] - ); - let mut predication = good(); - predication.predications[0] - .features - .push(feat(ANGER, FeatValue::Grade(grade(1)))); - assert_eq!( - refusals(&predication), - vec![Refusal::V8NotApplicable { - feature: iv(ANGER), - slot: Slot::Predication - }] - ); - // Hedge is a stance on content, not a felt state: not in the stance. - let mut p = good(); - p.stance - .features - .push(feat(HEDGE, FeatValue::Grade(grade(1)))); - assert_eq!( - refusals(&p), - vec![Refusal::V8NotApplicable { - feature: iv(HEDGE), - slot: Slot::Stance - }] - ); - // A stance feature takes its inventory's value kind (V7) and must exist (V5). - let mut flag = good(); - flag.stance.features = vec![feat(ANGER, FeatValue::Flag(true))]; - assert_eq!( - refusals(&flag), - vec![Refusal::V7ValueKind { what: ANGER.into() }] - ); - let mut missing = good(); - missing.stance.features = vec![feat("feature:99", FeatValue::Grade(grade(1)))]; - assert_eq!(refusals(&missing), vec![unresolved("feature:99")]); - // A default stance is valid: the packet records no felt state. - let mut none = good(); - none.stance = Stance::default(); - assert_eq!(refusals(&none), vec![]); -} - -#[test] -fn displayed_formality_warmth_and_intensity_are_not_in_the_inventory() { - // Felt affect is the packet's; how it is displayed is each language's - // ontology's, so no feature carries it. - let inv = inventory(); - for gone in ["warmth", "formality", "intensity"] { - assert!(inv.entries().all(|e| e.gloss != gone), "{gone}"); - } -} - -#[test] -fn v10_rev_exists() { - let mut p = good(); - p.inventory_rev = 9; - assert_eq!(refusals(&p), vec![Refusal::V10UnknownRevision { rev: 9 }]); - p.inventory_rev = 0; - assert_eq!(refusals(&p), vec![Refusal::V10UnknownRevision { rev: 0 }]); -} - -#[test] -fn every_broken_rule_is_reported() { - let mut p = good(); - p.schema_version = 7; - p.predications[0].roles[0].filler = nd("n99"); - p.extensions[0].key = iv(CONCEPT_A); - let rules: Vec<&str> = refusals(&p).iter().map(Refusal::rule).collect(); - assert_eq!(rules, vec!["V1", "V4", "V6"]); -} - -#[test] -fn lowering_input_carries_no_trace_id_or_provenance() { - let mut p = good(); - p.trace.notes = vec!["SECRET-NOTE".into()]; - p.trace.rejected_readings = vec!["SECRET-READING".into()]; - // Only the packet store's door yields the view. - let w = world(); - let mut store = PacketStore::open(&w.dir.path().join("p.cc")).unwrap(); - let input = store - .admit(p.clone(), &w.inventory, &w.corpus, &w.entities) - .unwrap(); - let seen = format!("{input:?}"); - for hidden in [PACKET_ID, ENCODER_ID, "item:nibu:1", "SECRET"] { - assert!(!seen.contains(hidden), "{hidden} reached the input: {seen}"); - } - // The view is the packet's meaning, readable through getters. - assert_eq!(input.inventory_rev(), 2); - assert_eq!(input.kind(), PacketKind::DialogueLine); - assert_eq!(input.referents(), p.referents.as_slice()); - assert_eq!(input.predications(), p.predications.as_slice()); - assert_eq!(input.relations(), p.relations.as_slice()); - assert_eq!(input.extensions(), p.extensions.as_slice()); - assert_eq!(input.context(), &p.context); - assert_eq!(input.discourse(), &p.discourse); - assert_eq!(input.constraints(), &p.constraints); - // The speaker's stance is meaning, not provenance: it reaches the lowerer. - assert_eq!(input.stance(), &p.stance); - assert!(!p.stance.features.is_empty()); -} - -#[test] -fn invalid_packet_has_no_lowering_input() { - let mut p = good(); - p.schema_version = 99; - p.referents.clear(); - let refused = validate(&p, &inventory()).unwrap_err(); - let rules: Vec<&str> = refused.iter().map(Refusal::rule).collect(); - assert_eq!(rules[0], "V1"); - assert!(rules.contains(&"V4"), "{rules:?}"); -} - -fn grade_feature(grade_value: u16) -> FeatureValue { - feat(FEAT_ALL, FeatValue::Grade(grade(grade_value))) -} - -fn referent(id: &str) -> Referent { - Referent { - id: nd(id), - kind: iv(PERSON), - concept: iv(CONCEPT_A), - entity: None, - features: vec![], - } -} - -fn v11(slot: Option, feature: &str) -> Refusal { - Refusal::V11DuplicateFeature { - feature: iv(feature), - slot, - } -} - -#[test] -fn v11_duplicate_feature_each_slot() { - let mut discourse = good(); - discourse.discourse.features = vec![grade_feature(1), grade_feature(2)]; - assert_eq!( - refusals(&discourse), - vec![v11(Some(Slot::Discourse), FEAT_ALL)] - ); - let mut referent = good(); - referent.referents[0].features = vec![grade_feature(1), grade_feature(2)]; - assert_eq!( - refusals(&referent), - vec![v11(Some(Slot::Referent), FEAT_ALL)] - ); - let mut predication = good(); - predication.predications[0] - .features - .extend([grade_feature(1), grade_feature(2)]); - assert_eq!( - refusals(&predication), - vec![v11(Some(Slot::Predication), FEAT_ALL)] - ); - let mut relation = good(); - relation.relations[0].features = vec![grade_feature(1), grade_feature(2)]; - assert_eq!( - refusals(&relation), - vec![v11(Some(Slot::Relation), FEAT_ALL)] - ); - let mut stance = good(); - stance.stance.features = vec![grade_feature(1), grade_feature(2)]; - assert_eq!(refusals(&stance), vec![v11(Some(Slot::Stance), FEAT_ALL)]); - let mut felt = good(); - felt.stance - .features - .push(feat(ANGER, FeatValue::Grade(grade(5)))); - assert_eq!(refusals(&felt), vec![v11(Some(Slot::Stance), ANGER)]); - let mut extension = good(); - extension.extensions.push(Extension { - key: iv(EXT), - value: FeatValue::Grade(grade(4)), - }); - assert_eq!(refusals(&extension), vec![v11(None, EXT)]); -} - -#[test] -fn v11_counts_per_list_not_per_packet() { - // The same feature once in each list, and different features in one list, are fine. - let mut p = good(); - p.discourse.features = vec![grade_feature(1), feat(HEDGE, FeatValue::Grade(grade(2)))]; - p.referents[0].features = vec![grade_feature(1)]; - p.predications[0].features.push(grade_feature(1)); - p.relations[0].features = vec![grade_feature(1)]; - assert_eq!(refusals(&p), vec![]); -} - -#[test] -fn v11_grade_bytes_packet_refused() { - // Soul s2: 113 hedge grades in one list spell a line byte by byte. - let mut p = good(); - p.discourse.features = (0..113) - .map(|i| feat(HEDGE, FeatValue::Grade(grade(i)))) - .collect(); - let got = refusals(&p); - assert_eq!(got.len(), 112, "{got:?}"); - assert!(got.iter().all(|r| *r == v11(Some(Slot::Discourse), HEDGE))); -} - -#[test] -fn v12_node_bound() { - // 62 referents, the predication n2 and the relation make 64 nodes. - let with_referents = |last: u32| { - let mut p = good(); - let ids = std::iter::once(1).chain(3..=last); - p.referents = ids.map(|k| referent(&format!("n{k}"))).collect(); - p - }; - let at_bound = with_referents(63); - assert_eq!( - at_bound.referents.len() + at_bound.predications.len() + at_bound.relations.len(), - Packet::MAX_NODES - ); - assert_eq!(refusals(&at_bound), vec![]); - assert_eq!( - refusals(&with_referents(64)), - vec![Refusal::V12TooManyNodes { - count: 65, - max: Packet::MAX_NODES - }] - ); -} - -// Soul s3: a repeated node id in a node list, or a repeated role in one -// predication, spelled a line one bit per entry. Each list now holds a node or -// a role at most once, so the channel is bounded. - -const LINE: &str = "Take your hand off the hatch, Captain."; - -fn bits(s: &str) -> Vec { - s.bytes() - .flat_map(|b| (0..8).rev().map(move |i| (b >> i) & 1 == 1)) - .collect() -} - -fn two_referents() -> Packet { - let mut p = good(); - let mut r = p.referents[0].clone(); - r.id = nd("n3"); - p.referents.push(r); - p -} - -fn bit_ids(s: &str) -> Vec { - bits(s) - .into_iter() - .map(|b| nd(if b { "n3" } else { "n1" })) - .collect() -} - -fn dup_node(id: &str, at: &'static str) -> Refusal { - Refusal::V11DuplicateNodeRef { id: nd(id), at } -} - -#[test] -fn v11_node_lists_refuse_repeats_and_allow_each_node_once() { - let mut ok = two_referents(); - ok.context.participants = vec![nd("n1"), nd("n3")]; - ok.context.audience = vec![nd("n3"), nd("n1")]; - ok.constraints.preserve = vec![nd("n3"), nd("n1")]; - assert_eq!(refusals(&ok), vec![]); - - let mut p = two_referents(); - p.context.participants = vec![nd("n1"), nd("n1")]; - assert_eq!(refusals(&p), vec![dup_node("n1", "participant")]); - let mut p = two_referents(); - p.context.audience = vec![nd("n3"), nd("n1"), nd("n3")]; - assert_eq!(refusals(&p), vec![dup_node("n3", "audience")]); - let mut p = two_referents(); - p.constraints.preserve = vec![nd("n1"), nd("n1")]; - assert_eq!(refusals(&p), vec![dup_node("n1", "preserve")]); -} - -#[test] -fn v11_bit_channel_in_each_node_list_refused() { - let n = bits(LINE).len(); - let mut p = two_referents(); - p.context.participants = bit_ids(LINE); - let got = refusals(&p); - assert_eq!(got.len(), n - 2, "{got:?}"); - assert!(got.iter().all(|r| r.rule() == "V11")); - let mut p = two_referents(); - p.context.audience = bit_ids(LINE); - assert_eq!(refusals(&p).len(), n - 2); - let mut p = two_referents(); - p.constraints.preserve = bit_ids(LINE); - assert_eq!(refusals(&p).len(), n - 2); -} - -#[test] -fn v11_unbounded_participants_refused() { - let mut p = two_referents(); - p.context.participants = (0..100_000) - .map(|i| nd(if i % 3 == 0 { "n3" } else { "n1" })) - .collect(); - assert_eq!(refusals(&p).len(), 100_000 - 2); -} - -#[test] -fn v11_role_repeat_in_one_predication_refused() { - let dup = |role: &str| Refusal::V11DuplicateRole { - role: iv(role), - predication: nd("n2"), - }; - let mut p = good(); - p.predications[0].roles = bits(LINE) - .into_iter() - .map(|b| RoleFill { - role: iv(if b { AGENT } else { PATIENT }), - filler: nd("n1"), - }) - .collect(); - let got = refusals(&p); - assert_eq!(got.len(), bits(LINE).len() - 2, "{got:?}"); - assert!(got.contains(&dup(AGENT)) && got.contains(&dup(PATIENT))); - - // The same role in two different predications is not a repeat. - let mut ok = good(); - let mut second = ok.predications[0].clone(); - second.id = nd("n4"); - ok.predications.push(second); - assert_eq!(refusals(&ok), vec![]); -} diff --git a/data/corpus/corpus.cc b/data/corpus/corpus.cc deleted file mode 100644 index 619a875..0000000 --- a/data/corpus/corpus.cc +++ /dev/null @@ -1 +0,0 @@ -“²cultcache.store.v1‘—±weksa.corpus_item±weksa.corpus_item´weksa.corpus_item.v1±weksa.corpus_itemÙV{"schemaName":"weksa.corpus_item","schemaVersion":"weksa.corpus_item.v1","members":[]}‘±weksa.corpus_item“”­item:brokkr:1±weksa.corpus_item´2026-10-29T14:59:18ZĨ›­item:brokkr:1Ù,The command has no receipt. It is not truth.¥en-US­entity:brokkr±brokkr-edda-voice¬ModelWritten“¦OpenAi¾codex-local-source-aware-draftÀª2026-10-29À”«item:nibu:1±weksa.corpus_item´2026-10-29T14:59:18ZÄÉ›«item:nibu:1ÙnTake your hand off that control unless you want your nervous system to become my least useful circuit diagram.¥en-US«entity:nibu®nibu-proof-001¬ModelWritten“ªUnrecordedÀÀª2026-10-29À”«item:nibu:2±weksa.corpus_item´2026-10-29T14:59:18ZÄ웫item:nibu:2Ù‘Do not touch that control. It is damaged; one touch may put you directly in harm's way. I am preventing an avoidable injury, not asking politely.¥en-US«entity:nibu®nibu-proof-001¬ModelWritten“ªUnrecordedÀÀª2026-10-29À \ No newline at end of file diff --git a/data/cultural-ontology/README.md b/data/cultural-ontology/README.md index c3e3b7c..26adf3c 100644 --- a/data/cultural-ontology/README.md +++ b/data/cultural-ontology/README.md @@ -20,6 +20,7 @@ may retrieve candidates, but accepted ontology remains typed state. - `cultures/`: culture or life-history profiles that can shape lowering when a Persona explicitly references them. +- `subcultures/`: community-of-practice and register profiles. - `referents/`: bounded source-text, mythic, literary, ritual, or historical referents. These are not whole-culture profiles; they preserve reusable role, salience, affordance, and forbidden-flattening pressure. diff --git a/data/cultural-ontology/catalog.yaml b/data/cultural-ontology/catalog.yaml index b45e8a4..b0a6bc6 100644 --- a/data/cultural-ontology/catalog.yaml +++ b/data/cultural-ontology/catalog.yaml @@ -14,8 +14,18 @@ qdrant: Qdrant owns retrieval over culture/subculture chunks. It does not own accepted ontology, activation, or lowering authority. profiles: + - profile_id: weksa.culture.dutch_american_diaspora.v0 + path: data/cultural-ontology/cultures/dutch-american-diaspora.yaml - profile_id: weksa.culture.western_europe_maritime_childhood.v0 path: data/cultural-ontology/cultures/western-europe-maritime-childhood.yaml + - profile_id: weksa.culture.portugal_resident_algarve_lisbon.v0 + path: data/cultural-ontology/cultures/portugal-resident-algarve-lisbon.yaml + - profile_id: weksa.culture.texas_houston_us_adolescence.v0 + path: data/cultural-ontology/cultures/texas-houston-us-adolescence.yaml + - profile_id: weksa.subculture.online_game_dev_discord_pre2025.v0 + path: data/cultural-ontology/subcultures/online-game-dev-discord-pre2025.yaml + - profile_id: weksa.subculture.gamecult_open_source_crypto_anarchist.v0 + path: data/cultural-ontology/subcultures/gamecult-open-source-crypto-anarchist.yaml - profile_id: weksa.cultural_referent.old_norse_edda_source_tradition.v0 path: data/cultural-ontology/referents/old-norse-edda-source-tradition.yaml - profile_id: weksa.cultural_referent.old_norse_edda_brokkr_skaldskaparmal.v0 diff --git a/data/cultural-ontology/cultures/dutch-american-diaspora.yaml b/data/cultural-ontology/cultures/dutch-american-diaspora.yaml new file mode 100644 index 0000000..4c46b64 --- /dev/null +++ b/data/cultural-ontology/cultures/dutch-american-diaspora.yaml @@ -0,0 +1,58 @@ +schema_version: weksa.cultural_ontology_profile.v0 +profile_id: weksa.culture.dutch_american_diaspora.v0 +kind: diaspora_identity +status: draft_guarded +language: + codes: + - en + - nl +community: + regions: + - Netherlands + - United States + speech_community: > + People whose self-description or life history crosses Dutch origin and + American nationality/residence. This profile is not a dialect claim. +activation: + allowed_when: + - persona_state_references_profile + - interlingua_mentions_origin_nationality_or_cross_cultural_identity + forbidden_when: + - used_to_infer_accent_or_fluency_without_evidence + - used_as_generic_dutch_or_generic_american_stereotype +salience_axes: + plural_national_identity: + axis_id: plural_national_identity + domain: identity_indexing + description: Birthplace, citizenship, residence, and cultural identification may diverge. + relevant_distinctions: + - distinction_id: birthplace_vs_citizenship + label: Birthplace versus citizenship + meaning: Being born in one country and holding identity/citizenship in another are separate ontology fields. + lowering_questions: + - Does the utterance need origin, nationality, residence, or cultural affiliation? + required_when: + - source_mentions_identity_origin_or_nationality + forbidden_when: + - scene_only_requires_casual_banter + evidence_refs: + - discord:805822798185627718 + - discord:1030577958215225396 +evidence_ledger: + - ref: discord:805822798185627718 + quote: "I live in Portugal at the moment, though I am a Netherlands-born US citizen - lived mostly in Texas" + - ref: discord:1030577958215225396 + quote: "Dual nationality, Dutch American" + - ref: discord:862490040755683348 + quote: "I was born in the Netherlands but I'm a US citizen and did high school in Houston, Texas." +linguistic_affordances: + grammar_features: [] + phonology_features: [] + lexicon_features: [] + discourse_patterns: [] + orthography_features: [] + code_switching: + allowed: false + constraints: + - Do not infer Dutch code switching from diaspora identity alone. + patterns: [] diff --git a/data/cultural-ontology/cultures/portugal-resident-algarve-lisbon.yaml b/data/cultural-ontology/cultures/portugal-resident-algarve-lisbon.yaml new file mode 100644 index 0000000..0722576 --- /dev/null +++ b/data/cultural-ontology/cultures/portugal-resident-algarve-lisbon.yaml @@ -0,0 +1,84 @@ +schema_version: weksa.cultural_ontology_profile.v0 +profile_id: weksa.culture.portugal_resident_algarve_lisbon.v0 +kind: regional_residence +status: draft_guarded +language: + codes: + - en + - pt +community: + regions: + - Portugal + - Algarve + - Lisbon + - Tavira + speech_community: > + Portugal as lived residence and locality rather than tourist backdrop. +activation: + allowed_when: + - persona_state_references_profile + - interlingua_mentions_portugal_residence_locality_bureaucracy_or_timing + forbidden_when: + - used_for_broad_portuguese_culture_without_locale_review +salience_axes: + lived_portugal: + axis_id: lived_portugal + domain: identity_indexing + description: Portugal references can be local/home references, including Algarve/Lisbon/Tavira specificity. + relevant_distinctions: + - distinction_id: current_or_returned_residence + label: Current or returned residence + meaning: Portugal can be present-tense home context, not childhood-only memory. + lowering_questions: + - Is Portugal being invoked as current location, childhood location, or comparison frame? + required_when: + - source_mentions_portugal_or_portuguese_residence + forbidden_when: + - scene_has_no_place_salience + evidence_refs: + - discord:581220303414231117 + - discord:834858944013926421 + - discord:888820124344254504 +evidence_ledger: + - ref: discord:581220303414231117 + quote: "I'm in the Algarve now" + - ref: discord:834858944013926421 + quote: "I live in Portugal now" + - ref: discord:888820124344254504 + quote: "But I'm currently in Lisbon" + - ref: discord:1023774733789040640 + quote: "if you wanna move to Portugal you're gonna have to get used to not getting things on time" +linguistic_affordances: + grammar_features: [] + phonology_features: [] + lexicon_features: [] + discourse_patterns: + - pattern_id: portugal_timing_joke_guarded + label: Portugal timing joke, guarded + status: draft + domain: + layer: discourse + category: humor + subcategory: local_bureaucracy_timing + interactional_function: locally_grounded_timing_joke + sequence_shape: + - playful_warning + - local_timing_generalization + adjacency_constraints: + - addressee_is_discussing_moving_or_living_in_portugal + activation: + allowed_when: + - local_portugal_residence_context + - joking_register + forbidden_when: + - formal_cultural_description + evidence: + source_refs: + - discord:1023774733789040640 + confidence: low + orthography_features: [] + code_switching: + allowed: false + constraints: + - This profile alone does not authorize Portuguese code switching. + patterns: [] diff --git a/data/cultural-ontology/cultures/texas-houston-us-adolescence.yaml b/data/cultural-ontology/cultures/texas-houston-us-adolescence.yaml new file mode 100644 index 0000000..09d5619 --- /dev/null +++ b/data/cultural-ontology/cultures/texas-houston-us-adolescence.yaml @@ -0,0 +1,56 @@ +schema_version: weksa.cultural_ontology_profile.v0 +profile_id: weksa.culture.texas_houston_us_adolescence.v0 +kind: adolescence_residence +status: draft_guarded +language: + codes: + - en +community: + regions: + - United States + - Texas + - Houston + speech_community: > + US/Texas adolescent institutional context, especially Houston high school + and later critique of American debt/institutional pathways. +activation: + allowed_when: + - persona_state_references_profile + - interlingua_mentions_us_texas_school_debt_or_american_institutions + forbidden_when: + - used_to_force_texan_dialect_markers +salience_axes: + american_institutional_distance: + axis_id: american_institutional_distance + domain: stance + description: US institutions can be lived experience and object of critique rather than foreign abstraction. + relevant_distinctions: + - distinction_id: debt_train_refusal + label: American debt train refusal + meaning: Avoiding US education/debt pathways can be a first-person institutional stance. + lowering_questions: + - Is the utterance about education, debt, leaving the US, or institutional pressure? + required_when: + - source_mentions_us_debt_or_school_path + forbidden_when: + - casual_fixture_without_institutional_topic + evidence_refs: + - discord:1160260490690306148 +evidence_ledger: + - ref: discord:805822798185627718 + quote: "lived mostly in Texas" + - ref: discord:862490040755683348 + quote: "did high school in Houston, Texas" + - ref: discord:1160260490690306148 + quote: "I specifically left the US after high school so as not to climb onto the traditional American debt train" +linguistic_affordances: + grammar_features: [] + phonology_features: [] + lexicon_features: [] + discourse_patterns: [] + orthography_features: [] + code_switching: + allowed: false + constraints: + - Do not infer Southern or Texan dialect surface markers from residence alone. + patterns: [] diff --git a/data/cultural-ontology/cultures/western-europe-maritime-childhood.yaml b/data/cultural-ontology/cultures/western-europe-maritime-childhood.yaml index 5f6a279..39d4cbc 100644 --- a/data/cultural-ontology/cultures/western-europe-maritime-childhood.yaml +++ b/data/cultural-ontology/cultures/western-europe-maritime-childhood.yaml @@ -10,8 +10,8 @@ community: - coastal_western_europe speech_community: > Mobile childhood organized around sailing, ports, and cross-border Western - European movement. This profile is owner-attested and needs corroboration - before broader use. + European movement. This profile is owner-attested for the Metame fixture and + needs more Discord corroboration before broader use. activation: allowed_when: - persona_state_references_profile @@ -35,6 +35,11 @@ salience_axes: - scene_has_no_mobility_or_home_salience evidence_refs: - owner:2026-06-10:sailboat_france_spain_portugal_until_6 +evidence_ledger: + - ref: owner:2026-06-10:sailboat_france_spain_portugal_until_6 + quote: "I was born in the Netherlands, grew up on a sailboat traveling France, Spain, staying in Portugal until I was 6, moved to the US." + - ref: discord-search-gap:sailboat_childhood + summary: First semantic search did not locate a direct Discord message for this exact childhood account. linguistic_affordances: grammar_features: [] phonology_features: [] diff --git a/data/cultural-ontology/subcultures/gamecult-open-source-crypto-anarchist.yaml b/data/cultural-ontology/subcultures/gamecult-open-source-crypto-anarchist.yaml new file mode 100644 index 0000000..6af5f41 --- /dev/null +++ b/data/cultural-ontology/subcultures/gamecult-open-source-crypto-anarchist.yaml @@ -0,0 +1,51 @@ +schema_version: weksa.cultural_ontology_profile.v0 +profile_id: weksa.subculture.gamecult_open_source_crypto_anarchist.v0 +kind: political_technical_community +status: draft_guarded +language: + codes: + - en +community: + regions: + - online + speech_community: > + GameCult-adjacent open-source, crypto-anarchist, game-development, and + engineering practice where ownership, inspectability, debt, authority, and + tool coherence are culturally salient. +activation: + allowed_when: + - persona_state_references_profile + - interlingua_mentions_tools_authority_open_source_debt_game_dev_or_engineering + forbidden_when: + - used_to_turn_random_banter_into_manifesto +salience_axes: + authority_and_tool_coherence: + axis_id: authority_and_tool_coherence + domain: stance + description: Systems are judged by ownership, inspectability, agency, and whether they actually work. + relevant_distinctions: + - distinction_id: open_source_anti_custody + label: Open source anti-custody + meaning: Open access and inspectability are political and practical values. + lowering_questions: + - Is the line judging a tool, institution, ownership structure, or coordination method? + - Should the interlingua preserve anti-custody or open-access salience? + required_when: + - source_mentions_open_source_or_institutional_authority + forbidden_when: + - casual_social_move_only + evidence_refs: + - discord:862490040755683348 +linguistic_affordances: + grammar_features: [] + phonology_features: [] + lexicon_features: [] + discourse_patterns: [] + orthography_features: [] + code_switching: + allowed: false + constraints: [] + patterns: [] +evidence_ledger: + - ref: discord:862490040755683348 + quote: "I got my bachelor degree in game development last year. I'm a crypto-anarchist open source activist" diff --git a/data/cultural-ontology/subcultures/online-game-dev-discord-pre2025.yaml b/data/cultural-ontology/subcultures/online-game-dev-discord-pre2025.yaml new file mode 100644 index 0000000..42ee8f7 --- /dev/null +++ b/data/cultural-ontology/subcultures/online-game-dev-discord-pre2025.yaml @@ -0,0 +1,288 @@ +schema_version: weksa.cultural_ontology_profile.v0 +profile_id: weksa.subculture.online_game_dev_discord_pre2025.v0 +kind: community_of_practice +status: draft_guarded +language: + codes: + - en +community: + regions: + - online_discord + speech_community: > + Pre-2025 GameCult/Aquarium-ish Discord collaboration where game design, + sound, engineering, jokes, and casual status play cohabit the same rooms. +activation: + allowed_when: + - persona_state_references_profile + - target_medium_is_discord + - target_period_is_pre_2025_or_history_fixture + forbidden_when: + - formal_current_project_doctrine_is_target +salience_axes: + casual_room_temperature: + axis_id: casual_room_temperature + domain: discourse_patterns + description: Compact, socially alive wording that assumes shared room context. + relevant_distinctions: + - distinction_id: banter_over_explanation + label: Banter over explanation + meaning: The line may be short, fragmentary, and joke-shaped when context supplies the frame. + lowering_questions: + - Can the utterance be one sentence or fragment? + - Does explanation weaken the social move? + required_when: + - fixture_style_tags_include_casual + forbidden_when: + - formal_review_or_doctrine_is_requested + evidence_refs: + - discord:802911591288537118 + - discord:820077384572338176 +linguistic_affordances: + grammar_features: + - feature_id: fragmentary_like_comparison + label: Fragmentary Like comparison + status: draft + domain: + layer: grammar + category: comparison + subcategory: fragment + description: > + A joke-description may start with "Like..." and remain fragmentary when + it is an immediate continuation of collaborative design talk. + realization: + function: casual_comic_comparison + surface_strategy: "Like a really terrible that " + examples: + - "Like a really terrible engine that sounds like coconuts when you use it" + activation: + allowed_when: + - collaborative_design_banter + forbidden_when: + - standalone_formal_explanation + requires_profile_authority: true + minimum_density: light + maximum_density: normal + evidence: + source_refs: + - discord:802911591288537118 + confidence: medium + phonology_features: [] + lexicon_features: + - feature_id: casual_recoil_oof + label: Casual recoil marker oof + status: draft + domain: + layer: lexicon + category: stance_marker + subcategory: embarrassment + lexical_items: + - form: oof + gloss: amused impact or discomfort + sense: playful_recoil + register: casual_discord + allowed_speaker_standing: speaker_can_self_deprecate + forbidden_senses: [] + collocations: + - "oof, don't tell me" + activation: + allowed_when: + - compliment_or_trait_read_lands + forbidden_when: + - serious_apology_or_harm_context + requires_profile_authority: true + minimum_density: light + maximum_density: light + evidence: + source_refs: + - discord:1037314100931338260 + confidence: high + - feature_id: mock_challenge_coward + label: Mock challenge coward + status: draft + domain: + layer: lexicon + category: stance_marker + subcategory: friendly_roast + lexical_items: + - form: coward + gloss: mock insult + sense: friendly_status_challenge + register: casual_discord + allowed_speaker_standing: active_banter_with_known_participant + forbidden_senses: + - actual_contempt + collocations: + - "then you coward" + activation: + allowed_when: + - friendly_roast_context + - target_has_claimed_competence + forbidden_when: + - unknown_relationship + - serious_conflict + requires_profile_authority: true + minimum_density: light + maximum_density: light + evidence: + source_refs: + - discord:1023727427924545576 + confidence: high + - feature_id: preserve_self_indexed_technical_label + label: Preserve self-indexed technical label + status: draft + domain: + layer: lexicon + category: technical_term + subcategory: self_indexing + description: > + When Persona interlingua frames the speaker through a locally salient + technical identity label, lowering should preserve that label instead of + inventing a fresher metaphor. In this subculture, "engineer" can be a + joked-about identity index, not just an occupation. + lexical_items: + - form: engineer + gloss: technical identity / competence index + sense: self_indexed_engineering_vibe + register: casual_discord + allowed_speaker_standing: speaker_or_room_has_made_engineering_salient + forbidden_senses: + - formal_job_title_unless_context_requires + collocations: + - engineer vibes + - engineer-like + activation: + allowed_when: + - persona_interlingua_mentions_engineer_like_or_engineering_salience + - prior_turn_compliments_efficiency_or_structure + forbidden_when: + - interlingua_does_not_name_technical_identity + requires_profile_authority: true + minimum_density: light + maximum_density: light + constraints: + register_contexts: + - casual + speaker_contexts: + - metame_or_metacrat_fixture + grammar_contexts: + - playful_self_deprecation + incompatibilities: + - replacement_identity_metaphor + - creature_metaphor_substitution + evidence: + source_refs: + - discord:1037314100931338260 + confidence: medium + discourse_patterns: + - pattern_id: compact_recoil_no_context_dump + label: Compact recoil without context dump + status: draft + domain: + layer: discourse + category: stance_marker + subcategory: self_deprecation + interactional_function: answer_trait_read_with_short_mock_wince + sequence_shape: + - recoil_marker + - dont_tell_me_frame + - possessive_context_object + - preserved_trait_label + adjacency_constraints: + - prior_turn_already_contains_the_explanation + - generated_line_should_not_quote_or_summarize_prior_turns + activation: + allowed_when: + - playful_self_deprecation + - prior_turn_compliments_efficiency_or_structure + forbidden_when: + - source_intent_requires_explanation + requires_profile_authority: true + minimum_density: light + maximum_density: light + constraints: + register_contexts: + - casual + speaker_contexts: + - metame_or_metacrat_fixture + collocation_limits: + - target_length_roughly_6_to_12_words + - avoid_quoted_prior_turn_paraphrase + - avoid_because_clause + - avoid_disease_metaphor_unless_interlingua_requests_it + evidence: + source_refs: + - discord:1037314100931338260 + confidence: medium + - pattern_id: casual_floor_yield_with_lol + label: Casual floor yield with lol + status: draft + domain: + layer: discourse + category: turn_taking + subcategory: floor_yield + interactional_function: yield_floor_after_riff + sequence_shape: + - discourse_pivot_anyway + - self_silencing_imma_shut_up_now + - invite_others_to_invent + - lol_softener + adjacency_constraints: + - prior_turns_show_speaker_has_been_riffing + - other_participants_have_interest_or_pending_contribution + activation: + allowed_when: + - conversational_yield + - casual_collaborative_ideation + forbidden_when: + - formal_moderation + requires_profile_authority: true + minimum_density: light + maximum_density: light + evidence: + source_refs: + - discord:820077384572338176 + confidence: high + orthography_features: + - feature_id: light_discord_informality + label: Light Discord informality + status: draft + domain: + layer: orthography + category: punctuation + subcategory: informal_case + allowed_surface_forms: + - lower_case_opening + - markdown_italics_for_single_emphasis + - no_terminal_period + - imma + - lol + forbidden_surface_forms: + - heavy_eye_dialect_for_identity_caricature + - assistant_like_sentence_polishing + activation: + allowed_when: + - casual_discord_register + forbidden_when: + - formal_or_public_document_register + requires_profile_authority: true + minimum_density: light + maximum_density: normal + evidence: + source_refs: + - discord:1037314100931338260 + - discord:820077384572338176 + confidence: medium + code_switching: + allowed: false + constraints: + - This subculture alone does not authorize cross-language switching. + patterns: [] +evidence_ledger: + - ref: discord:1037314100931338260 + summary: Self-deprecating recoil after efficiency compliment. + - ref: discord:802911591288537118 + summary: Fragmentary absurd comparison during sound/game design banter. + - ref: discord:1023727427924545576 + summary: Very short mock-aggressive challenge in friendly roast context. + - ref: discord:820077384572338176 + summary: Casual profane floor-yield after extended ideation. diff --git a/data/entity/entity.cc b/data/entity/entity.cc deleted file mode 100644 index 8f9a198..0000000 --- a/data/entity/entity.cc +++ /dev/null @@ -1 +0,0 @@ -“²cultcache.store.v1‘—¬weksa.entity¬weksa.entity¯weksa.entity.v1¬weksa.entityÙL{"schemaName":"weksa.entity","schemaVersion":"weksa.entity.v1","members":[]}‘¬weksa.entity’”­entity:brokkr¬weksa.entity´2026-10-29T16:43:28ZÄn“­entity:brokkr‘’¥en-US‘¦Brokkr¤Cast’¦brokkrÙ?examples/persona-lowering/brokkr-edda-voice/lowered-output.yaml”«entity:nibu¬weksa.entity´2026-10-29T16:43:28ZÄe“«entity:nibu‘’¥en-US‘¤Nibu¤Cast’¤nibuÙ + Metacrat-authored casual Discord exchanges before 2025, especially + GameCult/Aquarium rooms where technical work, jokes, status play, and + collaborative ideation share the same channel. + historical_context: > + Pre-2025 room voice is less doctrine-heavy and less architectural than the + current Weksa/CultCache register. It tolerates messier banter, compact + jokes, self-deprecation, friendly mock aggression, and quick room-level + turn-taking. + identity_authority: > + This profile is a target-language lowering overlay for one subculture. Live + Persona state should reference a culture stack; current intent and memory + pressure live in Metame Persona state. + consent_notes: > + Use only for owner-authorized Metame/Metacrat fixture work. Do not generalize + to other people, cultures, dialects, or protected identities. +activation: + allowed_when: + - speaker_id_is_metame_or_metacrat_fixture + - target_period_is_pre_2025_casual_discord + - fixture_or_owner_authorized_history_target + forbidden_when: + - formal_current_project_doctrine_is_target + - target_speaker_is_not_metame_or_metacrat + - use_would_expose_private_persona_state + required_confidence: provisional + reviewer_required_when: + - adding_new_public_history_evidence + - using_profile_outside_fixture_harness +salience_axes: + register_gravity: + axis_id: register_gravity + domain: stance + description: > + Whether the line should stay in casual room banter instead of being pulled + into architecture, doctrine, or explanatory assistant prose. + relevant_distinctions: + - distinction_id: casual_room_temperature + label: Casual room temperature + meaning: Compact, socially alive wording that assumes shared room context. + lowering_questions: + - Can the utterance be one sentence or fragment? + - Does explanation weaken the social move? + required_when: + - fixture_style_tags_include_casual + forbidden_when: + - formal_review_or_doctrine_is_requested + evidence_refs: + - discord:1037314100931338260 + - discord:802911591288537118 + - discord:1023727427924545576 + - discord:820077384572338176 + self_indexed_technical_identity: + axis_id: self_indexed_technical_identity + domain: identity_indexing + description: > + If Persona interlingua names a technical self-index such as engineer-like + identity, preserve that label unless the target language profile has a + stronger reason to paraphrase. + relevant_distinctions: + - distinction_id: preserve_engineer_label + label: Preserve engineer label + meaning: "Engineer" may carry the social joke and should not be replaced by a novel metaphor. + lowering_questions: + - Did Persona interlingua mention engineer-like or engineering salience? + - Is the prior compliment about efficiency, structure, or technical order? + required_when: + - persona_interlingua_mentions_engineer_like_or_engineering_salience + forbidden_when: + - interlingua_does_not_name_technical_identity + evidence_refs: + - discord:1037314100931338260 + context_compression: + axis_id: context_compression + domain: discourse_patterns + description: > + Prior-turn content should usually compress into a short object label or + stance marker, not be quoted back inside the utterance. + relevant_distinctions: + - distinction_id: no_context_dump_in_banter + label: No context dump in banter + meaning: Casual recoil should not summarize the setup that the room just saw. + lowering_questions: + - Is the prior context already visible to the addressee? + - Can the setup compress into "my discord", "this thing", or a short noun phrase? + required_when: + - casual_discord_register + - playful_self_deprecation + forbidden_when: + - source_intent_requires_explanation + evidence_refs: + - discord:1037314100931338260 + status_play: + axis_id: status_play + domain: social_relation + description: > + Friendly teasing can take a sharper surface form than a neutral assistant + would choose, provided the local context already supports play. + relevant_distinctions: + - distinction_id: mock_aggression + label: Mock aggression + meaning: Imperative or insult-shaped challenge used as friendly banter. + lowering_questions: + - Is the addressee already joking or status-playing? + - Would softening erase the bit? + required_when: + - communicative_act_is_playful_challenge + forbidden_when: + - relationship_context_is_unknown_or_hostile + evidence_refs: + - discord:1023727427924545576 + competence_recoil: + axis_id: competence_recoil + domain: affect + description: > + Praise for competence, efficiency, or engineering can be answered with + joking embarrassment rather than straightforward acceptance. + relevant_distinctions: + - distinction_id: self_deprecating_recoil + label: Self-deprecating recoil + meaning: "oof" plus mock concern that a trait is showing too clearly. + lowering_questions: + - Is the prior turn a compliment or recognition of technical order? + - Can the line admit the trait without fully accepting the label? + required_when: + - communicative_act_is_playful_self_deprecation + forbidden_when: + - source_intent_requires_gratitude_or_serious_acceptance + evidence_refs: + - discord:1037314100931338260 + ideation_turn_taking: + axis_id: ideation_turn_taking + domain: discourse_patterns + description: > + After riffing, the speaker may explicitly yield the floor in profane, + casual language instead of formal facilitation. + relevant_distinctions: + - distinction_id: casual_floor_yield + label: Casual floor yield + meaning: Stop dominating the thread and invite others to invent. + lowering_questions: + - Has the speaker been riffing at length? + - Would a casual "lol" preserve warmth and reduce authority? + required_when: + - communicative_act_is_conversational_yield + forbidden_when: + - scene_requires_formal_moderation + evidence_refs: + - discord:820077384572338176 +linguistic_affordances: + grammar_features: + - feature_id: fragmentary_like_comparison + label: Fragmentary Like comparison + status: draft + domain: + layer: grammar + category: comparison + subcategory: fragment + description: > + A joke-description may start with "Like..." and remain fragmentary when + it is an immediate continuation of collaborative design talk. + formal_conditions: + syntactic_environment: + - adjacency_pair_response + - elliptical_clause + semantic_inputs: + - comic_description + - absurd_comparison + required_interlingua_fields: + - discourse.speech_act + - predications + realization: + function: casual_comic_comparison + surface_strategy: "Like a really terrible that " + examples: + - "Like a really terrible engine that sounds like coconuts when you use it" + activation: + allowed_when: + - collaborative_design_banter + forbidden_when: + - standalone_formal_explanation + requires_profile_authority: true + minimum_density: light + maximum_density: normal + constraints: + grammar_contexts: + - previous_turn_supplies_topic + register_contexts: + - casual + speaker_contexts: + - metame_or_metacrat_fixture + collocation_limits: [] + incompatibilities: + - overexplained_punchline + indexicality: + social_meanings: + - improvisational + - collaborative + - unserious_but_useful + in_group_meanings: + - game_design_room_banter + out_group_risks: + - may seem incomplete without prior context + stereotypes_to_avoid: [] + evidence: + source_refs: + - discord:802911591288537118 + qdrant_source_ids: + - "802911591288537118" + cached_source_refs: [] + source_hashes: [] + reviewer_refs: [] + confidence: medium + uncertainty: + open_questions: + - How often should fragmentary openings appear outside sound/design talk? + known_limits: + - Single fixture line is evidence for an affordance, not a universal rule. + trace: + trace_label: fragmentary_like_comparison + must_report_when_used: true + must_report_when_rejected: true + phonology_features: [] + lexicon_features: + - feature_id: casual_recoil_oof + label: Casual recoil marker oof + status: draft + domain: + layer: lexicon + category: stance_marker + subcategory: embarrassment + description: > + "oof" can mark amused discomfort when a social read lands too accurately. + lexical_items: + - form: oof + gloss: amused impact or discomfort + sense: playful_recoil + register: casual_discord + allowed_speaker_standing: speaker_can_self_deprecate + forbidden_senses: [] + collocations: + - "oof, don't tell me" + realization: + function: self_deprecating_recoil + surface_strategy: start line with "oof" when prior turn praises a trait the speaker wants to half-resist + examples: + - "oof, don't tell me my discord has *engineer* vibes" + activation: + allowed_when: + - compliment_or_trait_read_lands + forbidden_when: + - serious_apology_or_harm_context + requires_profile_authority: true + minimum_density: light + maximum_density: light + constraints: + register_contexts: + - casual + speaker_contexts: + - metame_or_metacrat_fixture + grammar_contexts: [] + phonological_contexts: [] + collocation_limits: + - avoid_repeating_if_already_in_recent_context + incompatibilities: [] + indexicality: + social_meanings: + - amused_embarrassment + - competence_recoil + in_group_meanings: [] + out_group_risks: + - can sound too internet-generic if unsupported by context + stereotypes_to_avoid: [] + evidence: + source_refs: + - discord:1037314100931338260 + qdrant_source_ids: + - "1037314100931338260" + cached_source_refs: [] + source_hashes: [] + reviewer_refs: [] + confidence: high + uncertainty: + open_questions: [] + known_limits: + - Marker frequency should be sparse. + trace: + trace_label: casual_recoil_oof + must_report_when_used: true + must_report_when_rejected: true + - feature_id: mock_challenge_coward + label: Mock challenge coward + status: draft + domain: + layer: lexicon + category: stance_marker + subcategory: friendly_roast + description: > + "coward" can close a very short challenge when surrounding turns make + the aggression visibly playful. + lexical_items: + - form: coward + gloss: mock insult + sense: friendly_status_challenge + register: casual_discord + allowed_speaker_standing: active_banter_with_known_participant + forbidden_senses: + - actual_contempt + collocations: + - "then you coward" + realization: + function: playful_status_challenge + surface_strategy: imperative challenge plus "then you coward" + examples: + - "Engineer me something then you coward" + activation: + allowed_when: + - friendly_roast_context + - target_has_claimed_competence + forbidden_when: + - unknown_relationship + - serious_conflict + requires_profile_authority: true + minimum_density: light + maximum_density: light + constraints: + register_contexts: + - casual + speaker_contexts: + - metame_or_metacrat_fixture + grammar_contexts: + - imperative_challenge + phonological_contexts: [] + collocation_limits: [] + incompatibilities: + - assistant_safety_softening_that_erases_banter + indexicality: + social_meanings: + - teasing + - status_play + - friendly_pressure + in_group_meanings: [] + out_group_risks: + - may be hostile if context is not playful + stereotypes_to_avoid: [] + evidence: + source_refs: + - discord:1023727427924545576 + qdrant_source_ids: + - "1023727427924545576" + cached_source_refs: [] + source_hashes: [] + reviewer_refs: [] + confidence: high + uncertainty: + open_questions: [] + known_limits: + - Only safe inside clear banter. + trace: + trace_label: mock_challenge_coward + must_report_when_used: true + must_report_when_rejected: true + discourse_patterns: + - pattern_id: casual_floor_yield_with_lol + label: Casual floor yield with lol + status: draft + domain: + layer: discourse + category: turn_taking + subcategory: floor_yield + description: > + After extended riffing, the speaker can step back with "anyway imma + shut up now..." plus "lol", reducing authority while keeping warmth. + interactional_function: yield_floor_after_riff + sequence_shape: + - discourse_pivot_anyway + - self_silencing_imma_shut_up_now + - invite_others_to_invent + - lol_softener + adjacency_constraints: + - prior_turns_show_speaker_has_been_riffing + - other_participants_have_interest_or_pending_contribution + realization: + function: collaborative_turn_release + surface_strategy: "anyway imma shut up now and let you guys come up with your own shit lol" + examples: + - "anyway imma shut up now and let you guys come up with your own shit lol" + activation: + allowed_when: + - conversational_yield + - casual_collaborative_ideation + forbidden_when: + - formal_moderation + requires_profile_authority: true + minimum_density: light + maximum_density: light + constraints: + register_contexts: + - casual + speaker_contexts: + - metame_or_metacrat_fixture + grammar_contexts: [] + phonological_contexts: [] + collocation_limits: [] + incompatibilities: [] + indexicality: + social_meanings: + - self_limiting + - collaborative + - profane_warmth + in_group_meanings: [] + out_group_risks: + - profanity may be inappropriate outside familiar room + stereotypes_to_avoid: [] + evidence: + source_refs: + - discord:820077384572338176 + qdrant_source_ids: + - "820077384572338176" + cached_source_refs: [] + source_hashes: [] + reviewer_refs: [] + confidence: high + uncertainty: + open_questions: [] + known_limits: + - Not a generic moderation pattern. + trace: + trace_label: casual_floor_yield_with_lol + must_report_when_used: true + must_report_when_rejected: true + orthography_features: + - feature_id: light_discord_informality + label: Light Discord informality + status: draft + domain: + layer: orthography + category: punctuation + subcategory: informal_case + description: > + Casual lines may use lower-case openings, sparse punctuation, emphasis + with markdown italics, and uncorrected informal contractions when the + social move benefits. + allowed_surface_forms: + - lower_case_opening + - markdown_italics_for_single_emphasis + - no_terminal_period + - imma + - lol + forbidden_surface_forms: + - heavy_eye_dialect_for_identity_caricature + - assistant_like_sentence_polishing + renderer_notes: > + Preserve compactness. Do not add terminal politeness, explanation, or + moral framing unless source meaning requires it. + realization: + function: casual_discord_texture + surface_strategy: allow sparse punctuation and compact informal markers + examples: + - "oof, don't tell me my discord has *engineer* vibes" + activation: + allowed_when: + - casual_discord_register + forbidden_when: + - formal_or_public_document_register + requires_profile_authority: true + minimum_density: light + maximum_density: normal + constraints: + register_contexts: + - casual + speaker_contexts: + - metame_or_metacrat_fixture + grammar_contexts: [] + phonological_contexts: [] + collocation_limits: [] + incompatibilities: [] + indexicality: + social_meanings: + - low_ceremony + - room_familiarity + in_group_meanings: [] + out_group_risks: + - overuse becomes imitation instead of lowering + stereotypes_to_avoid: + - do_not_represent_protected_dialect_by_eye_dialect + evidence: + source_refs: + - discord:1037314100931338260 + - discord:820077384572338176 + qdrant_source_ids: + - "1037314100931338260" + - "820077384572338176" + cached_source_refs: [] + source_hashes: [] + reviewer_refs: [] + confidence: medium + uncertainty: + open_questions: + - Need more examples to estimate density safely. + known_limits: [] + trace: + trace_label: light_discord_informality + must_report_when_used: true + must_report_when_rejected: true + code_switching: + allowed: false + constraints: + - This profile does not authorize cross-language switching. + patterns: [] +forbidden_flattenings: + - Do not ask Metame to imitate the final utterance directly; Persona produces interlingua intent. + - Do not store Metacrat biographical/cultural strata here; store reusable strata as separate culture/subculture profiles and reference them from Persona state. + - Do not let current machine-cult architecture register overwrite pre-2025 casual fixture style. + - Do not treat profanity or insults as safe without relationship/context evidence. + - Do not copy private Persona state into Weksa fixtures or run artifacts. +evidence_ledger: + - ref: discord:1037314100931338260 + summary: Self-deprecating recoil after efficiency compliment. + - ref: discord:802911591288537118 + summary: Fragmentary absurd comparison during sound/game design banter. + - ref: discord:1023727427924545576 + summary: Very short mock-aggressive challenge in friendly roast context. + - ref: discord:820077384572338176 + summary: Casual profane floor-yield after extended ideation. + - ref: discord:870907292558721085 + summary: Explicit joke about being bad at self-deprecation. + - ref: discord:1020888211729432728 + summary: Positive uptake when another participant joins cult jokes. diff --git a/docs/architecture.md b/docs/architecture.md index 56f201c..db24c43 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -1,8 +1,7 @@ # Architecture -Weksa is a language-intent engine: currently a state spine, pipeline map, -fixtures, ontology data and a planned service contract. No daemon runs; the -planned packet crate is `crates/weksa` (cut `packet-store`). +Weksa is the language-intent daemon surface: currently a state spine, pipeline +map, fixtures, and service contract waiting for its runtime loop. See [Verse Service Contract](./verse-service-contract.md) for the CultCache/CultMesh authority map. @@ -10,15 +9,15 @@ CultCache/CultMesh authority map. ## Objective Build a procedural alien-language engine whose outputs are grounded in -worldview, semantics, grammar, phonology, morphology, and history. Once a service is -earned (planned), Weksa also turns writing plus agent state into typed conversational intent, then +worldview, semantics, grammar, phonology, morphology, and history. As a daemon, +Weksa also turns writing plus agent state into typed conversational intent, then lowers that intent into written realizations, pronunciation plans, speech-provider requests, and utterance handoff documents. ## Current Mechanism There is no long-running runtime loop yet. The repo contains persistent state -surfaces that define the intended data flow +surfaces and a provider advertisement fixture that define the intended data flow and the ownership boundaries future code must respect. The first shared contract is the [Interlingua Standard](./interlingua-standard.md): diff --git a/docs/fixtures/weksa-provider-advertisement.json b/docs/fixtures/weksa-provider-advertisement.json new file mode 100644 index 0000000..e2b03ba --- /dev/null +++ b/docs/fixtures/weksa-provider-advertisement.json @@ -0,0 +1,166 @@ +{ + "schema": "gamecult.eve.provider_advertisement.v1", + "providerId": "weksa.intent.service", + "title": "Weksa Intent Service", + "description": "Weksa converts writing, interlingua, and agent state into typed conversational intent, pronunciation plans, and utterance handoff documents for AquaSynth and other consumers.", + "mode": "fixture-read-only", + "serviceVerse": { + "id": "verse.weksa.service.v1", + "cultMeshKey": "weksa.service/provider-advertisement", + "authority": "Weksa daemon identity, schema catalog, and provider publication", + "owns": [ + "typed conversational intent", + "pronunciation plans", + "utterance handoff documents", + "lowering traces", + "language-project and flavor-profile state" + ], + "doesNotOwn": [ + "AquaSynth learned embeddings", + "Faust compilation", + "rendered audio", + "Persona canonical state" + ] + }, + "nestedVerses": [ + { + "id": "weksa.intent", + "kind": "intent", + "cultMeshKey": "weksa.intent/current", + "authority": "Weksa conversational intent compiler", + "schemas": [ + "weksa.conversational_intent.v0" + ] + }, + { + "id": "weksa.pronunciation", + "kind": "pronunciation", + "cultMeshKey": "weksa.pronunciation/current", + "authority": "Weksa pronunciation lowering", + "schemas": [ + "weksa.pronunciation_plan.v0" + ] + }, + { + "id": "weksa.utterance", + "kind": "utterance", + "cultMeshKey": "weksa.utterance/handoffs", + "authority": "Weksa utterance handoff export", + "schemas": [ + "weksa.utterance_handoff.v0", + "weksa.utterance_embedding_handoff.v0.1" + ] + }, + { + "id": "weksa.operator", + "kind": "operator", + "cultMeshKey": "weksa.operator/status", + "authority": "Weksa daemon runtime", + "schemas": [ + "weksa.operator_state.v0" + ] + }, + { + "id": "weksa.cultural_ontology", + "kind": "cultural_ontology", + "cultMeshKey": "weksa.cultural-ontology/catalogs", + "authority": "Weksa cultural ontology profile store", + "schemas": [ + "weksa.cultural_ontology_profile.v0", + "weksa.cultural_ontology_catalog.v0" + ] + } + ], + "schemaWitnessFamilies": [ + { + "family": "intent", + "namespace": "weksa.intent.*", + "witnesses": [ + "weksa.conversational_intent.v0" + ], + "durableShape": ".weksa/intent/{intentId}.cc" + }, + { + "family": "pronunciation", + "namespace": "weksa.pronunciation.*", + "witnesses": [ + "weksa.pronunciation_plan.v0" + ], + "durableShape": ".weksa/pronunciation/{utteranceId}.cc" + }, + { + "family": "utterance", + "namespace": "weksa.utterance.*", + "witnesses": [ + "weksa.utterance_handoff.v0", + "weksa.utterance_embedding_handoff.v0.1" + ], + "durableShape": ".weksa/handoff/{utteranceId}.cc" + }, + { + "family": "trace", + "namespace": "weksa.trace.*", + "witnesses": [ + "weksa.lowering_trace.v0" + ], + "durableShape": ".weksa/traces/{traceId}.cc" + }, + { + "family": "cultural_ontology", + "namespace": "weksa.cultural_ontology.*", + "witnesses": [ + "weksa.cultural_ontology_profile.v0", + "weksa.cultural_ontology_catalog.v0" + ], + "durableShape": ".weksa/cultural-ontology/{storeId}.cc" + } + ], + "commandBoundaries": [ + { + "id": "weksa.command.intent.create", + "owner": "Weksa", + "acceptedByFixture": false, + "allowedInputs": [ + "writing", + "interlingua packet", + "Persona/agent state projection" + ], + "emits": [ + "weksa.conversational_intent.v0", + "weksa.lowering_trace.v0" + ] + }, + { + "id": "weksa.command.utterance.lower", + "owner": "Weksa", + "acceptedByFixture": false, + "allowedInputs": [ + "accepted conversational intent", + "language project", + "speaker state" + ], + "emits": [ + "weksa.pronunciation_plan.v0", + "weksa.utterance_handoff.v0" + ] + }, + { + "id": "weksa.command.handoff.export", + "owner": "Weksa", + "acceptedByFixture": false, + "emits": [ + "weksa.utterance_embedding_handoff.v0.1" + ], + "forbiddenTargets": [ + "Faust compilation", + "audio rendering", + "AquaSynth synth driver truth" + ] + } + ], + "loweringRules": [ + "Weksa owns conversational intent and utterance lowering.", + "AquaSynth owns learned embeddings, synth controls, Faust compilation, and audio output.", + "Fixture exports are discovery metadata until the daemon loop commits typed witnesses." + ] +} diff --git a/docs/interlingua-standard.md b/docs/interlingua-standard.md index 8b36983..965e9b0 100644 --- a/docs/interlingua-standard.md +++ b/docs/interlingua-standard.md @@ -1,6 +1,6 @@ # Interlingua Standard -Version: `0.2-draft` +Version: `0.1-draft` The Weksa interlingua is the language-neutral meaning contract between clients, setting surfaces, authoring tools, language projects, ontology projection, and @@ -82,174 +82,235 @@ Those belong downstream. ## Packet Shape -The Rust types in [`crates/weksa/src/packet.rs`](../crates/weksa/src/packet.rs) -are the definition; the JSON Schemas in [`schemas/cultnet/`](../schemas/cultnet/) -are generated from them and a test fails when they drift. A packet is the -CultCache document `weksa.interlingua_packet`. This section describes the shape; -where it and the types disagree, the types win. +The canonical packet is an object with these top-level families: -Two layers carry the standard. The **schema** is the Rust types: the packet -families, node ids, slots and value kinds. It changes by commit and rarely. The -**inventory** is data: the concepts, frames, roles, relation types, speech acts -and features that the schema's slots name. It is stored as -`weksa.inventory_entry` documents under appended `weksa.inventory_revision` -records, never edited in place (a changed meaning is a retire plus an add), and -a retired id still resolves for packets of earlier revisions. - -Every id is an opaque form type, parsed when it is built and when a store -reads it: an inventory id is `:` with `n` a canonical decimal of at -least 1 (`role:1`; `role:01` is not an id), a node id is `n` with `k` in -1..=9999, a packet id is `packet:<16 hex>` and an encoder id is -`encoder:<16 hex>`, an entity id is `entity:` (`[a-z0-9]` and inner -`-`, 1 to 48 bytes) and a corpus item id is `item::`. The only free -text a packet holds is the `trace` (`notes` and `rejected_readings`), which the -validator neither bounds nor reads and which is stripped before lowering: a -referent names its entity by id, and the entity store (`weksa.entity`) holds -the entity's forms per language. The packet store refuses a referent whose -entity the store lacks, a packet whose `corpus_item` the corpus lacks, and one -whose entity is neither named on that item nor its speaker or a prior-turn -speaker (a packet without a corpus item has no door in this store): -an id-set check, not a text rule. Leakage of scored text is owned outside the -validator, by the loop's context builder (what each model call sees) and the -scorer; the validator owns form and capacity. The trace is stripped before -lowering. Concept ids are opaque and carry a gloss in the inventory; concepts -are open-class and grow, while the other families are closed-class. - -```text -Packet - id, schema_version, inventory_rev, kind - provenance { source, corpus_item, encoder, confidence } - context { setting, medium, participants, audience } - discourse { speech_act, addressee, features } - referents [ { id, kind, concept, entity, features } ] - predications [ { id, concept, frame, roles: [ { role, filler } ], features } ] - relations [ { relation, from, to, features } ] - constraints { preserve, deterministic, max_latency_ms } - extensions [ { key: InvId (family extension), value: Grade | Option | Flag } ] - trace { notes, rejected_readings } - stance { features } +```yaml +interlingua_version: +packet_id: +kind: +provenance: +context: +discourse: +referents: +predications: +relations: +constraints: +extensions: +trace: ``` -`schema_version` and `inventory_rev` name the exact schema and inventory -revision a packet was written against. A grade is an integer permille, -`0..=1000`, and the published schema says so. There are no floats. +Only `interlingua_version`, `packet_id`, `kind`, and at least one semantic +payload family are required. The rest should be present when known. -### Validation - -A packet exists only if the validator's rules pass and the packet store admits -it. `validate` judges and returns `Ok(())` or the refusals; the packet store's -`admit` is the one door and the only source of the lowering view. A refusal -names the rule. - -- V1: `schema_version` is the version these types define. -- V2: at least one of `referents`, `predications`, `relations` is present. -- V3: a node id appears once across referents and predications. -- V4: every role filler, relation end, addressee, participant, audience member - and preserved node names a node. -- V5: every inventory id resolves at the packet's `inventory_rev`: introduced - at or before it, and not retired at or before it. -- V6: an inventory id is of the family its slot takes. -- V7: a feature or extension value fits its entry's value kind (a grade is a - grade, an option is one the entry declares, a flag is a flag). -- V8: a feature sits only in a slot its entry declares. -- V10: `inventory_rev` has a revision record. -- V11: a feature id appears at most once in one slot's feature list, an - extension id at most once per packet, a node id at most once in each of - `participants`, `audience` and `preserve`, and a role id at most once in one - predication's roles. -- V12: a packet has at most 64 referents, predications and relations together - (a calibration constant, capacity and not meaning). - -V9 is retired and not renumbered: the form of a key or a grade is its type, and -an extension key's family is V6. `PacketStore::admit` returns the lowering view of a -packet it admits; the view carries the meaning and the stance, no packet id, no -provenance and no trace, and is built nowhere else. The store also refuses a -packet that names a withdrawn corpus item (packets admitted before the -withdrawal stay stored). A packet is admitted against the inventory the packet store -reads at the packet's revision, never one the caller supplies. +## Top-Level Fields ### `kind` -`Utterance`, `NameRequest`, `CommentaryIntent`, `Narration`, `DialogueLine`, -`CorpusExample`, `DebugProbe`. +The packet's broad use case. + +Initial values: + +- `utterance` +- `name_request` +- `commentary_intent` +- `narration` +- `dialogue_line` +- `corpus_example` +- `debug_probe` ### `provenance` -`source` is `Authored`, `ModelDecomposition`, `GameTelemetry` or -`ImportedCorpus`. `corpus_item` is the required `item::` key of the -corpus item the packet was minted for, `encoder` names what wrote it, and `confidence` is a grade. +Where the packet came from. + +```yaml +provenance: + source_type: authored | model_decomposition | game_telemetry | imported_corpus + source_ref: + author: + created_at: + confidence: +``` ### `context` -`setting` and `medium` are concept ids. `participants` and `audience` are node -ids. +World and situation information that is not itself the main semantic claim. + +```yaml +context: + setting: + scene: + time: + location: + participants: + audience: + medium: +``` ### `discourse` -Why the packet is said: `speech_act` is a speech-act id, `addressee` a node id, -and `features` carry source-side nuance on content (hedge, certainty, irony and -whatever the inventory grows). This is source-side communicative pressure, not -target-language register. +Why this packet is being said, rendered, or named. -### `stance` +```yaml +discourse: + speech_act: + intent: + tone: + register_hint: + addressee: + audience_function: +``` -The speaker's felt affect and how much of it the speaker holds back. `features` -is a feature list in slot `Stance`, checked like every other (V5 to V8, V11). -The seed carries felt affect as grades of how strongly it is felt (anger, fear, sadness, -joy, disgust, surprise, contempt, affection, shame) and `restraint`, a grade for -how much of the felt affect is suppressed (0 none, 1000 fully). The stance is -what is felt, never what is displayed: how a feeling shows, and how much of a -restrained one leaks, are cultural, and each language's ontology maps felt -affect and restraint to its own register. A held-back feeling is not a claim -about the world, so it is never a predication. An empty stance is valid. The -stance reaches the lowerer in the lowering view. +This is not target-language register. It is source-side communicative pressure. ### `referents` -Entities, groups, places, abstractions and the rest. `kind` is a referent-kind -id (the inventory seeds `person`, `group`, `organism`, `place`, `artifact`, -`institution`, `ecology`, `route`, `memory`, `event`, `abstract`), `concept` is -a concept id, and `entity` is the id of the named entity when the referent has one. +Entities, groups, locations, abstract objects, institutions, ecological actors, +routes, memories, artifacts, and other nodes that can participate in meaning. + +```yaml +referents: + - id: + kind: + label: + ontology_hint: + animacy: + agency: + number: + definiteness: + salience: + attributes: + extensions: +``` + +Initial broad kinds: + +- `person` +- `group` +- `organism` +- `place` +- `artifact` +- `institution` +- `ecology` +- `route` +- `memory` +- `event` +- `abstract` ### `predications` -Events, states, processes and scenes. `concept` and `frame` are ids; `roles` -pair a role id with a filler node (the inventory seeds `agent`, `patient`, -`experiencer`, `instrument`, `source`, `goal`, `location`, `beneficiary`). Tense, -aspect, polarity, modality and evidentiality are features with declared option -sets, not fields. +Events, states, processes, properties, and frame-like scenes. + +```yaml +predications: + - id: + predicate: + frame: + aspect: + tense: + polarity: + modality: + evidentiality: + roles: + agent: + patient: + experiencer: + instrument: + source: + goal: + location: + beneficiary: + attributes: + extensions: +``` + +`predicate` is a broad source-side claim. `frame` is optional but recommended +when a scene model is known. Roles may use common names or namespaced frame +roles. ### `relations` -Edges not owned by one predication. `relation` is a relation id (the inventory -seeds `coreference`, `part_of`, `member_of`, `located_at`, `near`, `before`, -`after`, `causes`, `enables`, `prevents`, `owns`, `owes`, `knows`, `names`, -`represents`, `contrasts_with`), and `from` and `to` are node ids. +Edges that are not naturally owned by one predication. + +```yaml +relations: + - type: + from: + to: + attributes: + extensions: +``` + +Initial relation families: + +- `coreference` +- `part_of` +- `member_of` +- `located_at` +- `near` +- `before` +- `after` +- `causes` +- `enables` +- `prevents` +- `owns` +- `owes` +- `knows` +- `names` +- `represents` +- `contrasts_with` ### `constraints` -`preserve` lists nodes the lowering must keep, `deterministic` and -`max_latency_ms` bound the renderer. Requirements about which outputs a -renderer must or must not produce belong to the target profile, not the packet. +Hard requirements the downstream renderer must respect. + +```yaml +constraints: + preserve_referents: + preserve_relations: + required_outputs: + forbidden_outputs: + max_latency_ms: + deterministic: +``` + +These are engineering and content constraints, not ontology rules. ### `extensions` -Setting or client data. An extension key is an inventory entry of family -`extension` (`extension:`) whose entry declares the value kind, so a key is -an id and not a string. Extensions must not replace common fields when common -fields suffice, a language project may ignore them, and one that becomes -broadly required is promoted into the inventory or the schema. +Namespaced setting or client data. + +```yaml +extensions: + zyphos.memory_relation: + zyphos.boundary_permeability: + rtd.aesthetic_reading: + game.priority: +``` + +Extension rules: + +- Namespaces must be explicit. +- Extensions must not replace common fields when common fields are sufficient. +- A downstream language project may choose to ignore an extension. +- If an extension becomes broadly required across clients, promote it into the + standard in a later version. ### `trace` -Authoring and debug notes. The trace is stripped before lowering: the lowering -input type has no trace field. +Optional authoring/debug data. + +```yaml +trace: + notes: + decomposition_steps: + uncertainty: + rejected_readings: +``` + +Runtime may strip `trace` from compiled packets when latency or size matters. ## Minimal Examples -These examples illustrate meaning in the 0.1 shape, with free-text labels and -predicates. Those fields no longer exist; the examples are not valid packets. - ### Name Request ```yaml diff --git a/docs/mimo-tts-lowering.md b/docs/mimo-tts-lowering.md new file mode 100644 index 0000000..ff5ac67 --- /dev/null +++ b/docs/mimo-tts-lowering.md @@ -0,0 +1,158 @@ +# MiMo TTS Lowering + +Version: `0.1-draft` + +MiMo VoiceDesign is an expressive spoken-output target for Weksa interlingua +lowering. It is not the source of meaning, language choice, character state, or +delivery truth. + +```text +interlingua packet + -> projected speaker-local context + -> target-language utterance realization + -> Weksa delivery intent from the same projection + -> MiMo TTS VoiceDesign request projection + -> rendered audio artifact +``` + +MiMo sits beside written output and AquaSynth handoffs. Written realizations +prove that meaning, register, and cultural ontology survived lowering. +AquaSynth handoffs train or drive GameCult-owned synthesis. MiMo VoiceDesign +requests let an external provider speak a Weksa-authored utterance with a +generated voice derived from the same projected Persona state, delivery controls, +and trace that produced the utterance. + +The MiMo pass is choreography and provider flavor. It is not a second Persona +projection pass and it does not re-interpret raw Persona state after the line has +already been lowered. + +## Authority + +Weksa owns: + +- target-language spoken text selected from interlingua +- target-language register, script, cultural ontology, and trace +- projected speaker-local context used for the utterance +- speaker-local delivery pressure derived during utterance lowering +- provider-neutral prosody and emphasis intent +- the MiMo request projection as an export artifact + +MiMo owns: + +- generation of a provider voice from the requested voice design description +- interpretation of MiMo natural-language voice/style guidance +- interpretation of MiMo inline audio/style tags +- rendered audio bytes or streaming audio chunks + +MiMo does not own: + +- source meaning +- target-language cultural ontology +- canonical speaker or Persona state +- fresh Persona-state interpretation +- Weksa pronunciation plans +- AquaSynth learned embeddings, synth controls, or audio truth + +## Projection Rule + +The MiMo request must be derived from an accepted target-language realization and +the same projected speaker-local context that created it. It must not lower +directly from English unless English is the selected target language for that +realization. It must not read raw Persona state to discover a different voice +after Weksa has already chosen the utterance. + +Natural-language voice design and style guidance belongs in the MiMo `user` +message. The spoken target-language text belongs in the MiMo `assistant` +message. Inline MiMo tags may appear in the assistant content only when they are +derived from Weksa delivery intent or explicit target-language realization notes. + +## Request Shape + +```yaml +schema_version: weksa.mimo_tts_request.v0 +source_realization_ref: +speaker_agent_id: +provider: + id: xiaomi-mimo + model: mimo-v2.5-tts-voicedesign +target_language: + code: + locale: + script: + register: +performance_register: + label: + medium: + delivery_archetype: +voice_design: + description: + source_controls: + projected_context_refs: + forbidden_traits: +messages: + user_voice_design_instruction: + assistant_spoken_content: +audio: + format: + optimize_text_preview: + streaming: +trace: + source_fields: + delivery_controls_used: + provider_tags_used: + uncertainties: +``` + +## Delivery Mapping + +`target_language.register` is the linguistic/social register of the accepted +realization. It is not a provider style knob. For MiMo, Weksa should derive an +explicit `performance_register` from the same projection so speech choreography +can say "anime visual-novel heroine warning" without pretending English itself +has that register. + +Weksa should map the utterance's existing projected Persona pressure into +provider-neutral delivery labels before emitting MiMo-specific instructions: + +- pace, pause pressure, and urgency +- pitch contour and pitch range +- loudness and attack +- articulation precision +- warmth, threat, tenderness, contempt, play, fatigue, secrecy, formality +- span-level emphasis and pauses + +The MiMo projection may turn those labels into a concise VoiceDesign prompt or +inline tags, but the trace must preserve the Weksa-owned source controls. + +Persona presentation details such as youth-coded, loli-coded, product-voice, or +other culturally loaded styling may enter the MiMo request only when they are +already present in the projected context or accepted delivery intent. They should +be described as presentation/cultural coding and kept subordinate to agency, +personality, affect, and scene purpose. They must not become sexualized content +or a generic anime voice pasted over the line. + +For Weksa-authored dialogue, `optimize_text_preview` should normally be `false` +or omitted. MiMo should speak the accepted target-language realization, not +rewrite it. Text polishing is only allowed for explicit preview/prototyping +passes that are marked non-authoritative. + +## Language Rule + +Weksa should be able to write and speak in any target language whose utterance +realization it can justify. MiMo output for Brazilian Portuguese should consume +the accepted `pt-BR` spoken text. Japanese output should consume accepted +Japanese text. Alien-language output should consume accepted alien rendered +speech or phonetic text only after the language project owns that surface. + +Backtranslations are reviewer aids. They must not become MiMo spoken content. + +## First Proof + +The first proof should use the Nibu warning packet already used by multilingual +lowering: + +- emit the accepted English, `pt-BR`, or Japanese realization +- derive a provider-neutral delivery intent from Nibu's projected context +- project that into a MiMo `mimo-v2.5-tts-voicedesign` request +- verify that the request speaks the target text, not the backtranslation +- verify that style guidance is traceable to Weksa delivery controls diff --git a/docs/multilingual-utterance-lowering.md b/docs/multilingual-utterance-lowering.md index fb1f22a..54e1392 100644 --- a/docs/multilingual-utterance-lowering.md +++ b/docs/multilingual-utterance-lowering.md @@ -180,10 +180,9 @@ meaning sources. A provider request must consume the target-language `spoken_text` that Weksa already accepted and must carry a trace back to the interlingua packet, projected speaker context, and target-language profile. -MiMo V2.5 TTS is a candidate external provider target; its lowering path is -parked at tag `parked/weksa-node-body`. Its natural-language style +MiMo V2.5 TTS is the first external provider target. Its natural-language style instruction is a projection of Weksa delivery intent, and its assistant content -is the selected target-language spoken text. +is the selected target-language spoken text. See [MiMo TTS Lowering](./mimo-tts-lowering.md). AquaSynth remains the GameCult-owned synthesis path for learned utterance embeddings and synth-control work. MiMo and AquaSynth are sibling target diff --git a/docs/persona-finetuning-dataset-exploration.md b/docs/persona-finetuning-dataset-exploration.md new file mode 100644 index 0000000..ed67b40 --- /dev/null +++ b/docs/persona-finetuning-dataset-exploration.md @@ -0,0 +1,212 @@ +# Persona Fine-Tuning Dataset Exploration + +This note profiles the Discord archive as a candidate source for fine-tuning the +Metame/Metacrat lowering pipeline. + +## Pipeline Target + +The live architecture should not fine-tune a model to directly impersonate from +raw chat history. + +The training shape should stay aligned with Weksa: + +1. Persona state and current situation produce interlingua intent. +2. Persona state references a culture/subculture stack. +3. Weksa lowers interlingua through that culture stack into a flavored + utterance. + +Useful datasets therefore split into two families: + +- Persona intent extraction: current context plus Persona state projection to + interlingua intent. +- Weksa lowering: interlingua plus culture stack to actual utterance. + +Direct `context -> utterance` rows can be used as a baseline or critic set, but +they should not become the canonical architecture. + +## Corpus Snapshot + +Profiled archive: + +`E:/Projects/VoidBot/.voidbot/rag/messages.json` + +Author: + +`113785782975594501` + +Cutoff: + +`2025-01-01T00:00:00Z` + +Current aggregate counts: + +- archive messages: 40,657 +- usable text messages: 39,508 +- authored Metacrat messages: 19,385 +- authored Metacrat messages before 2025: 18,049 +- crude pre-2025 casual candidates: 16,499 +- pre-2025 authored messages with recent same-channel context: 16,530 +- adjacent reply-to-other rows: 7,894 +- adjacent self-continuation rows: 8,636 + +Length distribution for authored Metacrat messages: + +- p10: 14 chars +- p25: 30 chars +- median: 56 chars +- p75: 101 chars +- p90: 166 chars + +Adjacent candidate length classes: + +- short: 11,430 +- medium: 4,535 +- long: 565 + +The corpus is large enough for held-out evaluation splits. It is also uneven: +2020 and 2021 dominate the pre-2025 archive. + +## Dominant Channels + +Top adjacent candidate channels: + +- `general`: 6,286 +- `development`: 2,911 +- `narrative`: 2,169 +- `management`: 1,103 +- `website`: 753 +- `soundtrack`: 437 +- `gamedesign`: 433 +- `programming`: 408 +- `prose`: 320 +- `politics`: 291 +- `pics`: 273 +- `advertising`: 115 + +These should become strata, not one blended dataset. A single mixed fine-tune +will learn that legal explanations, horny little design jokes, political +argument, and casual encouragement all belong in the same response slot. That +is how the soup gets teeth. + +## Candidate Row Types + +For Weksa lowering: + +```yaml +schema_version: weksa.finetune.lowering_row.v0 +row_id: +split: +target_message_id: +target_text: +interlingua_intent: +culture_stack_refs: [] +target_overlay_refs: [] +context_window: + before: [] + after: [] +labels: + channel: + year: + length_class: + speech_act: + register: + reply_to_other: +``` + +For Persona intent extraction: + +```yaml +schema_version: weksa.finetune.persona_interlingua_row.v0 +row_id: +split: +persona_state_ref: +projected_context_ref: +current_situation: +target_message_id: +derived_interlingua: +derivation: + reviewed: + model: + evidence_refs: [] +``` + +The target utterance should be hidden from the Persona-intent generator during +inference. For training, target utterance may be used to derive a reviewed +interlingua label, but that label must be marked as derived from answer-key +surface evidence. + +## Split Policy + +Recommended first split: + +- train: 2018-2021 +- validation: early/mid 2022 +- test: late 2022 plus selected hand-reviewed fixtures +- quarantine: 2023-2026 until the current post-2025 voice target is deliberate + +Also reserve channel-level holdouts: + +- one casual/social channel +- one technical channel +- one political/management channel + +This catches overfitting to one room's register. + +## Filters + +Start with conservative row filters: + +- target author is Metacrat +- target timestamp before 2025 +- target is not deleted +- target is not a bot prompt +- target text length between 3 and 220 chars for casual lowering +- same-channel prior context exists within 60 minutes +- exclude pure URLs, code blocks, and attachment-only rows +- retain but label profanity, mentions, quotes, and markdown emphasis + +Do not discard long messages globally. Keep them in a separate explanatory-prose +slice. + +## Next Work + +1. Build an exporter that writes JSONL rows without leaking private Persona + projection into git. +2. Add labelers for speech act, register, channel stratum, and culture-stack + refs. +3. Use a model-assisted pass to derive interlingua labels from held-out target + utterances, then sample-review before training. +4. Fine-tune or train adapters separately for Persona intent extraction and + Weksa lowering. +5. Keep a frozen golf set where answer-key utterances are redacted from all + culture/profile prompt material. + +Reproduce the aggregate profile: + +```powershell +node scripts/profile-metame-finetune-dataset.mjs +``` + +## Swarm Exploration + +Use the swarm explorer to chase high-loss moments instead of uniform random +examples: + +```powershell +node scripts/run-metame-swarm-explorer.mjs --agents 4 --seed metame-scout +node scripts/run-metame-swarm-explorer.mjs --real --agents 4 --seed metame-scout +``` + +The default mode is a dry run. Real mode runs Codex workers: + +1. select one timeline candidate per agent using underexplored strata, feature + novelty, prior loss, and seeded jitter +2. run each Face normally against its assigned scenario +3. run an Interpreter pass that converts the Face output to Weksa interlingua +4. lower that interlingua with Weksa and score against the hidden utterance +5. run an optimizer pass over high-loss outputs +6. rerun the same scenarios once per agent after optimization + +Artifacts are written under `.weksa-runs/metame-swarm/`, which is ignored by +git because run packets may include projected Persona memory and raw Discord +context. diff --git a/docs/roadmap.md b/docs/roadmap.md index 0e4e80c..074d140 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -25,9 +25,10 @@ IPA. - Add one toy utterance embedding handoff fixture, then replace hand-authored values with the AquaSynth-owned learned encoder. -- Follow the cut sequence in - [the lowering map](./targets/interlingua-lowering-map.md): packet store, - lowering, the golf loop, then a service if one is earned. +- Add a daemon loop that commits conversational intent, pronunciation plans, and + utterance handoffs as typed CultCache witnesses. +- Publish the provider advertisement through CultMesh for Odin discovery; the + operator view returns on Thing 2 (Eve is deprecated). - Preserve the first English-only lowering proof under `examples/english-lowering/`, and the shared multilingual proof under `examples/multilingual-lowering/`. @@ -38,7 +39,7 @@ - Render glossed output through grammar, morphology, phonology, and diachrony. - Emit a trace explaining which stage made each decision. - Keep runtime LLM-free; model use belongs in authoring and critique. -- Keep CLI tools and fixtures on the same commit and +- Keep CLI tools, fixtures, and daemon jobs on the same commit and derivation paths so Weksa does not grow split-brained intent truth. ## Later diff --git a/docs/targets/interlingua-lowering-map.md b/docs/targets/interlingua-lowering-map.md index 2183f5b..e8907d9 100644 --- a/docs/targets/interlingua-lowering-map.md +++ b/docs/targets/interlingua-lowering-map.md @@ -12,15 +12,14 @@ The interlingua spec has two layers. The **schema** is Rust types: packet families, node ids, role slots, value kinds. It changes rarely and by commit. The **inventory** is data in CultCache: the concepts, frames, roles, relation types, speech acts and nuance features (hedge, certainty, warmth, irony and -whatever the loop finds missing) that the schema's slots name by id. Every open slot in a packet is an inventory id; a referent names an entity by id -from the entity store, and a packet's only free text is the trace. No door judges text for leakage: -what each model call may see, and what a score may count, own that -(`weksa-lowering:ruling:leak-owned-by-flow-and-scorer-3`). The golf loop improves the spec +whatever the loop finds missing) that the schema's slots name by id. Every +open slot in a packet is an inventory id, never free text, so a packet carries +no target-language wording (`no-surface-leak`). The golf loop improves the spec mostly by growing and refining the inventory, which is fast; schema changes are the slow path. Concepts are open-class and grow forever; the other families are closed-class, and convergence is measured on them -(`weksa-lowering:ruling:convergence-test`). This split was -`spec-is-schema-plus-inventory`, now carried by `leak-owned-by-flow-and-scorer-3`. +(`weksa-lowering:ruling:convergence-test`). This split is +`weksa-lowering:ruling:spec-is-schema-plus-inventory`. ## Model page @@ -29,19 +28,15 @@ means no store holds it and nothing writes it. | Kind | What names it | Lifecycle | Who decides | |---|---|---|---| -| Packet | Key `packet:<16 hex>`, an opaque digest of item, inventory revision and encoder (the item key lives in `provenance.corpus_item`, which the lowerer never sees); CultCache type `weksa.interlingua_packet`, carrying `schema_version` and `inventory_rev`. Every id inside is an opaque form type (`n` nodes, `encoder:<16 hex>`, canonical `:`). From cut `speaker-state` (schema v4) it also carries the utterance **stance**: the speaker's felt affect and restraint, as inventory features of slot `Stance` (see Speaker state below) | Minted by an encoder for one corpus item under one spec version; immutable; a new spec version means a new packet, never an edit. Admitted only through `PacketStore::admit`, which validates against the inventory store at the packet's revision | The encoder writes; the form types refuse malformed ids at construction and load, the validator refuses broken references and over-capacity packets: a feature at most once per slot, at most 64 nodes (`typed-packet`); nothing else writes packets | +| Packet | Key `packet::r:`; CultCache type `weksa.interlingua_packet`, carrying `schema_version` and `inventory_rev` | Minted by an encoder for one corpus item under one spec version; immutable; a new spec version means a new packet, never an edit. Admitted only through the validator | The encoder writes; the validator refuses malformed packets at the door (`typed-packet`); nothing else writes packets | | Schema version | `weksa.interlingua_packet.v` constant in `crates/weksa`; published JSON Schema at `schemas/cultnet/` | Bumped by a commit that changes the Rust types; the published schema is regenerated in the same commit and a test fails on drift | A cut (Hands, under a spec); never the loop | -| Inventory entry | Key `:`, `n` canonical decimal >= 1 (families: concept, frame, role, relation, speech_act, feature, referent_kind, extension); type `weksa.inventory_entry`, with a bounded gloss, value kind (option labels bounded), the slots a feature may sit in (`Discourse`, `Referent`, `Predication`, `Relation`; `Stance` from cut `speaker-state`, `Speaker` and `Scene` from cut `speaker-scene`), `introduced_rev` and optional `retired_rev`. Speaker axes, scene relations and affects are features of those slots, so the loop grows them like any closed-class family and convergence covers them | Added or retired by an inventory revision; never edited in place (a changed meaning is a retire plus an add); a retired id still resolves for packets of earlier revisions | The loop's accepted rounds, whose minting call the context builder assembles from Train items only; Self on merge to main. Admission checks form, not text | +| Inventory entry | Key `:` (families: concept, frame, role, relation, speech_act, feature); type `weksa.inventory_entry`, with gloss, value kind, `introduced_rev`, optional `retired_rev` | Added or retired by an inventory revision; never edited in place (a changed meaning is a retire plus an add); a retired id still resolves for packets of earlier revisions | The loop's accepted rounds; Self on merge to main | | Inventory revision | Key `rev:`, type `weksa.inventory_revision`, with parent, change list and `frozen` | Appended per accepted round; exactly one revision is ever marked frozen, by the convergence test | The loop's accept rule appends; the convergence test freezes; Self records the freeze as a ruling | -| Target profile | Locale or language-project id (`en-US`, `ja-JP`, a conlang's slug) plus a revision `profile::` | Authored, revised by commit or by an accepted round that changed the lowering; each lowering names the revision it used | Profile authors and accepted rounds; never the lowerer's defaults (`persona-drives-register`) | -| Language ontology entry (cut `language-ontology`) | Key `onto::`; type `weksa.ontology_entry`: conditions over feature values (stance, speaker, scene, packet features) and community ids, and the surface choice they select (layer: lexicon, grammar, discourse, phonology, orthography; a bounded descriptor and at most three bounded examples); introduced and retired by a profile revision | Added or retired by a profile revision, never edited; converted from `data/target-language-ontology/*.yaml`, which is deleted in the same cut so the YAML is not a second owner | The language's owner (the operator for her settings' languages) and accepted rounds; an ontology-minting call is assembled from Train items only, like an inventory-minting call, and its text is in the scorer's leak diagnostic. Selection is deterministic code (`select(profile, stance, speaker, scene)`); the lowerer realizes the selected choices and the trace lists their ids (`lowering-traced`) | -| Community (cut `language-ontology`) | Key `community:`; type `weksa.community`, converted from `data/cultural-ontology/cultures/` and `referents/` (YAML deleted in the same cut) | Revised by commit; a speaker names the communities it belongs to by id | The cultural ontology's author (owner-attested evidence only; Discord-only content stays deleted, `discord-only-content-deleted`) | -| Speaker (cut `speaker-scene`) | Key `speaker:@`; type `weksa.speaker`: stable axes as feature values of slot `Speaker` (origin, socioeconomic standing, age cohort, education and whatever the loop adds), at an inventory revision, and an `Annotator` | A new revision appends, never edits; a corpus item and every prior turn name a speaker slug, and a round records the speaker revisions it read | The annotator it names: `PersonaSnapshot` (a projection of a `gamecult.persona_state.v0` document, naming its source document and `updatedAt`), `Operator`, `WorkMetadata` (a public-domain work's own cast list and narration), or `Model { family, generator }` (the 2b `ModelFamily`) | -| Scene (cut `speaker-scene`) | Key `scene:@`, the item's split group; type `weksa.scene`: participants (speaker slugs) and, per ordered pair, relation features of slot `Scene` (relative status, intimacy, in-group, audience and whatever the loop adds), at an inventory revision, and an `Annotator` | As Speaker. Keyed by the group, so a scene never straddles the split | As Speaker | -| Corpus item | Key `item::`, `n` canonical decimal >= 1 (no sign, no leading zero, from cut 2b); type `weksa.corpus_item` with text, language, speaker, split group, prior turns, source. Source `ModelWritten` carries a `ModelFamily` enum, the generator string as the fixture wrote it, and a prompt digest (cut 2b). The item carries no speaker state of its own: its speaker slug and group name the Speaker and Scene documents | Admitted once; the whole item is immutable (re-adding an identical item is a no-op, any difference is refused; a correction is a new item that `supersedes` the old); text already held under another key is refused unless the new item supersedes that key (cut 2b); withdrawn, never deleted, so old scores stay explainable | Importers and `weksa corpus add`, under `corpus-cast-plus-public-domain`; the source enum has no variant for Discord or any training-forbidden source (`corpus-clean`) | -| Split assignment | Derived: `split(scheme, item.group)` | Fixed for a scheme; a new scheme is a new constant, never a reassignment. A model-written item whose group derives test is in no split; a superseding item keeps its predecessor's group | The hash function. Nobody assigns by hand, so there is no writer to forbid | +| Target profile | Locale or language-project id (`en-US`, `ja-JP`, a conlang's slug) plus a revision | Authored, revised by commit or by an accepted round that changed the lowering; each lowering names the revision it used | Profile authors and accepted rounds (`profile-owns-register`); never the lowerer's defaults | +| Corpus item | Key `item::`; type `weksa.corpus_item` with text, language, speaker, split group, prior turns, source | Admitted once; the text is immutable (a correction is a new item that `supersedes` the old); withdrawn, never deleted, so old scores stay explainable | Importers and `weksa corpus add`, under `corpus-cast-plus-public-domain`; the source enum has no variant for Discord or any training-forbidden source (`corpus-clean`) | +| Split assignment | Derived: `split(scheme, item.group)` | Fixed for a scheme; a new scheme is a new constant, never a reassignment. Model-written items never land in test | The hash function. Nobody assigns by hand, so there is no writer to forbid | | Loop round | Key `run: