diff --git a/.gitignore b/.gitignore index f5ad87f..1428a57 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,7 @@ state/scratch-runtime/ .weksa-runs/ .weksa-smoke/ .weksa/*.cc +*.cc.lock .weksa/*.json .weksa/*.log .weksa/*.pid @@ -20,3 +21,4 @@ dist/ build/ .venv/ venv/ +target/ diff --git a/AGENTS.md b/AGENTS.md index 8befd5d..ce540a6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -46,7 +46,7 @@ The live state architecture is ported from EpiphanyAgent in reduced form: - `state/map.yaml` is canonical and slow-changing. - `state/scratch.md` is temporary and disposable. - `schemas/` explains state contracts in human language. -- `packages/weksa-state-model` owns executable typed state. +- `crates/weksa` owns executable typed state. Project truth does not belong in agent personality memory. If Weksa later grows role dossiers or heartbeat scheduling, those surfaces must remain separate from diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..4f5d537 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,910 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "android_system_properties" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc" +dependencies = [ + "libc", +] + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "ar_archive_writer" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73cd58deff2140a0a8eae87e417bd01db68a33e148aa93d1e8cd837e55e312b6" +dependencies = [ + "object", +] + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "bitflags" +version = "2.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "cc" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86e055fa6e3e40ad6187ba25b6855eeb635d5271f8e0f0e1e9a5826b8d75bea0" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "iana-time-zone", + "js-sys", + "num-traits", + "serde", + "wasm-bindgen", + "windows-link", +] + +[[package]] +name = "clap" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa8876b300ab35ba921adea3dfd70157a46249b33f95c9084ae5709785478946" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0797fb7aeb1406c84efac526901f7ec3ead2124f946b494e72879d4b54704d" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9c751b79415d4e559e3d1fcf128e09e720eb673a06d26cf6f392d37d75b66e0" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "clap_lex" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c133bc6a41be0d194c306b5506d15e6feeea7b1d6604bd3f8310dfb2ca96486" + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "cultcache-rs" +version = "0.3.1" +source = "git+https://github.com/GameCult/CultLib.git?rev=64b86e0a273a3936939357e9deb186b7bf3bc417#64b86e0a273a3936939357e9deb186b7bf3bc417" +dependencies = [ + "anyhow", + "chrono", + "cultcache-rs-derive", + "fs2", + "redb", + "rmp-serde", + "serde", + "serde_bytes", + "serde_json", + "serde_stacker", + "stacker", + "uuid", + "windows-sys 0.59.0", +] + +[[package]] +name = "cultcache-rs-derive" +version = "0.2.0" +source = "git+https://github.com/GameCult/CultLib.git?rev=64b86e0a273a3936939357e9deb186b7bf3bc417#64b86e0a273a3936939357e9deb186b7bf3bc417" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "dyn-clone" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "find-msvc-tools" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c66915cc5391097de7a2b0b972e8505aa33c0df401fd3a6811dbdb32881ba48" + +[[package]] +name = "fs2" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "slab", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi", +] + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "js-sys" +version = "0.3.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7883d941dae510fb2d978fc3fe018c71c9e2892fd38854de3e8b92c2e5ad9cc5" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "libc" +version = "0.2.191" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "log" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "object" +version = "0.39.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e5a6c098c7a3b6547378093f5cc30bc54fd361ce711e05293a5cc589562739b" +dependencies = [ + "memchr", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "psm" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4dcd034599e63b970727f70d79e02d62390a4a84f7c6b827c27c46d5ac3fa622" +dependencies = [ + "ar_archive_writer", + "cc", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "redb" +version = "4.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39178e0a867f9e3eaa6292a4ab1dd3613f2519c5b564341e1e7b29b270736cc4" +dependencies = [ + "libc", +] + +[[package]] +name = "ref-cast" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e440fb4e4b4147295338efb76001ab9e4efc0e5839df2c47fc5ac2381d365c3" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92ecd8964f8453721699a1ed72037b0db49ce2f5a5138486ee89bed6f67cdf3a" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "rmp" +version = "0.8.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ba8be72d372b2c9b35542551678538b562e7cf86c3315773cae48dfbfe7790c" +dependencies = [ + "num-traits", +] + +[[package]] +name = "rmp-serde" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f81bee8c8ef9b577d1681a70ebbc962c232461e397b22c208c43c04b67a155" +dependencies = [ + "rmp", + "serde", +] + +[[package]] +name = "rustix" +version = "1.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5e76742e5294153d4dde1207dbec5bdd81eef6fabdc0d2118babadabfd858aa" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "schemars" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab508826f74a77ca9d5aba6ff19b522583ee3eaf28a19384ff3d0e5835fadf6e" +dependencies = [ + "dyn-clone", + "ref-cast", + "schemars_derive", + "serde", + "serde_json", +] + +[[package]] +name = "schemars_derive" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1c3a92094fa7d61aa124645844facb6b554dfc797136d0f5fd1f890e2bffc69" +dependencies = [ + "proc-macro2", + "quote", + "serde_derive_internals", + "syn 3.0.9", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_bytes" +version = "0.11.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5d440709e79d88e51ac01c4b72fc6cb7314017bb7da9eeff678aa94c10e3ea8" +dependencies = [ + "serde", + "serde_core", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "serde_derive_internals" +version = "0.30.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f852137cce035d6a4df67ccce505ff6b3e9fd3a10e3e52b24dc71e650bb1a9bd" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "serde_json" +version = "1.0.154" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_stacker" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4936375d50c4be7eff22293a9344f8e46f323ed2b3c243e52f89138d9bb0f4a" +dependencies = [ + "serde", + "serde_core", + "stacker", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "stacker" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "707f49d46706bacf8a2b00d51dace3f9de527c13eec3778f570c411f89e69967" +dependencies = [ + "cc", + "cfg-if", + "libc", + "psm", + "windows-sys 0.61.2", +] + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom", + "once_cell", + "rustix", + "windows-sys 0.61.2", +] + +[[package]] +name = "unicode-ident" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "uuid" +version = "1.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a0e2d11620c50695fe12bbbfde96b3477fe606e7284c22d505c12ce51214514" +dependencies = [ + "getrandom", + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bb54f33acc68fd454578d9820b0bde1a1a3d17aa17bb7b6595806d02886d409" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e29d0c35b16e224a7eeb5cd2d25e3e1968fbd65604117b44d3b789d00ee8535" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f501a8bc3719dba86ef8ae4728879c08001bea749eb1333ac5b91e040e2a6b7" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 3.0.9", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23f0c9c52aa7cd7d77769a4cfe2a9adb1b331f489a41d912ce14513d5ab995c6" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "weksa" +version = "0.1.0" +dependencies = [ + "anyhow", + "clap", + "cultcache-rs", + "schemars", + "serde", + "serde_json", + "tempfile", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..956fefd --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,3 @@ +[workspace] +resolver = "3" +members = ["crates/weksa"] diff --git a/README.md b/README.md index 72440d5..eae6668 100644 --- a/README.md +++ b/README.md @@ -17,14 +17,20 @@ measured. **Entry points.** - [Interlingua Standard](./docs/interlingua-standard.md): the packet contract. +- `crates/weksa`: the packet, inventory and corpus types, `validate`, the + derived split and the `weksa` CLI (`weksa validate`, `corpus`, `inventory`, + `split`, `schema`); published schemas in `schemas/cultnet/`. +- `data/corpus/corpus.cc` and `data/interlingua/inventory.cc`: the English + corpus and the inventory, read with `weksa corpus dump` and + `weksa inventory dump`. - [Lowering target](./docs/targets/interlingua-lowering.md) and [its map](./docs/targets/interlingua-lowering-map.md): the design, scoring and cut sequence. - `data/`: the cultural ontology and target-language profile data. - `settings/`: setting-specific proving grounds for language projects. -**Live versus planned.** Live: docs, fixtures, examples and ontology data. -Planned: the packet crate `crates/weksa` (cut `packet-store`), lowering, and +**Live versus planned.** Live: docs, fixtures, examples, ontology data and the +packet crate `crates/weksa`. Planned: scoring, lowering, and the golf loop that scores lowered English against an expressive corpus. No daemon runs; the earlier Node daemon is parked at tag `parked/weksa-node-body`. @@ -85,7 +91,8 @@ AquaSynth or expanding pronunciation coverage. ## Implementation Status The persistent state spine is live: map, scratch, schemas, docs, examples and -data. Packet storage, lowering and the service are planned (see the cut +data. Packet types, validation and the corpus and inventory stores are live in +`crates/weksa`. Scoring, lowering and the service are planned (see the cut sequence in [the lowering map](./docs/targets/interlingua-lowering-map.md)). Writing and agent state will enter Weksa as intent pressure, and typed utterance documents will leave for AquaSynth. diff --git a/crates/weksa/Cargo.toml b/crates/weksa/Cargo.toml new file mode 100644 index 0000000..f3ae47d --- /dev/null +++ b/crates/weksa/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "weksa" +version = "0.1.0" +edition = "2024" +publish = false + +[lib] +name = "weksa" +path = "src/lib.rs" + +[[bin]] +name = "weksa" +path = "src/main.rs" + +[dependencies] +anyhow = "1" +clap = { version = "4", features = ["derive"] } +cultcache-rs = { git = "https://github.com/GameCult/CultLib.git", rev = "64b86e0a273a3936939357e9deb186b7bf3bc417", package = "cultcache-rs" } +schemars = "1" +serde = { version = "1", features = ["derive"] } +serde_json = "1" + +[dev-dependencies] +tempfile = "3" diff --git a/crates/weksa/src/corpus.rs b/crates/weksa/src/corpus.rs new file mode 100644 index 0000000..ee7e447 --- /dev/null +++ b/crates/weksa/src/corpus.rs @@ -0,0 +1,77 @@ +//! The English target corpus. An item's text is immutable; a correction is a new +//! item that supersedes the old, and an item is withdrawn, never deleted. The +//! source enum has no variant for a source whose terms forbid training. + +use cultcache_rs::DatabaseEntry; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct Turn { + pub speaker: String, + pub text: String, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub enum Source { + /// A line from a cast member's hand-authored fixture. + CastAuthored { + persona: String, + fixture: String, + }, + OperatorWritten, + OperatorApproved { + note: String, + }, + PublicDomain { + work: String, + author: String, + edition_url: String, + }, + /// Labelled model output; never in the test split, never judged by its own family. + ModelWritten { + model_family: String, + prompt_digest: Option, + }, +} + +/// Key: `item::`. +#[derive(Clone, Debug, PartialEq, Eq, JsonSchema, DatabaseEntry)] +#[cultcache(type = "weksa.corpus_item")] +pub struct CorpusItem { + #[cultcache(key = 0)] + pub id: String, + #[cultcache(key = 1)] + pub text: String, + #[cultcache(key = 2)] + pub language: String, + #[cultcache(key = 3)] + pub speaker: String, + /// The unit the split hashes: a scene, a fixture, a work. + #[cultcache(key = 4)] + pub group: String, + #[cultcache(key = 5)] + pub prior_turns: Vec, + #[cultcache(key = 6)] + pub source: Source, + #[cultcache(key = 7)] + pub admitted_on: String, + /// Key of the item this one corrects. + #[cultcache(key = 8)] + pub supersedes: Option, + #[cultcache(key = 9)] + pub withdrawn: bool, +} + +/// `item::`. +pub fn item_key(slug: &str, n: u32) -> String { + format!("item:{slug}:{n}") +} + +/// The slug of an item key, if it is `item::`. +pub fn key_slug(key: &str) -> Option<&str> { + let rest = key.strip_prefix("item:")?; + let (slug, n) = rest.rsplit_once(':')?; + n.parse::().ok()?; + (!slug.is_empty()).then_some(slug) +} diff --git a/crates/weksa/src/form.rs b/crates/weksa/src/form.rs new file mode 100644 index 0000000..bbde59e --- /dev/null +++ b/crates/weksa/src/form.rs @@ -0,0 +1,62 @@ +//! Form types. A form lives in a newtype whose only constructor is a fallible +//! `parse`; deserializing goes through the same parse, so a malformed value can +//! neither be built in Rust nor read back from a store. + +use std::fmt; + +/// A value that is not of the form its type names. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct FormError { + pub form: &'static str, + pub got: String, +} + +impl FormError { + pub fn new(form: &'static str, got: impl fmt::Display) -> Self { + Self { + form, + got: got.to_string(), + } + } +} + +impl fmt::Display for FormError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?} is not a {}", self.got, self.form) + } +} + +impl std::error::Error for FormError {} + +/// Wires a string form type (with `parse` and `Display`) to FromStr, serde's +/// `try_from`/`into` String and a published schema. +macro_rules! string_form { + ($t:ident, $schema:tt) => { + impl std::str::FromStr for $t { + type Err = $crate::form::FormError; + fn from_str(s: &str) -> Result { + Self::parse(s) + } + } + impl TryFrom for $t { + type Error = $crate::form::FormError; + fn try_from(s: String) -> Result { + Self::parse(&s) + } + } + impl From<$t> for String { + fn from(v: $t) -> String { + v.to_string() + } + } + impl schemars::JsonSchema for $t { + fn schema_name() -> std::borrow::Cow<'static, str> { + stringify!($t).into() + } + fn json_schema(_: &mut schemars::SchemaGenerator) -> schemars::Schema { + schemars::json_schema!($schema) + } + } + }; +} +pub(crate) use string_form; diff --git a/crates/weksa/src/inventory.rs b/crates/weksa/src/inventory.rs new file mode 100644 index 0000000..1106229 --- /dev/null +++ b/crates/weksa/src/inventory.rs @@ -0,0 +1,456 @@ +//! The inventory: the ids packets name, as CultCache documents under appended +//! revisions. An entry is never edited in place; a changed meaning is a retire +//! plus an add, and a retired id still resolves for packets of earlier revisions. + +use std::collections::BTreeMap; +use std::fmt; +use std::num::NonZeroU32; + +use cultcache_rs::DatabaseEntry; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +use crate::form::{FormError, string_form}; + +#[derive( + Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize, JsonSchema, +)] +pub enum Family { + Concept, + Frame, + Role, + Relation, + SpeechAct, + Feature, + ReferentKind, + /// A game- or loop-declared packet field; the entry carries its value kind. + Extension, +} + +impl Family { + pub const ALL: [Family; 8] = [ + Family::Concept, + Family::Frame, + Family::Role, + Family::Relation, + Family::SpeechAct, + Family::Feature, + Family::ReferentKind, + Family::Extension, + ]; + + pub fn prefix(self) -> &'static str { + match self { + Family::Concept => "concept", + Family::Frame => "frame", + Family::Role => "role", + Family::Relation => "relation", + Family::SpeechAct => "speech_act", + Family::Feature => "feature", + Family::ReferentKind => "referent_kind", + Family::Extension => "extension", + } + } + + /// The id of the `n`th entry of this family. + pub fn id(self, n: NonZeroU32) -> InvId { + InvId { family: self, n } + } +} + +/// Id of an inventory entry: `:`, `n` a canonical decimal +/// (no sign, no leading zero) of at least 1. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +#[serde(try_from = "String", into = "String")] +pub struct InvId { + family: Family, + n: NonZeroU32, +} + +impl InvId { + pub fn parse(s: &str) -> Result { + let bad = || FormError::new("InvId", s); + let (prefix, n) = s.split_once(':').ok_or_else(bad)?; + let family = Family::ALL + .into_iter() + .find(|f| f.prefix() == prefix) + .ok_or_else(bad)?; + if n.starts_with('0') || !n.bytes().all(|b| b.is_ascii_digit()) { + return Err(bad()); + } + let n = n.parse::().map_err(|_| bad())?; + Ok(Self { family, n }) + } + + pub fn family(self) -> Family { + self.family + } +} + +impl fmt::Display for InvId { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}:{}", self.family.prefix(), self.n) + } +} + +string_form!(InvId, { "type": "string", "pattern": "^(concept|frame|role|relation|speech_act|feature|referent_kind|extension):[1-9][0-9]*$" }); + +/// A value label of an options feature: `^[a-z][a-z0-9_]{0,23}$`. +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(try_from = "String", into = "String")] +pub struct OptionLabel(String); + +impl OptionLabel { + pub fn parse(s: &str) -> Result { + let mut bytes = s.bytes(); + let ok = s.len() <= 24 + && bytes.next().is_some_and(|b| b.is_ascii_lowercase()) + && bytes.all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'_'); + if ok { + Ok(Self(s.to_string())) + } else { + Err(FormError::new("OptionLabel", s)) + } + } + + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl fmt::Display for OptionLabel { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&self.0) + } +} + +string_form!(OptionLabel, { "type": "string", "pattern": "^[a-z][a-z0-9_]{0,23}$" }); + +/// 1 to 16 distinct labels. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(try_from = "Vec", into = "Vec")] +pub struct OptionSet(Vec); + +impl OptionSet { + pub const MAX: usize = 16; + + pub fn new(labels: Vec) -> Result { + let mut distinct: Vec<&OptionLabel> = labels.iter().collect(); + distinct.sort(); + distinct.dedup(); + if (1..=Self::MAX).contains(&labels.len()) && distinct.len() == labels.len() { + Ok(Self(labels)) + } else { + Err(FormError::new( + "OptionSet", + format!("{} labels", labels.len()), + )) + } + } + + pub fn contains(&self, label: &OptionLabel) -> bool { + self.0.contains(label) + } +} + +impl TryFrom> for OptionSet { + type Error = FormError; + fn try_from(v: Vec) -> Result { + Self::new(v) + } +} + +impl From for Vec { + fn from(s: OptionSet) -> Self { + s.0 + } +} + +impl JsonSchema for OptionSet { + fn schema_name() -> std::borrow::Cow<'static, str> { + "OptionSet".into() + } + fn json_schema(g: &mut schemars::SchemaGenerator) -> schemars::Schema { + schemars::json_schema!({ + "type": "array", + "items": g.subschema_for::(), + "minItems": 1, + "maxItems": 16, + "uniqueItems": true + }) + } +} + +/// Where a feature may sit in a packet. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub enum Slot { + Discourse, + Referent, + Predication, + Relation, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub enum ValueKind { + Grade, + Options(OptionSet), + Flag, +} + +/// Key: the entry's id, `:`. +#[derive(Clone, Debug, PartialEq, Eq, JsonSchema, DatabaseEntry)] +#[cultcache(type = "weksa.inventory_entry")] +pub struct InventoryEntry { + #[cultcache(key = 0)] + pub id: InvId, + /// 1 to 120 bytes, no control character (checked at admission). + #[cultcache(key = 1)] + pub gloss: String, + /// Present exactly on features and extensions. + #[cultcache(key = 2)] + pub value_kind: Option, + /// Non-empty exactly on features. + #[cultcache(key = 3)] + pub applies_to: Vec, + #[cultcache(key = 4)] + pub introduced_rev: u32, + #[cultcache(key = 5)] + pub retired_rev: Option, +} + +impl InventoryEntry { + pub fn family(&self) -> Family { + self.id.family() + } + + /// Whether the entry's shape matches its family: features carry a value kind + /// and slots, extensions a value kind and no slots, nothing else either. + pub fn well_formed(&self) -> bool { + match self.family() { + Family::Feature => self.value_kind.is_some() && !self.applies_to.is_empty(), + Family::Extension => self.value_kind.is_some() && self.applies_to.is_empty(), + _ => self.value_kind.is_none() && self.applies_to.is_empty(), + } + } + + /// Whether a packet of inventory revision `rev` may name this entry. + pub fn live_at(&self, rev: u32) -> bool { + self.introduced_rev <= rev && self.retired_rev.is_none_or(|r| r > rev) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub enum Change { + Add(InvId), + Retire(InvId), +} + +/// Key: `rev:`. +#[derive(Clone, Debug, PartialEq, Eq, JsonSchema, DatabaseEntry)] +#[cultcache(type = "weksa.inventory_revision")] +pub struct InventoryRevision { + #[cultcache(key = 0)] + pub n: u32, + #[cultcache(key = 1)] + pub parent: Option, + #[cultcache(key = 2)] + pub changes: Vec, + #[cultcache(key = 3)] + pub frozen: bool, +} + +pub fn revision_key(n: u32) -> String { + format!("rev:{n}") +} + +/// The whole inventory as the store holds it. +#[derive(Clone, Debug, Default)] +pub struct Inventory { + entries: BTreeMap, + revisions: BTreeMap, +} + +impl Inventory { + /// An inventory exists only as `InventoryStore::inventory` returns it, so + /// no caller can validate or admit against a fabricated one. + /// + /// ```compile_fail,E0624 + /// // `Inventory::new` is not callable outside the crate. + /// let _ = weksa::Inventory::new(vec![], vec![]); + /// ``` + pub(crate) fn new(entries: Vec, revisions: Vec) -> Self { + Self { + entries: entries.into_iter().map(|e| (e.id, e)).collect(), + revisions: revisions.into_iter().map(|r| (r.n, r)).collect(), + } + } + + pub fn entries(&self) -> impl Iterator { + self.entries.values() + } + + pub fn entry(&self, id: &InvId) -> Option<&InventoryEntry> { + self.entries.get(id) + } + + pub fn revisions(&self) -> impl Iterator { + self.revisions.values() + } + + pub fn has_revision(&self, n: u32) -> bool { + self.revisions.contains_key(&n) + } + + pub fn latest(&self) -> Option<&InventoryRevision> { + self.revisions.values().next_back() + } + + /// The entry `id`, if a packet of revision `rev` may name it. + pub fn resolve(&self, id: &InvId, rev: u32) -> Option<&InventoryEntry> { + self.entry(id).filter(|e| e.live_at(rev)) + } +} + +/// Revision 1: the standard's enumerated values and the nuance features the +/// loop starts from. Concepts start empty. +pub fn seed() -> (InventoryRevision, Vec) { + let mut entries: Vec = Vec::new(); + let mut add = |family: Family, gloss: &str, kind: Option, slots: &[Slot]| { + let taken = entries.iter().filter(|e| e.family() == family).count() as u32; + entries.push(InventoryEntry { + id: family.id(NonZeroU32::MIN.saturating_add(taken)), + gloss: gloss.to_string(), + value_kind: kind, + applies_to: slots.to_vec(), + introduced_rev: 1, + retired_rev: None, + }); + }; + for g in [ + "person", + "group", + "organism", + "place", + "artifact", + "institution", + "ecology", + "route", + "memory", + "event", + "abstract", + ] { + add(Family::ReferentKind, g, None, &[]); + } + for g in [ + "coreference", + "part_of", + "member_of", + "located_at", + "near", + "before", + "after", + "causes", + "enables", + "prevents", + "owns", + "owes", + "knows", + "names", + "represents", + "contrasts_with", + ] { + add(Family::Relation, g, None, &[]); + } + for g in [ + "agent", + "patient", + "experiencer", + "instrument", + "source", + "goal", + "location", + "beneficiary", + ] { + add(Family::Role, g, None, &[]); + } + for g in [ + "assert", + "ask", + "request", + "promise", + "warn", + "greet", + "thank", + "apologise", + ] { + add(Family::SpeechAct, g, None, &[]); + } + let options = |labels: &[&str]| { + let labels = labels + .iter() + .map(|l| OptionLabel::parse(l).expect("seed labels are well formed")) + .collect(); + Some(ValueKind::Options( + OptionSet::new(labels).expect("seed option sets are well formed"), + )) + }; + let pred = [Slot::Predication]; + add( + Family::Feature, + "polarity", + options(&["positive", "negative"]), + &pred, + ); + add( + Family::Feature, + "tense", + options(&["past", "present", "future", "timeless"]), + &pred, + ); + add( + Family::Feature, + "aspect", + options(&[ + "perfective", + "imperfective", + "progressive", + "habitual", + "perfect", + ]), + &pred, + ); + add( + Family::Feature, + "modality", + options(&[ + "actual", + "potential", + "possible", + "necessary", + "desired", + "obligated", + ]), + &pred, + ); + add( + Family::Feature, + "evidentiality", + options(&["direct", "inferred", "reported", "assumed"]), + &pred, + ); + let nuance = [Slot::Discourse, Slot::Predication]; + for g in ["hedge", "certainty", "warmth", "formality", "intensity"] { + add(Family::Feature, g, Some(ValueKind::Grade), &nuance); + } + add(Family::Feature, "irony", Some(ValueKind::Flag), &nuance); + let changes = entries.iter().map(|e| Change::Add(e.id)).collect(); + ( + InventoryRevision { + n: 1, + parent: None, + changes, + frozen: false, + }, + entries, + ) +} diff --git a/crates/weksa/src/lib.rs b/crates/weksa/src/lib.rs new file mode 100644 index 0000000..0be0bcc --- /dev/null +++ b/crates/weksa/src/lib.rs @@ -0,0 +1,25 @@ +//! Weksa's typed interlingua state: packet, inventory and corpus documents in +//! CultCache, the validator that decides whether a packet exists, the derived +//! train/validation/test split, and the stores that admit them. + +pub mod corpus; +pub mod form; +pub mod inventory; +pub mod packet; +pub mod schema; +pub mod split; +pub mod store; +pub mod validate; + +pub use corpus::{CorpusItem, Source, Turn}; +pub use form::FormError; +pub use inventory::{ + Change, Family, InvId, Inventory, InventoryEntry, InventoryRevision, OptionLabel, OptionSet, + Slot, ValueKind, +}; +pub use packet::{ + EncoderId, Grade, NodeId, Packet, PacketId, PacketKind, ProperName, SCHEMA_VERSION, +}; +pub use split::{Split, SplitScheme}; +pub use store::{AddOutcome, CorpusStore, InventoryStore, PacketStore, StoreError, StoreRefusal}; +pub use validate::{LoweringInput, Refusal, validate}; diff --git a/crates/weksa/src/main.rs b/crates/weksa/src/main.rs new file mode 100644 index 0000000..7887488 --- /dev/null +++ b/crates/weksa/src/main.rs @@ -0,0 +1,272 @@ +//! The `weksa` command line: the same validator and store admission the library +//! exposes, plus text projections (`dump`) that are for review and are not stores. + +use std::path::PathBuf; + +use anyhow::{Context, Result, bail}; +use clap::{Args, Parser, Subcommand, ValueEnum}; +use weksa::corpus::{CorpusItem, Source, Turn}; +use weksa::inventory::seed; +use weksa::{CorpusStore, InventoryStore, PacketStore, SplitScheme, StoreError, validate}; + +#[derive(Parser)] +#[command( + name = "weksa", + about = "Weksa interlingua packet, inventory and corpus tools" +)] +struct Cli { + /// The corpus store. + #[arg(long, global = true, default_value = "data/corpus/corpus.cc")] + corpus: PathBuf, + /// The inventory store. + #[arg(long, global = true, default_value = "data/interlingua/inventory.cc")] + inventory: PathBuf, + #[command(subcommand)] + command: Command, +} + +#[derive(Subcommand)] +enum Command { + /// Validate the packets in a store (or one packet) against the inventory. + Validate { store: PathBuf, key: Option }, + #[command(subcommand)] + Corpus(CorpusCommand), + #[command(subcommand)] + Inventory(InventoryCommand), + /// Print the split of a corpus item. + Split { item_key: String }, + /// Write the published JSON Schemas. + Schema { + #[arg(long, default_value = "schemas/cultnet")] + out: PathBuf, + }, +} + +#[derive(Subcommand)] +enum CorpusCommand { + Add(Box), + /// Text projection of the corpus for review; not a store. + Dump, + Withdraw { + key: String, + }, +} + +#[derive(Subcommand)] +enum InventoryCommand { + /// Text projection of the inventory for review; not a store. + Dump, + /// Admit revision 1, the standard's enumerated values and the nuance features. + Seed, +} + +#[derive(Clone, Copy, ValueEnum)] +enum SourceKind { + CastAuthored, + OperatorWritten, + OperatorApproved, + PublicDomain, + ModelWritten, +} + +#[derive(Args)] +struct AddArgs { + #[arg(long)] + slug: String, + #[arg(long)] + text: String, + #[arg(long, default_value = "en-US")] + language: String, + #[arg(long)] + speaker: String, + #[arg(long)] + group: String, + /// `speaker=text`, repeated, oldest first. + #[arg(long = "prior-turn")] + prior_turns: Vec, + #[arg(long, value_enum)] + source: SourceKind, + #[arg(long)] + persona: Option, + #[arg(long)] + fixture: Option, + #[arg(long)] + note: Option, + #[arg(long)] + work: Option, + #[arg(long)] + author: Option, + #[arg(long)] + edition_url: Option, + #[arg(long)] + model_family: Option, + #[arg(long)] + prompt_digest: Option, + #[arg(long)] + admitted_on: String, + #[arg(long)] + supersedes: Option, +} + +fn need(value: Option, flag: &str) -> Result { + value.with_context(|| format!("this source needs --{flag}")) +} + +impl AddArgs { + fn source(&self) -> Result { + let get = |v: &Option, flag: &str| need(v.clone(), flag); + Ok(match self.source { + SourceKind::CastAuthored => Source::CastAuthored { + persona: get(&self.persona, "persona")?, + fixture: get(&self.fixture, "fixture")?, + }, + SourceKind::OperatorWritten => Source::OperatorWritten, + SourceKind::OperatorApproved => Source::OperatorApproved { + note: get(&self.note, "note")?, + }, + SourceKind::PublicDomain => Source::PublicDomain { + work: get(&self.work, "work")?, + author: get(&self.author, "author")?, + edition_url: get(&self.edition_url, "edition-url")?, + }, + SourceKind::ModelWritten => Source::ModelWritten { + model_family: get(&self.model_family, "model-family")?, + prompt_digest: self.prompt_digest.clone(), + }, + }) + } + + fn item(self, id: String) -> Result { + let prior_turns = self + .prior_turns + .iter() + .map(|t| { + let (speaker, text) = t + .split_once('=') + .with_context(|| format!("--prior-turn {t:?} is not speaker=text"))?; + Ok(Turn { + speaker: speaker.to_string(), + text: text.to_string(), + }) + }) + .collect::>>()?; + Ok(CorpusItem { + id, + source: self.source()?, + text: self.text, + language: self.language, + speaker: self.speaker, + group: self.group, + prior_turns, + admitted_on: self.admitted_on, + supersedes: self.supersedes, + withdrawn: false, + }) + } +} + +/// The split of an item as `Train`, `Validation`, `Test` or `excluded`. +fn split_text(item: &CorpusItem) -> String { + SplitScheme::V1 + .split(item) + .map_or_else(|| "excluded".to_string(), |s| format!("{s:?}")) +} + +fn refused(e: StoreError) -> anyhow::Error { + anyhow::anyhow!("{e}") +} + +fn main() -> Result<()> { + let cli = Cli::parse(); + match cli.command { + Command::Validate { store, key } => { + let inventory = InventoryStore::open(&cli.inventory)?.inventory()?; + let packets = PacketStore::open(&store)?; + let packets = match key { + Some(key) => vec![ + packets + .get(&key)? + .with_context(|| format!("no packet {key}"))?, + ], + None => packets.all()?, + }; + let mut bad = 0; + for p in &packets { + match validate(p, &inventory) { + Ok(_) => println!("ok {}", p.id), + Err(refusals) => { + bad += 1; + for r in refusals { + println!("refused {} {r}", p.id); + } + } + } + } + if bad > 0 { + bail!("{bad} of {} packets refused", packets.len()); + } + } + Command::Corpus(CorpusCommand::Add(args)) => { + let mut store = CorpusStore::open(&cli.corpus)?; + let key = store.next_key(&args.slug)?; + let outcome = store.add((*args).item(key.clone())?).map_err(refused)?; + println!("{outcome:?} {key}"); + } + Command::Corpus(CorpusCommand::Dump) => { + for i in CorpusStore::open(&cli.corpus)?.all()? { + let split = split_text(&i); + let withdrawn = if i.withdrawn { " withdrawn" } else { "" }; + println!( + "{} [{} {} group={} {split}{withdrawn}]\n {:?}\n source: {:?}", + i.id, i.language, i.speaker, i.group, i.text, i.source + ); + } + } + Command::Corpus(CorpusCommand::Withdraw { key }) => { + CorpusStore::open(&cli.corpus)? + .withdraw(&key) + .map_err(refused)?; + println!("withdrawn {key}"); + } + Command::Inventory(InventoryCommand::Dump) => { + let inv = InventoryStore::open(&cli.inventory)?.inventory()?; + for r in inv.revisions() { + println!( + "rev:{} parent={:?} frozen={} changes={}", + r.n, + r.parent, + r.frozen, + r.changes.len() + ); + } + for e in inv.entries() { + println!( + "{} introduced={} retired={:?} {:?} {:?} {}", + e.id, e.introduced_rev, e.retired_rev, e.value_kind, e.applies_to, e.gloss + ); + } + } + Command::Inventory(InventoryCommand::Seed) => { + let (rev, entries) = seed(); + let count = entries.len(); + InventoryStore::open(&cli.inventory)? + .admit_revision(rev, entries) + .map_err(refused)?; + println!("seeded rev:1 with {count} entries"); + } + Command::Split { item_key } => { + let item = CorpusStore::open(&cli.corpus)? + .get(&item_key)? + .with_context(|| format!("no item {item_key}"))?; + println!("{}", split_text(&item)); + } + Command::Schema { out } => { + std::fs::create_dir_all(&out)?; + for (name, json) in weksa::schema::published() { + std::fs::write(out.join(&name), json)?; + println!("wrote {name}"); + } + } + } + Ok(()) +} diff --git a/crates/weksa/src/packet.rs b/crates/weksa/src/packet.rs new file mode 100644 index 0000000..cb56672 --- /dev/null +++ b/crates/weksa/src/packet.rs @@ -0,0 +1,365 @@ +//! The interlingua packet. Every id is an opaque form type: a node id `n`, a +//! packet id `packet:`, an encoder id `encoder:`, an inventory id +//! `:`. The free text is a bounded [`ProperName`] and the trace, which +//! is stripped before lowering. A malformed id or name cannot be built, and +//! cannot be read back from a store, so a packet cannot carry target-language +//! wording through an id slot. + +use std::fmt; + +use cultcache_rs::DatabaseEntry; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +use crate::form::{FormError, string_form}; +pub use crate::inventory::InvId; +use crate::inventory::OptionLabel; + +/// Bumped by a commit that changes these types; the published schema moves with it. +pub const SCHEMA_VERSION: u32 = 1; + +/// Id of a referent or predication within one packet: `n`, `k` a canonical +/// decimal 1..=9999. +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(try_from = "String", into = "String")] +pub struct NodeId(String); + +impl NodeId { + pub fn parse(s: &str) -> Result { + let k = s + .strip_prefix('n') + .filter(|d| !d.starts_with('0') && d.bytes().all(|b| b.is_ascii_digit())) + .and_then(|d| d.parse::().ok()) + .filter(|k| (1..=9999).contains(k)); + k.map(|_| Self(s.to_string())) + .ok_or_else(|| FormError::new("NodeId", s)) + } + + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl fmt::Display for NodeId { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&self.0) + } +} + +string_form!(NodeId, { "type": "string", "pattern": "^n[1-9][0-9]{0,3}$" }); + +/// `:<16 lowercase hex>`. +fn hex16(prefix: &str, s: &str) -> bool { + s.strip_prefix(prefix) + .is_some_and(|h| h.len() == 16 && h.bytes().all(|b| matches!(b, b'0'..=b'9' | b'a'..=b'f'))) +} + +/// Key of a packet: `packet:`, a digest the loop derives from the item, +/// the inventory revision and the encoder. +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(try_from = "String", into = "String")] +pub struct PacketId(String); + +impl PacketId { + pub fn parse(s: &str) -> Result { + if hex16("packet:", s) { + Ok(Self(s.to_string())) + } else { + Err(FormError::new("PacketId", s)) + } + } + + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl fmt::Display for PacketId { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&self.0) + } +} + +string_form!(PacketId, { "type": "string", "pattern": "^packet:[0-9a-f]{16}$" }); + +/// `encoder:`, a digest of the model id and the prompt digest. +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(try_from = "String", into = "String")] +pub struct EncoderId(String); + +impl EncoderId { + pub fn parse(s: &str) -> Result { + if hex16("encoder:", s) { + Ok(Self(s.to_string())) + } else { + Err(FormError::new("EncoderId", s)) + } + } + + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl fmt::Display for EncoderId { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&self.0) + } +} + +string_form!(EncoderId, { "type": "string", "pattern": "^encoder:[0-9a-f]{16}$" }); + +/// Permille, 0..=1000. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(try_from = "u16", into = "u16")] +pub struct Grade(u16); + +impl Grade { + pub const MAX: u16 = 1000; + + pub fn new(permille: u16) -> Result { + if permille <= Self::MAX { + Ok(Self(permille)) + } else { + Err(FormError::new("Grade", permille)) + } + } + + pub fn get(self) -> u16 { + self.0 + } +} + +impl TryFrom for Grade { + type Error = FormError; + fn try_from(v: u16) -> Result { + Self::new(v) + } +} + +impl From for u16 { + fn from(g: Grade) -> u16 { + g.0 + } +} + +impl JsonSchema for Grade { + fn schema_name() -> std::borrow::Cow<'static, str> { + "Grade".into() + } + fn json_schema(_: &mut schemars::SchemaGenerator) -> schemars::Schema { + schemars::json_schema!({ "type": "integer", "minimum": 0, "maximum": 1000 }) + } +} + +/// A proper name: 1 to 4 tokens joined by single spaces, 64 bytes at most. A +/// token is 1 to 24 letters, apostrophes (`'` or U+2019) and hyphens, beginning +/// and ending with a letter. The form bounds capacity only: names are text the +/// scorer masks, and no door judges them for leakage. +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(try_from = "String", into = "String")] +pub struct ProperName(String); + +impl ProperName { + pub const MAX_BYTES: usize = 64; + pub const MAX_TOKENS: usize = 4; + pub const MAX_TOKEN_CHARS: usize = 24; + + pub fn parse(s: &str) -> Result { + let tokens: Vec<&str> = s.split(' ').collect(); + if s.len() <= Self::MAX_BYTES + && tokens.len() <= Self::MAX_TOKENS + && tokens.iter().all(|t| Self::token_ok(t)) + { + Ok(Self(s.to_string())) + } else { + Err(FormError::new("ProperName", s)) + } + } + + fn token_ok(t: &str) -> bool { + let (Some(first), Some(last)) = (t.chars().next(), t.chars().next_back()) else { + return false; + }; + t.chars().count() <= Self::MAX_TOKEN_CHARS + && first.is_alphabetic() + && last.is_alphabetic() + && t.chars() + .all(|c| c.is_alphabetic() || matches!(c, '\'' | '\u{2019}' | '-')) + } + + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl fmt::Display for ProperName { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&self.0) + } +} + +string_form!(ProperName, { + "type": "string", + "maxLength": 64, + "pattern": "^\\p{L}([\\p{L}'\u{2019}-]{0,22}\\p{L})?( \\p{L}([\\p{L}'\u{2019}-]{0,22}\\p{L})?){0,3}$", + "description": "1 to 4 space-joined tokens of letters, apostrophes and hyphens." +}); + +#[derive(Clone, Debug, PartialEq, Eq, JsonSchema, DatabaseEntry)] +#[cultcache(type = "weksa.interlingua_packet")] +pub struct Packet { + #[cultcache(key = 0)] + pub id: PacketId, + #[cultcache(key = 1)] + pub schema_version: u32, + #[cultcache(key = 2)] + pub inventory_rev: u32, + #[cultcache(key = 3)] + pub kind: PacketKind, + #[cultcache(key = 4)] + pub provenance: Provenance, + #[cultcache(key = 5)] + pub context: Context, + #[cultcache(key = 6)] + pub discourse: Discourse, + #[cultcache(key = 7)] + pub referents: Vec, + #[cultcache(key = 8)] + pub predications: Vec, + #[cultcache(key = 9)] + pub relations: Vec, + #[cultcache(key = 10)] + pub constraints: Constraints, + #[cultcache(key = 11)] + pub extensions: Vec, + #[cultcache(key = 12)] + pub trace: Trace, +} + +impl Packet { + /// The most referents, predications and relations one packet holds: a + /// calibration constant of ours, capacity and not meaning. + pub const MAX_NODES: usize = 64; +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub enum PacketKind { + Utterance, + NameRequest, + CommentaryIntent, + Narration, + DialogueLine, + CorpusExample, + DebugProbe, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub enum ProvenanceSource { + Authored, + ModelDecomposition, + GameTelemetry, + ImportedCorpus, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct Provenance { + pub source: ProvenanceSource, + /// Key of the corpus item this packet was minted for. + pub corpus_item: Option, + pub encoder: EncoderId, + pub confidence: Grade, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct Context { + /// Concept ids. + pub setting: Option, + pub medium: Option, + pub participants: Vec, + pub audience: Vec, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct Discourse { + /// Speech-act id. + pub speech_act: Option, + pub addressee: Option, + pub features: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct FeatureValue { + /// Feature id. + pub feature: InvId, + pub value: FeatValue, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub enum FeatValue { + Grade(Grade), + /// A label the feature's inventory entry declares. + Option(OptionLabel), + Flag(bool), +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct Referent { + pub id: NodeId, + /// Referent-kind id. + pub kind: InvId, + /// Concept id. + pub concept: InvId, + /// A proper name, the one place a referent carries wording. It is + /// form-bounded text; the scorer masks it. + pub name: Option, + pub features: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct Predication { + pub id: NodeId, + /// Concept id. + pub concept: InvId, + /// Frame id. + pub frame: InvId, + pub roles: Vec, + pub features: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct RoleFill { + /// Role id. + pub role: InvId, + pub filler: NodeId, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct Relation { + /// Relation id. + pub relation: InvId, + pub from: NodeId, + pub to: NodeId, + pub features: Vec, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct Constraints { + pub preserve: Vec, + pub deterministic: bool, + pub max_latency_ms: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct Extension { + /// Extension-entry id; the inventory declares its value kind. + pub key: InvId, + pub value: FeatValue, +} + +/// Authoring and debug notes. Stripped before lowering. +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct Trace { + pub notes: Vec, + pub rejected_readings: Vec, +} diff --git a/crates/weksa/src/schema.rs b/crates/weksa/src/schema.rs new file mode 100644 index 0000000..f3efc73 --- /dev/null +++ b/crates/weksa/src/schema.rs @@ -0,0 +1,23 @@ +//! JSON Schema publication. JSON is only the xenos-boundary export of the typed +//! documents; nothing in this crate reads JSON. + +use crate::corpus::CorpusItem; +use crate::inventory::{InventoryEntry, InventoryRevision}; +use crate::packet::Packet; +use cultcache_rs::DatabaseEntry; +use schemars::{JsonSchema, schema_for}; + +fn published_for() -> (String, String) { + let json = serde_json::to_string_pretty(&schema_for!(T)).expect("a schema serializes"); + (format!("{}.schema.json", T::TYPE), format!("{json}\n")) +} + +/// `(file name, contents)` for every document type, as `schemas/cultnet/` holds them. +pub fn published() -> Vec<(String, String)> { + vec![ + published_for::(), + published_for::(), + published_for::(), + published_for::(), + ] +} diff --git a/crates/weksa/src/split.rs b/crates/weksa/src/split.rs new file mode 100644 index 0000000..d2258a2 --- /dev/null +++ b/crates/weksa/src/split.rs @@ -0,0 +1,54 @@ +//! The train/validation/test split is derived from an item's group and nothing +//! else. No stored assignment exists, so no writer can reassign one. + +use crate::corpus::{CorpusItem, Source}; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Split { + Train, + Validation, + Test, +} + +/// A scheme is a constant; a new scheme is a new variant, never a reassignment. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum SplitScheme { + V1, +} + +pub fn fnv1a64(bytes: &[u8]) -> u64 { + let mut h: u64 = 0xcbf2_9ce4_8422_2325; + for b in bytes { + h ^= u64::from(*b); + h = h.wrapping_mul(0x0000_0100_0000_01b3); + } + h +} + +impl SplitScheme { + /// 0..100, FNV-1a 64 of `weksa-split-v1:` + group, mod 100. + pub fn bucket(self, group: &str) -> u64 { + match self { + SplitScheme::V1 => fnv1a64(format!("weksa-split-v1:{group}").as_bytes()) % 100, + } + } + + /// <40 Train, <80 Validation, else Test. + pub fn of_bucket(self, bucket: u64) -> Split { + match self { + SplitScheme::V1 if bucket < 40 => Split::Train, + SplitScheme::V1 if bucket < 80 => Split::Validation, + SplitScheme::V1 => Split::Test, + } + } + + /// The split of an item: its group's, except that a model-written item of a + /// test group is in no split. It is never moved to train: a model line of + /// the test meaning would teach the test. + pub fn split(self, item: &CorpusItem) -> Option { + match self.of_bucket(self.bucket(&item.group)) { + Split::Test if matches!(item.source, Source::ModelWritten { .. }) => None, + split => Some(split), + } + } +} diff --git a/crates/weksa/src/store.rs b/crates/weksa/src/store.rs new file mode 100644 index 0000000..5bcb40c --- /dev/null +++ b/crates/weksa/src/store.rs @@ -0,0 +1,361 @@ +//! Admission to the corpus, inventory and packet stores. Each store is a +//! CultCache over one `SingleFileMessagePackBackingStore` file. These functions +//! are the only writers, so the rules S1..S4 hold for every caller. + +use std::collections::BTreeSet; +use std::fmt; +use std::path::Path; + +use cultcache_rs::{CultCache, SingleFileMessagePackBackingStore}; + +use crate::corpus::{CorpusItem, item_key, key_slug}; +use crate::inventory::{Change, InvId, Inventory, InventoryEntry, InventoryRevision, revision_key}; +use crate::packet::Packet; +use crate::validate::{Refusal, validate}; + +/// A broken store rule, named. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum StoreRefusal { + /// S1: an existing item key re-added with different text. + S1ItemTextImmutable { key: String }, + /// S2: a revision that is not latest+1 with parent latest. + S2RevisionChain { expected: u32, got: u32 }, + /// S3: a revision after a frozen one. + S3Frozen { frozen: u32 }, + /// S4: an entry whose introduced_rev is not the revision adding it. + S4IntroducedRev { + id: InvId, + introduced: u32, + revision: u32, + }, + /// S5: a superseding item's group differs from its predecessor's. + S5SupersedeGroup { + key: String, + expected: String, + got: String, + }, + /// A `provenance.corpus_item` key names no item. + UnknownCorpusItem { key: String }, + /// A packet key already holds a different packet. + PacketImmutable { key: String }, + /// Any other malformed document or change. + Malformed { key: String, why: &'static str }, +} + +#[derive(Debug)] +pub enum StoreError { + Refused(StoreRefusal), + /// The packet broke validator rules. + Invalid(Vec), + Io(anyhow::Error), +} + +impl fmt::Display for StoreError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + StoreError::Refused(r) => write!(f, "refused: {r:?}"), + StoreError::Invalid(rs) => { + write!(f, "invalid packet:")?; + rs.iter().try_for_each(|r| write!(f, " [{r}]")) + } + StoreError::Io(e) => write!(f, "{e:#}"), + } + } +} + +impl std::error::Error for StoreError {} + +impl From for StoreError { + fn from(e: anyhow::Error) -> Self { + StoreError::Io(e) + } +} + +impl From for StoreError { + fn from(r: StoreRefusal) -> Self { + StoreError::Refused(r) + } +} + +fn malformed(key: &str, why: &'static str) -> StoreError { + StoreRefusal::Malformed { + key: key.to_string(), + why, + } + .into() +} + +fn open_cache( + path: &Path, + register: impl FnOnce(&mut CultCache) -> anyhow::Result<()>, +) -> anyhow::Result { + if let Some(dir) = path.parent() { + std::fs::create_dir_all(dir)?; + } + let mut cache = CultCache::new(); + register(&mut cache)?; + cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(path))?; + cache.pull_all_backing_stores()?; + Ok(cache) +} + +#[derive(Debug, PartialEq, Eq)] +pub enum AddOutcome { + Added, + /// The key already held this text; nothing was written. + Unchanged, +} + +pub struct CorpusStore { + cache: CultCache, +} + +impl CorpusStore { + pub fn open(path: &Path) -> anyhow::Result { + Ok(Self { + cache: open_cache(path, |c| c.register_entry_type::())?, + }) + } + + pub fn all(&self) -> anyhow::Result> { + let mut items = self.cache.get_all::()?; + items.sort_by(|a, b| a.id.cmp(&b.id)); + Ok(items) + } + + pub fn get(&self, key: &str) -> anyhow::Result> { + self.cache.get::(key) + } + + /// The next free key for `slug`; withdrawn items keep their numbers. + pub fn next_key(&self, slug: &str) -> anyhow::Result { + let highest = self + .all()? + .iter() + .filter(|i| key_slug(&i.id) == Some(slug)) + .filter_map(|i| i.id.rsplit_once(':')?.1.parse::().ok()) + .max() + .unwrap_or(0); + Ok(item_key(slug, highest + 1)) + } + + /// The item `key` names, or `UnknownCorpusItem` when the key is not + /// `item::` or holds nothing. + fn resolve(&self, key: &str) -> Result { + let unknown = || StoreRefusal::UnknownCorpusItem { + key: key.to_string(), + }; + if key_slug(key).is_none() { + return Err(unknown().into()); + } + self.get(key)?.ok_or_else(|| unknown().into()) + } + + /// S1: an item key holds its text for good. S5: a correction stays in its + /// predecessor's group, so it cannot move across the split. + pub fn add(&mut self, item: CorpusItem) -> Result { + if key_slug(&item.id).is_none() { + return Err(malformed(&item.id, "item key is not item::")); + } + if item.withdrawn { + return Err(malformed( + &item.id, + "an item is admitted live, then withdrawn", + )); + } + if let Some(existing) = self.get(&item.id)? { + if existing.text == item.text { + return Ok(AddOutcome::Unchanged); + } + return Err(StoreRefusal::S1ItemTextImmutable { key: item.id }.into()); + } + if let Some(old) = &item.supersedes { + let Some(old) = self.get(old)? else { + return Err(malformed( + &item.id, + "supersedes an item that does not exist", + )); + }; + if old.group != item.group { + return Err(StoreRefusal::S5SupersedeGroup { + key: item.id, + expected: old.group, + got: item.group, + } + .into()); + } + } + self.cache.put(item.id.clone(), &item)?; + Ok(AddOutcome::Added) + } + + /// Marks an item withdrawn; its text stays. + pub fn withdraw(&mut self, key: &str) -> Result<(), StoreError> { + let Some(mut item) = self.get(key)? else { + return Err(malformed(key, "no such item")); + }; + if !item.withdrawn { + item.withdrawn = true; + self.cache.put(key, &item)?; + } + Ok(()) + } +} + +pub struct InventoryStore { + cache: CultCache, +} + +impl InventoryStore { + pub fn open(path: &Path) -> anyhow::Result { + let cache = open_cache(path, |c| { + c.register_entry_type::()?; + c.register_entry_type::() + })?; + Ok(Self { cache }) + } + + pub fn inventory(&self) -> anyhow::Result { + Ok(Inventory::new( + self.cache.get_all::()?, + self.cache.get_all::()?, + )) + } + + /// Admits revision `rev` and the entries it adds, whole or not at all. + /// S2: latest+1 with parent latest. S3: nothing follows a frozen revision. + /// S4: an added entry's introduced_rev is this revision. + /// An added entry's gloss is 1 to 120 bytes without a control character. + /// No door judges text against the corpus: leak is owned by the loop's + /// context builder and the scorer. + pub fn admit_revision( + &mut self, + rev: InventoryRevision, + entries: Vec, + ) -> Result<(), StoreError> { + let inv = self.inventory()?; + let latest = inv.latest(); + if let Some(l) = latest.filter(|l| l.frozen) { + return Err(StoreRefusal::S3Frozen { frozen: l.n }.into()); + } + let (expected, parent) = latest.map_or((1, None), |l| (l.n + 1, Some(l.n))); + if rev.n != expected || rev.parent != parent { + return Err(StoreRefusal::S2RevisionChain { + expected, + got: rev.n, + } + .into()); + } + let mut touched: BTreeSet = BTreeSet::new(); + let mut staged = Vec::new(); + for change in &rev.changes { + match change { + Change::Add(id) => { + let Some(entry) = entries.iter().find(|e| &e.id == id) else { + return Err(malformed(&id.to_string(), "added id has no entry")); + }; + if entry.introduced_rev != rev.n { + let (id, introduced, revision) = (*id, entry.introduced_rev, rev.n); + return Err(StoreRefusal::S4IntroducedRev { + id, + introduced, + revision, + } + .into()); + } + let key = id.to_string(); + if !entry.well_formed() || entry.retired_rev.is_some() { + return Err(malformed(&key, "entry shape does not match its family")); + } + if inv.entry(id).is_some() || !touched.insert(*id) { + return Err(malformed(&key, "id is already in the inventory")); + } + check_entry_text(entry)?; + staged.push(self.cache.prepare_entry_named(key, entry)?.0); + } + Change::Retire(id) => { + let key = id.to_string(); + let Some(current) = inv.entry(id).filter(|e| e.retired_rev.is_none()) else { + return Err(malformed( + &key, + "retire target is missing or already retired", + )); + }; + if !touched.insert(*id) { + return Err(malformed(&key, "id changes twice in one revision")); + } + let retired = InventoryEntry { + retired_rev: Some(rev.n), + ..current.clone() + }; + staged.push(self.cache.prepare_entry_named(key, &retired)?.0); + } + } + } + if let Some(stray) = entries.iter().find(|e| !touched.contains(&e.id)) { + return Err(malformed( + &stray.id.to_string(), + "entry is not in the revision's changes", + )); + } + staged.push(self.cache.prepare_entry_named(revision_key(rev.n), &rev)?.0); + self.cache.put_prepared_batch(staged)?; + Ok(()) + } +} + +fn check_entry_text(entry: &InventoryEntry) -> Result<(), StoreError> { + if !(1..=120).contains(&entry.gloss.len()) || entry.gloss.chars().any(char::is_control) { + return Err(malformed( + &entry.id.to_string(), + "gloss is 1 to 120 bytes with no control character", + )); + } + Ok(()) +} + +pub struct PacketStore { + cache: CultCache, +} + +impl PacketStore { + pub fn open(path: &Path) -> anyhow::Result { + Ok(Self { + cache: open_cache(path, |c| c.register_entry_type::())?, + }) + } + + pub fn all(&self) -> anyhow::Result> { + let mut packets = self.cache.get_all::()?; + packets.sort_by(|a, b| a.id.cmp(&b.id)); + Ok(packets) + } + + pub fn get(&self, key: &str) -> anyhow::Result> { + self.cache.get::(key) + } + + /// The only door: a packet exists only if `validate` passes against the + /// inventory the store holds at the packet's revision, its `corpus_item` + /// (when set) resolves, and a key holds its packet for good. + pub fn admit( + &mut self, + packet: Packet, + inventory: &InventoryStore, + corpus: &CorpusStore, + ) -> Result<(), StoreError> { + validate(&packet, &inventory.inventory()?).map_err(StoreError::Invalid)?; + if let Some(item_key) = &packet.provenance.corpus_item { + corpus.resolve(item_key)?; + } + let key = packet.id.to_string(); + if let Some(existing) = self.get(&key)? { + if existing == packet { + return Ok(()); + } + return Err(StoreRefusal::PacketImmutable { key }.into()); + } + self.cache.put(key, &packet)?; + Ok(()) + } +} diff --git a/crates/weksa/src/validate.rs b/crates/weksa/src/validate.rs new file mode 100644 index 0000000..854e099 --- /dev/null +++ b/crates/weksa/src/validate.rs @@ -0,0 +1,475 @@ +//! The validator owns whether a packet exists. Every path that admits or lowers a +//! packet calls `validate`, and its Ok value, [`LoweringInput`], is the only +//! thing a lowerer sees. The rules are V1..V8 and V10..V12: the shape of an id +//! or a grade is its type's parse, not a rule, so V9 is retired and not +//! renumbered. +//! +//! `validate` owns form and capacity and makes no leak decision. A packet's +//! free text is form-bounded and every list is bounded, so a packet has no +//! unbounded channel. What bounds each list (V11, V12): a feature list holds at +//! most one value per feature id and `extensions` at most one per extension id, +//! so each is at most the inventory's ids of that kind; `participants`, +//! `audience` and `preserve` hold each node id at most once and every id names +//! a node (V4), so each is at most `Packet::MAX_NODES`; a predication's `roles` +//! hold each role id at most once, so at most the inventory's roles; and the +//! nodes themselves are at most `Packet::MAX_NODES` (V12). Whether a score may +//! rise because corpus text reached a packet is owned by the golf loop's +//! context builder and the scorer (ruling `leak-owned-by-flow-and-scorer`), +//! never by a text rule here, in `admit_revision` or in `PacketStore::admit`. + +use std::collections::BTreeSet; +use std::fmt; + +use crate::inventory::{Family, InvId, Inventory, InventoryEntry, Slot, ValueKind}; +use crate::packet::{ + Constraints, Context, Discourse, Extension, FeatValue, FeatureValue, NodeId, Packet, + PacketKind, Predication, Referent, Relation, SCHEMA_VERSION, +}; + +/// A broken rule, named. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum Refusal { + /// V1: the packet's schema_version is not the one these types define. + V1SchemaVersion { found: u32 }, + /// V2: no referents, predications or relations. + V2PayloadRequired, + /// V3: a node id repeats across referents and predications. + V3DuplicateNode { id: NodeId }, + /// V4: a reference names no node. + V4DanglingRef { id: NodeId, at: &'static str }, + /// V5: an inventory id does not resolve at the packet's revision. + V5Unresolved { id: InvId }, + /// V6: an inventory id is of the wrong family for its slot. + V6WrongFamily { + id: InvId, + expected: Family, + at: &'static str, + }, + /// V7: a feature or extension value does not fit its value kind. + V7ValueKind { what: String }, + /// V8: a feature is not declared for the slot it sits on. + V8NotApplicable { feature: InvId, slot: Slot }, + /// V10: the packet's inventory revision has no record. + V10UnknownRevision { rev: u32 }, + /// V11: a feature id repeats in one slot's list (`slot` is `None` for an + /// extension id repeated in the packet). + V11DuplicateFeature { feature: InvId, slot: Option }, + /// V11: a node id repeats in `participants`, `audience` or `preserve` + /// (`at` names the list). + V11DuplicateNodeRef { id: NodeId, at: &'static str }, + /// V11: a role id repeats in one predication's roles. + V11DuplicateRole { role: InvId, predication: NodeId }, + /// V12: more referents, predications and relations than `Packet::MAX_NODES`. + V12TooManyNodes { count: usize, max: usize }, +} + +impl Refusal { + pub fn rule(&self) -> &'static str { + match self { + Refusal::V1SchemaVersion { .. } => "V1", + Refusal::V2PayloadRequired => "V2", + Refusal::V3DuplicateNode { .. } => "V3", + Refusal::V4DanglingRef { .. } => "V4", + Refusal::V5Unresolved { .. } => "V5", + Refusal::V6WrongFamily { .. } => "V6", + Refusal::V7ValueKind { .. } => "V7", + Refusal::V8NotApplicable { .. } => "V8", + Refusal::V10UnknownRevision { .. } => "V10", + Refusal::V11DuplicateFeature { .. } + | Refusal::V11DuplicateNodeRef { .. } + | Refusal::V11DuplicateRole { .. } => "V11", + Refusal::V12TooManyNodes { .. } => "V12", + } + } +} + +impl fmt::Display for Refusal { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}: {self:?}", self.rule()) + } +} + +/// Whether `packet` may exist under `inventory`. All broken rules are returned; +/// a packet that passes yields the lowering view, which only this function builds. +pub fn validate(packet: &Packet, inventory: &Inventory) -> Result> { + let mut out = Vec::new(); + if packet.schema_version != SCHEMA_VERSION { + out.push(Refusal::V1SchemaVersion { + found: packet.schema_version, + }); + } + if packet.referents.is_empty() && packet.predications.is_empty() && packet.relations.is_empty() + { + out.push(Refusal::V2PayloadRequired); + } + let count = packet.referents.len() + packet.predications.len() + packet.relations.len(); + if count > Packet::MAX_NODES { + out.push(Refusal::V12TooManyNodes { + count, + max: Packet::MAX_NODES, + }); + } + let nodes = check_nodes(packet, &mut out); + check_refs(packet, &nodes, &mut out); + check_repeats(packet, &mut out); + if inventory.has_revision(packet.inventory_rev) { + Inventoried { + inv: inventory, + rev: packet.inventory_rev, + out: &mut out, + } + .check(packet); + } else { + out.push(Refusal::V10UnknownRevision { + rev: packet.inventory_rev, + }); + } + if out.is_empty() { + Ok(LoweringInput::of(packet)) + } else { + Err(out) + } +} + +fn check_nodes(packet: &Packet, out: &mut Vec) -> BTreeSet { + let mut nodes = BTreeSet::new(); + let ids = packet + .referents + .iter() + .map(|r| &r.id) + .chain(packet.predications.iter().map(|p| &p.id)); + for id in ids { + if !nodes.insert(id.clone()) { + out.push(Refusal::V3DuplicateNode { id: id.clone() }); + } + } + nodes +} + +fn check_refs(packet: &Packet, nodes: &BTreeSet, out: &mut Vec) { + let mut need = |id: &NodeId, at: &'static str| { + if !nodes.contains(id) { + out.push(Refusal::V4DanglingRef { id: id.clone(), at }); + } + }; + for p in &packet.predications { + for r in &p.roles { + need(&r.filler, "role filler"); + } + } + for r in &packet.relations { + need(&r.from, "relation from"); + need(&r.to, "relation to"); + } + if let Some(a) = &packet.discourse.addressee { + need(a, "addressee"); + } + for (list, at) in node_lists(packet) { + for id in list { + need(id, at); + } + } +} + +fn node_lists(packet: &Packet) -> [(&Vec, &'static str); 3] { + [ + (&packet.context.participants, "participant"), + (&packet.context.audience, "audience"), + (&packet.constraints.preserve, "preserve"), + ] +} + +/// V11 for the lists that hold nodes and roles: each node id once per node list, +/// each role id once per predication. +fn check_repeats(packet: &Packet, out: &mut Vec) { + for (list, at) in node_lists(packet) { + let mut seen = BTreeSet::new(); + for id in list { + if !seen.insert(id) { + out.push(Refusal::V11DuplicateNodeRef { id: id.clone(), at }); + } + } + } + for p in &packet.predications { + let mut seen = BTreeSet::new(); + for r in &p.roles { + if !seen.insert(r.role) { + out.push(Refusal::V11DuplicateRole { + role: r.role, + predication: p.id.clone(), + }); + } + } + } +} + +/// The inventory-dependent rules V5..V8 and V11, read at the packet's revision. +struct Inventoried<'a> { + inv: &'a Inventory, + rev: u32, + out: &'a mut Vec, +} + +impl<'a> Inventoried<'a> { + fn check(&mut self, p: &Packet) { + let Packet { + context, + discourse, + referents, + predications, + relations, + extensions, + .. + } = p; + let Context { + setting, medium, .. + } = context; + for id in setting.iter().chain(medium) { + self.slot(id, Family::Concept, "context concept"); + } + let Discourse { + speech_act, + features, + .. + } = discourse; + if let Some(id) = speech_act { + self.slot(id, Family::SpeechAct, "speech act"); + } + self.features(features, Slot::Discourse); + for Referent { + kind, + concept, + features, + .. + } in referents + { + self.slot(kind, Family::ReferentKind, "referent kind"); + self.slot(concept, Family::Concept, "referent concept"); + self.features(features, Slot::Referent); + } + for Predication { + concept, + frame, + roles, + features, + .. + } in predications + { + self.slot(concept, Family::Concept, "predication concept"); + self.slot(frame, Family::Frame, "frame"); + for r in roles { + self.slot(&r.role, Family::Role, "role"); + } + self.features(features, Slot::Predication); + } + for Relation { + relation, features, .. + } in relations + { + self.slot(relation, Family::Relation, "relation"); + self.features(features, Slot::Relation); + } + let mut seen = BTreeSet::new(); + for Extension { key, value } in extensions { + if !seen.insert(*key) { + self.out.push(Refusal::V11DuplicateFeature { + feature: *key, + slot: None, + }); + } + if let Some(entry) = self.slot(key, Family::Extension, "extension") + && !fits(entry.value_kind.as_ref(), value) + { + self.out.push(Refusal::V7ValueKind { + what: key.to_string(), + }); + } + } + } + + /// V5, then V6 when it resolved. + fn slot(&mut self, id: &InvId, want: Family, at: &'static str) -> Option<&'a InventoryEntry> { + let entry = self.resolve(id)?; + if entry.family() != want { + self.out.push(Refusal::V6WrongFamily { + id: *id, + expected: want, + at, + }); + return None; + } + Some(entry) + } + + fn resolve(&mut self, id: &InvId) -> Option<&'a InventoryEntry> { + let found = self.inv.resolve(id, self.rev); + if found.is_none() { + self.out.push(Refusal::V5Unresolved { id: *id }); + } + found + } + + fn features(&mut self, features: &[FeatureValue], slot: Slot) { + let mut seen = BTreeSet::new(); + for f in features { + if !seen.insert(f.feature) { + self.out.push(Refusal::V11DuplicateFeature { + feature: f.feature, + slot: Some(slot), + }); + } + let Some(entry) = self.slot(&f.feature, Family::Feature, "feature") else { + continue; + }; + if !entry.applies_to.contains(&slot) { + self.out.push(Refusal::V8NotApplicable { + feature: f.feature, + slot, + }); + } + if !fits(entry.value_kind.as_ref(), &f.value) { + self.out.push(Refusal::V7ValueKind { + what: f.feature.to_string(), + }); + } + } + } +} + +fn fits(kind: Option<&ValueKind>, value: &FeatValue) -> bool { + match (kind, value) { + (Some(ValueKind::Grade), FeatValue::Grade(_)) => true, + (Some(ValueKind::Options(labels)), FeatValue::Option(l)) => labels.contains(l), + (Some(ValueKind::Flag), FeatValue::Flag(_)) => true, + _ => false, + } +} + +/// What the lowerer sees: a validated packet's meaning, with no packet id, no +/// provenance (so no corpus item key, encoder or source) and no trace. Its +/// fields are private and `validate` is the only constructor, so a lowerer +/// cannot be handed an unvalidated packet or the key to an answer text. +/// +/// Each field has a doctest that reads it from outside the crate. A read +/// compiles exactly when the field is `pub`, so the doctest (which must fail to +/// compile) fails as soon as any one field is made public. A struct literal +/// cannot pin this: it fails to compile while any other field stays private, +/// and stable rustdoc does not check the error code. +/// +/// ```compile_fail +/// fn read(l: weksa::LoweringInput) { let _ = l.inventory_rev; } +/// ``` +/// +/// ```compile_fail +/// fn read(l: weksa::LoweringInput) { let _ = l.kind; } +/// ``` +/// +/// ```compile_fail +/// fn read(l: weksa::LoweringInput) { let _ = l.context; } +/// ``` +/// +/// ```compile_fail +/// fn read(l: weksa::LoweringInput) { let _ = l.discourse; } +/// ``` +/// +/// ```compile_fail +/// fn read(l: weksa::LoweringInput) { let _ = l.referents; } +/// ``` +/// +/// ```compile_fail +/// fn read(l: weksa::LoweringInput) { let _ = l.predications; } +/// ``` +/// +/// ```compile_fail +/// fn read(l: weksa::LoweringInput) { let _ = l.relations; } +/// ``` +/// +/// ```compile_fail +/// fn read(l: weksa::LoweringInput) { let _ = l.constraints; } +/// ``` +/// +/// ```compile_fail +/// fn read(l: weksa::LoweringInput) { let _ = l.extensions; } +/// ``` +/// +/// The constructor is private: a call from outside the crate compiles only if +/// `LoweringInput::of` is `pub`. The first block is the same code without the +/// call, so the second fails for that call and nothing else. +/// +/// ``` +/// fn build(p: &weksa::packet::Packet) { let _ = p; } +/// ``` +/// +/// ```compile_fail +/// fn build(p: &weksa::packet::Packet) { let _ = weksa::LoweringInput::of(p); } +/// ``` +/// +/// A lowering view is never read from bytes: `Deserialize` would let a caller +/// build one without `validate`. The first block proves the bound resolves for +/// a type that does implement it. +/// +/// ``` +/// fn de() {} +/// de::(); +/// ``` +/// +/// ```compile_fail +/// fn de() {} +/// de::(); +/// ``` +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct LoweringInput { + inventory_rev: u32, + kind: PacketKind, + context: Context, + discourse: Discourse, + referents: Vec, + predications: Vec, + relations: Vec, + constraints: Constraints, + extensions: Vec, +} + +impl LoweringInput { + fn of(p: &Packet) -> Self { + Self { + inventory_rev: p.inventory_rev, + kind: p.kind, + context: p.context.clone(), + discourse: p.discourse.clone(), + referents: p.referents.clone(), + predications: p.predications.clone(), + relations: p.relations.clone(), + constraints: p.constraints.clone(), + extensions: p.extensions.clone(), + } + } + + pub fn inventory_rev(&self) -> u32 { + self.inventory_rev + } + pub fn kind(&self) -> PacketKind { + self.kind + } + pub fn context(&self) -> &Context { + &self.context + } + pub fn discourse(&self) -> &Discourse { + &self.discourse + } + pub fn referents(&self) -> &[Referent] { + &self.referents + } + pub fn predications(&self) -> &[Predication] { + &self.predications + } + pub fn relations(&self) -> &[Relation] { + &self.relations + } + pub fn constraints(&self) -> &Constraints { + &self.constraints + } + pub fn extensions(&self) -> &[Extension] { + &self.extensions + } +} diff --git a/crates/weksa/tests/cli.rs b/crates/weksa/tests/cli.rs new file mode 100644 index 0000000..1d4f19a --- /dev/null +++ b/crates/weksa/tests/cli.rs @@ -0,0 +1,227 @@ +mod common; +use std::process::{Command, Output}; +use weksa::{InventoryStore, PacketStore}; + +fn weksa(dir: &std::path::Path, args: &[&str]) -> Output { + Command::new(env!("CARGO_BIN_EXE_weksa")) + .current_dir(dir) + .args(["--corpus", "./c.cc", "--inventory", "./i.cc"]) + .args(args) + .output() + .unwrap() +} + +fn out(o: &Output) -> String { + String::from_utf8_lossy(&o.stdout).into_owned() + &String::from_utf8_lossy(&o.stderr) +} + +const ADD: [&str; 14] = [ + "corpus", + "add", + "--slug", + "nibu", + "--text", + "Mind the hatch.", + "--speaker", + "nibu", + "--group", + "scene-1", + "--admitted-on", + "2026-10-29", + "--source", + "cast-authored", +]; + +#[test] +fn seed_add_dump_split_withdraw() { + let d = tempfile::tempdir().unwrap(); + let seeded = weksa(d.path(), &["inventory", "seed"]); + assert!(seeded.status.success(), "{}", out(&seeded)); + assert!(out(&seeded).contains("seeded rev:1")); + let again = weksa(d.path(), &["inventory", "seed"]); + assert!( + !again.status.success(), + "a second seed is refused: {}", + out(&again) + ); + let dump = out(&weksa(d.path(), &["inventory", "dump"])); + assert!( + dump.contains("rev:1") && dump.contains("speech_act:5") && dump.contains("warn"), + "{dump}" + ); + + // A cast-authored item needs its persona and fixture. + let missing = weksa(d.path(), &ADD); + assert!( + !missing.status.success() && out(&missing).contains("--persona"), + "{}", + out(&missing) + ); + let mut add = ADD.to_vec(); + add.extend(["--persona", "nibu", "--fixture", "f.yaml"]); + let added = weksa(d.path(), &add); + assert!( + added.status.success() && out(&added).contains("Added item:nibu:1"), + "{}", + out(&added) + ); + let second = weksa(d.path(), &add); + assert!( + out(&second).contains("Added item:nibu:2"), + "{}", + out(&second) + ); + + let dump = out(&weksa(d.path(), &["corpus", "dump"])); + assert!( + dump.contains("item:nibu:1") + && dump.contains("Mind the hatch.") + && dump.contains("CastAuthored"), + "{dump}" + ); + let split = out(&weksa(d.path(), &["split", "item:nibu:1"])); + assert!( + ["Train", "Validation", "Test"] + .iter() + .any(|s| split.trim() == *s), + "{split}" + ); + assert!(!weksa(d.path(), &["split", "item:nibu:9"]).status.success()); + + let withdrawn = weksa(d.path(), &["corpus", "withdraw", "item:nibu:1"]); + assert!(withdrawn.status.success()); + assert!(out(&weksa(d.path(), &["corpus", "dump"])).contains("withdrawn")); +} + +#[test] +fn model_written_and_public_domain_sources_name_their_provenance() { + let d = tempfile::tempdir().unwrap(); + let base = [ + "corpus", + "add", + "--slug", + "x", + "--text", + "t", + "--speaker", + "s", + "--group", + "g", + "--admitted-on", + "2026-10-29", + ]; + let run = |extra: &[&str]| { + let mut args = base.to_vec(); + args.extend_from_slice(extra); + weksa(d.path(), &args) + }; + let no_family = run(&["--source", "model-written"]); + assert!( + !no_family.status.success() && out(&no_family).contains("--model-family"), + "{}", + out(&no_family) + ); + let model = run(&["--source", "model-written", "--model-family", "m"]); + assert!(model.status.success(), "{}", out(&model)); + let no_work = run(&["--source", "public-domain", "--author", "a"]); + assert!( + !no_work.status.success() && out(&no_work).contains("--work"), + "{}", + out(&no_work) + ); + let pd = run(&[ + "--source", + "public-domain", + "--work", + "w", + "--author", + "a", + "--edition-url", + "https://example.org/w", + ]); + assert!(pd.status.success(), "{}", out(&pd)); + let bad_turn = run(&[ + "--source", + "operator-written", + "--prior-turn", + "no equals sign", + ]); + assert!( + !bad_turn.status.success() && out(&bad_turn).contains("speaker=text"), + "{}", + out(&bad_turn) + ); + let turn = run(&[ + "--source", + "operator-written", + "--prior-turn", + "player=hello", + ]); + assert!(turn.status.success(), "{}", out(&turn)); +} + +#[test] +fn validate_reports_each_packet() { + let d = tempfile::tempdir().unwrap(); + let w = common::world(); + let full = w.dir.path().join("i.cc"); + let packets = d.path().join("p.cc"); + let packet = common::good(); + PacketStore::open(&packets) + .unwrap() + .admit(packet.clone(), &w.inventory, &w.corpus) + .unwrap(); + + let validate = |inventory: &std::path::Path, extra: &[&str]| { + Command::new(env!("CARGO_BIN_EXE_weksa")) + .arg("--inventory") + .arg(inventory) + .arg("validate") + .arg(&packets) + .args(extra) + .output() + .unwrap() + }; + let ok = validate(&full, &[]); + assert!( + ok.status.success() && out(&ok).contains(&format!("ok {}", packet.id)), + "{}", + out(&ok) + ); + assert!(validate(&full, &[packet.id.as_str()]).status.success()); + assert!(!validate(&full, &["packet:missing"]).status.success()); + // Against an inventory that only reaches revision 1 the same packet is refused, by rule. + let short = d.path().join("short.cc"); + let (rev1, entries1) = weksa::inventory::seed(); + InventoryStore::open(&short) + .unwrap() + .admit_revision(rev1, entries1) + .unwrap(); + let refused = validate(&short, &[]); + assert!( + !refused.status.success() && out(&refused).contains("V10"), + "{}", + out(&refused) + ); +} + +#[test] +fn schema_writes_the_published_files() { + let d = tempfile::tempdir().unwrap(); + let o = weksa(d.path(), &["schema", "--out", "s"]); + assert!(o.status.success(), "{}", out(&o)); + let mut names: Vec = std::fs::read_dir(d.path().join("s")) + .unwrap() + .map(|e| e.unwrap().file_name().into_string().unwrap()) + .collect(); + names.sort(); + assert_eq!( + names, + [ + "weksa.corpus_item.schema.json", + "weksa.interlingua_packet.schema.json", + "weksa.inventory_entry.schema.json", + "weksa.inventory_revision.schema.json" + ] + ); +} diff --git a/crates/weksa/tests/common/mod.rs b/crates/weksa/tests/common/mod.rs new file mode 100644 index 0000000..7f7aea0 --- /dev/null +++ b/crates/weksa/tests/common/mod.rs @@ -0,0 +1,274 @@ +#![allow(dead_code)] + +use std::sync::OnceLock; + +use tempfile::TempDir; +use weksa::inventory::{Change, seed}; +use weksa::packet::*; +use weksa::{ + CorpusItem, CorpusStore, InvId, Inventory, InventoryEntry, InventoryRevision, InventoryStore, + OptionLabel, Slot, Source, SplitScheme, ValueKind, +}; + +pub const PERSON: &str = "referent_kind:1"; +pub const AGENT: &str = "role:1"; +pub const PATIENT: &str = "role:2"; +pub const WARN: &str = "speech_act:5"; +pub const POLARITY: &str = "feature:1"; +pub const MODALITY: &str = "feature:4"; +pub const HEDGE: &str = "feature:6"; +pub const IRONY: &str = "feature:11"; +/// Introduced at rev 2. +pub const CONCEPT_A: &str = "concept:1"; +pub const CONCEPT_B: &str = "concept:2"; +/// Introduced at rev 2, retired at rev 3. +pub const CONCEPT_RETIRED: &str = "concept:3"; +/// Introduced at rev 4. +pub const CONCEPT_LATE: &str = "concept:4"; +pub const FRAME: &str = "frame:1"; +/// An extension entry of value kind Grade, introduced at rev 2. +pub const EXT: &str = "extension:1"; +/// A Grade feature introduced at rev 2 that sits in every slot. +pub const FEAT_ALL: &str = "feature:50"; + +pub const PACKET_ID: &str = "packet:0123456789abcdef"; +pub const ENCODER_ID: &str = "encoder:fedcba9876543210"; + +pub fn iv(s: &str) -> InvId { + s.parse().unwrap() +} + +pub fn nd(s: &str) -> NodeId { + NodeId::parse(s).unwrap() +} + +pub fn lab(s: &str) -> OptionLabel { + OptionLabel::parse(s).unwrap() +} + +pub fn grade(v: u16) -> Grade { + Grade::new(v).unwrap() +} + +pub fn group_in_bucket(want: u64) -> String { + (0..100_000) + .map(|i| format!("g{i}")) + .find(|g| SplitScheme::V1.bucket(g) == want) + .unwrap_or_else(|| panic!("no group lands in bucket {want}")) +} + +pub fn cast() -> Source { + Source::CastAuthored { + persona: "nibu".into(), + fixture: "f.yaml".into(), + } +} + +pub fn model() -> Source { + Source::ModelWritten { + model_family: "m".into(), + prompt_digest: None, + } +} + +pub fn item_in(key: &str, text: &str, group: &str, source: Source) -> CorpusItem { + CorpusItem { + id: key.into(), + text: text.into(), + language: "en-US".into(), + speaker: "nibu".into(), + group: group.into(), + prior_turns: vec![], + source, + admitted_on: "2026-10-29".into(), + supersedes: None, + withdrawn: false, + } +} + +pub fn entry(id: &str, introduced: u32) -> InventoryEntry { + InventoryEntry { + id: iv(id), + gloss: id.into(), + value_kind: None, + applies_to: vec![], + introduced_rev: introduced, + retired_rev: None, + } +} + +pub fn feature(id: &str, kind: ValueKind, slots: &[Slot]) -> InventoryEntry { + InventoryEntry { + value_kind: Some(kind), + applies_to: slots.to_vec(), + ..entry(id, 1) + } +} + +pub fn rev_with(changes: Vec, n: u32, parent: Option) -> InventoryRevision { + InventoryRevision { + n, + parent, + changes, + frozen: false, + } +} + +/// Admits `entries` as revision `n` and returns nothing; the stores are the only door. +pub fn admit(store: &mut InventoryStore, n: u32, entries: Vec) { + let changes = entries.iter().map(|e| Change::Add(e.id)).collect(); + let parent = n.checked_sub(1).filter(|p| *p > 0); + store + .admit_revision(rev_with(changes, n, parent), entries) + .unwrap(); +} + +/// Stores a test can admit through: a corpus holding `item:nibu:1` in a Train +/// group, and an inventory of revisions 1 (the seed), 2 (concepts A, B, +/// RETIRED, a frame, a Grade extension), 3 (RETIRED retired) and 4 (LATE). +pub struct World { + pub dir: TempDir, + pub inventory: InventoryStore, + pub corpus: CorpusStore, +} + +impl World { + pub fn inv(&self) -> Inventory { + self.inventory.inventory().unwrap() + } +} + +pub fn world() -> World { + let dir = tempfile::tempdir().unwrap(); + let mut corpus = CorpusStore::open(&dir.path().join("c.cc")).unwrap(); + let train = group_in_bucket(10); + corpus + .add(item_in( + "item:nibu:1", + "Nibu says mind the hatch.", + &train, + cast(), + )) + .unwrap(); + let mut inventory = InventoryStore::open(&dir.path().join("i.cc")).unwrap(); + let (rev1, entries) = seed(); + inventory.admit_revision(rev1, entries).unwrap(); + let mut ext = entry(EXT, 2); + ext.value_kind = Some(ValueKind::Grade); + admit( + &mut inventory, + 2, + vec![ + entry(CONCEPT_A, 2), + entry(CONCEPT_B, 2), + entry(CONCEPT_RETIRED, 2), + entry(FRAME, 2), + ext, + InventoryEntry { + introduced_rev: 2, + ..feature( + FEAT_ALL, + ValueKind::Grade, + &[ + Slot::Discourse, + Slot::Referent, + Slot::Predication, + Slot::Relation, + ], + ) + }, + ], + ); + inventory + .admit_revision( + rev_with(vec![Change::Retire(iv(CONCEPT_RETIRED))], 3, Some(2)), + vec![], + ) + .unwrap(); + admit(&mut inventory, 4, vec![entry(CONCEPT_LATE, 4)]); + World { + dir, + inventory, + corpus, + } +} + +/// The world's inventory, built once. +pub fn inventory() -> Inventory { + static INV: OnceLock = OnceLock::new(); + INV.get_or_init(|| world().inv()).clone() +} + +pub fn good() -> Packet { + Packet { + id: PacketId::parse(PACKET_ID).unwrap(), + schema_version: SCHEMA_VERSION, + inventory_rev: 2, + kind: PacketKind::DialogueLine, + provenance: Provenance { + source: ProvenanceSource::Authored, + corpus_item: Some("item:nibu:1".into()), + encoder: EncoderId::parse(ENCODER_ID).unwrap(), + confidence: grade(900), + }, + context: Context { + setting: Some(iv(CONCEPT_A)), + medium: Some(iv(CONCEPT_B)), + participants: vec![nd("n1")], + audience: vec![nd("n1")], + }, + discourse: Discourse { + speech_act: Some(iv(WARN)), + addressee: Some(nd("n1")), + features: vec![FeatureValue { + feature: iv(HEDGE), + value: FeatValue::Grade(grade(300)), + }], + }, + referents: vec![Referent { + id: nd("n1"), + kind: iv(PERSON), + concept: iv(CONCEPT_A), + name: Some(ProperName::parse("Nibu").unwrap()), + features: vec![], + }], + predications: vec![Predication { + id: nd("n2"), + concept: iv(CONCEPT_B), + frame: iv(FRAME), + roles: vec![ + RoleFill { + role: iv(AGENT), + filler: nd("n1"), + }, + RoleFill { + role: iv(PATIENT), + filler: nd("n1"), + }, + ], + features: vec![FeatureValue { + feature: iv(MODALITY), + value: FeatValue::Option(lab("potential")), + }], + }], + relations: vec![Relation { + relation: iv("relation:8"), + from: nd("n2"), + to: nd("n1"), + features: vec![], + }], + constraints: Constraints { + preserve: vec![nd("n1")], + deterministic: true, + max_latency_ms: Some(50), + }, + extensions: vec![Extension { + key: iv(EXT), + value: FeatValue::Grade(grade(3)), + }], + trace: Trace { + notes: vec!["a note".into()], + rejected_readings: vec![], + }, + } +} diff --git a/crates/weksa/tests/forms.rs b/crates/weksa/tests/forms.rs new file mode 100644 index 0000000..eac5dc0 --- /dev/null +++ b/crates/weksa/tests/forms.rs @@ -0,0 +1,248 @@ +mod common; +use common::*; +use weksa::packet::{EncoderId, Grade, NodeId, PacketId, ProperName, Referent}; +use weksa::{Family, InvId, OptionLabel, OptionSet}; + +/// Soul's probe: a whole corpus line, and the snake_case form that would +/// pass for an identifier. +const LINE: &str = "Take your hand off that control unless you want your nervous system to become my least useful circuit diagram."; +const SNAKE: &str = "take_your_hand_off_that_control_unless_you_want_your_nervous_system_to_become_my_least_useful_circuit_diagram"; + +#[test] +fn forms_refuse_surface_text() { + for text in [LINE, SNAKE, "", " "] { + assert!(NodeId::parse(text).is_err(), "NodeId {text:?}"); + assert!(PacketId::parse(text).is_err(), "PacketId {text:?}"); + assert!(EncoderId::parse(text).is_err(), "EncoderId {text:?}"); + assert!(InvId::parse(text).is_err(), "InvId {text:?}"); + assert!(OptionLabel::parse(text).is_err(), "OptionLabel {text:?}"); + } + // Soul's extension key `nibu.` is not an inventory id either. + assert!(InvId::parse(&format!("extension:{SNAKE}")).is_err()); + assert!(InvId::parse(&format!("nibu.{SNAKE}")).is_err()); + // NodeId is n, k canonical 1..=9999. + for ok in ["n1", "n10", "n9999"] { + assert_eq!(NodeId::parse(ok).unwrap().as_str(), ok); + } + for bad in [ + "n0", "n01", "n+1", "n-1", "n10000", "n", "1", "N1", "n1 ", "r1", "n1x", + ] { + assert!(NodeId::parse(bad).is_err(), "{bad:?}"); + } + // PacketId and EncoderId are a prefix and exactly 16 lowercase hex digits. + assert!(PacketId::parse(PACKET_ID).is_ok()); + assert!(EncoderId::parse(ENCODER_ID).is_ok()); + for bad in [ + "packet:0123456789abcde", + "packet:0123456789abcdef0", + "packet:0123456789ABCDEF", + "packet:0123456789abcdeg", + "encoder:0123456789abcdef", + "0123456789abcdef", + "packet:item:nibu:1:r2:abc", + ] { + assert!(PacketId::parse(bad).is_err(), "{bad:?}"); + } + assert!(EncoderId::parse("packet:0123456789abcdef").is_err()); + assert!(EncoderId::parse("encoder:0123456789abcdeg").is_err()); + assert!(EncoderId::parse("encoder:0123456789ABCDEF").is_err()); + assert!(EncoderId::parse("encoder:0123456789abcde").is_err()); + assert!(EncoderId::parse("encoder:0123456789abcdef0").is_err()); +} + +#[test] +fn a_refusal_names_the_form_and_the_value_and_a_form_reads_back_as_written() { + let e = NodeId::parse("n0").unwrap_err(); + assert_eq!(e.form, "NodeId"); + assert_eq!(e.got, "n0"); + assert_eq!(e.to_string(), "\"n0\" is not a NodeId"); + assert_eq!( + Grade::new(1001).unwrap_err().to_string(), + "\"1001\" is not a Grade" + ); + assert_eq!(EncoderId::parse(ENCODER_ID).unwrap().as_str(), ENCODER_ID); + assert_eq!(PacketId::parse(PACKET_ID).unwrap().as_str(), PACKET_ID); + assert_eq!( + EncoderId::parse(ENCODER_ID).unwrap().to_string(), + ENCODER_ID + ); + assert_eq!(lab("potential").as_str(), "potential"); +} + +#[test] +fn a_malformed_value_does_not_deserialize() { + // A node id that is the snake line fails where a store would read it. + let mut referent = serde_json::to_value(good().referents[0].clone()).unwrap(); + assert!(serde_json::from_value::(referent.clone()).is_ok()); + referent["id"] = SNAKE.into(); + assert!(serde_json::from_value::(referent.clone()).is_err()); + referent["id"] = "n1".into(); + referent["name"] = LINE.into(); + assert!(serde_json::from_value::(referent.clone()).is_err()); + referent["name"] = "Nibu".into(); + referent["concept"] = "concept:01".into(); + assert!(serde_json::from_value::(referent).is_err()); + assert!(serde_json::from_str::("1001").is_err()); + assert_eq!(serde_json::from_str::("1000").unwrap().get(), 1000); + assert!(serde_json::from_str::("[]").is_err()); +} + +#[test] +fn proper_name_bounds() { + for ok in [ + "Nibu", + "Brokkr Ivaldi", + "O'Neil", + "O\u{2019}Neil", + "Jean-Luc", + "A B C D", + ] { + assert_eq!(ProperName::parse(ok).unwrap().as_str(), ok); + } + // A caseless script is a name; so is a capital beyond ASCII. + assert!(ProperName::parse("\u{5c71}\u{7530}").is_ok()); + assert!(ProperName::parse("\u{c9}owyn").is_ok()); + for bad in [ + LINE, + "A B C D E", + "Nibu.", + "Nibu, Ivaldi", + "Nibu Ivaldi", + " Nibu", + "Nibu ", + "", + "Nibu2", + "'Nibu", + "-Nibu", + "Nibu-", + "Nibu'", + "Nibu_Ivaldi", + ] { + assert!(ProperName::parse(bad).is_err(), "{bad:?}"); + } + // 24 chars a token, 4 tokens and 64 bytes are the bounds. + let token24 = format!("A{}", "b".repeat(23)); + assert!(ProperName::parse(&token24).is_ok()); + assert!(ProperName::parse(&format!("{token24}b")).is_err()); + let sized = |lens: [usize; 4]| { + lens.iter() + .map(|n| format!("A{}", "b".repeat(n - 1))) + .collect::>() + .join(" ") + }; + let at_limit = sized([15, 15, 15, 16]); + assert_eq!(at_limit.len(), 64); + assert!(ProperName::parse(&at_limit).is_ok()); + let over = sized([15, 15, 15, 17]); + assert_eq!(over.len(), 65); + assert!(ProperName::parse(&over).is_err()); + // The byte bound counts bytes, not characters. + let wide = format!( + "{0} {0} {0}", + "\u{e9}".repeat(11).replace('\u{e9}', "\u{c9}") + ); + assert!(wide.chars().count() < 64 && wide.len() > 64); + assert!(ProperName::parse(&wide).is_err()); +} + +#[test] +fn inv_id_canonical() { + for bad in [ + "concept:01", + "concept:+1", + "concept:0", + "concept:-1", + "concept:", + "concept:1x", + "concept: 1", + "nofamily:1", + "Concept:1", + "concept", + "concept:99999999999", + "concept:1:2", + ] { + assert!(InvId::parse(bad).is_err(), "{bad:?}"); + } + let id = InvId::parse("concept:1").unwrap(); + assert_eq!(id.family(), Family::Concept); + assert_eq!(id.to_string(), "concept:1"); + for f in Family::ALL { + let id = InvId::parse(&format!("{}:12", f.prefix())).unwrap(); + assert_eq!(id.family(), f); + assert_eq!(id.to_string(), format!("{}:12", f.prefix())); + } + assert_eq!(iv("extension:1").family(), Family::Extension); +} + +#[test] +fn options_bounded() { + let labels = |n: usize| (0..n).map(|i| lab(&format!("l{i}"))).collect::>(); + assert!(OptionSet::new(labels(0)).is_err()); + assert!(OptionSet::new(labels(1)).is_ok()); + assert!(OptionSet::new(labels(16)).is_ok()); + assert!(OptionSet::new(labels(17)).is_err()); + assert!(OptionSet::new(vec![lab("a"), lab("a")]).is_err()); + assert!(OptionSet::new(vec![lab("a"), lab("b"), lab("a")]).is_err()); + // Soul's probe: the snake line is not a label; labels are short snake words. + assert!(OptionLabel::parse(SNAKE).is_err()); + assert!(OptionLabel::parse(&"a".repeat(24)).is_ok()); + assert!(OptionLabel::parse(&"a".repeat(25)).is_err()); + for bad in ["A", "1a", "_a", "a-b", "a b", "a.b", "\u{e9}"] { + assert!(OptionLabel::parse(bad).is_err(), "{bad:?}"); + } + for ok in ["a", "a1", "a_b", "potential"] { + assert!(OptionLabel::parse(ok).is_ok(), "{ok:?}"); + } +} + +#[test] +fn grade_bound_is_the_type() { + assert!(Grade::new(1001).is_err()); + assert_eq!(Grade::new(1000).unwrap().get(), 1000); + assert_eq!(Grade::new(0).unwrap().get(), 0); + // The published schema carries the same bound. + let (_, json) = weksa::schema::published() + .into_iter() + .find(|(n, _)| n == "weksa.interlingua_packet.schema.json") + .unwrap(); + let schema: serde_json::Value = serde_json::from_str(&json).unwrap(); + let grade = &schema["$defs"]["Grade"]; + assert_eq!(grade["maximum"], 1000, "{grade}"); + assert_eq!(grade["minimum"], 0, "{grade}"); + assert_eq!(schema["$defs"]["NodeId"]["pattern"], "^n[1-9][0-9]{0,3}$"); +} + +#[test] +fn proper_name_bound_in_chars() { + // The token bound counts characters: 24 two-byte letters are 48 bytes and parse. + let cyrillic = |n: usize| "\u{411}".repeat(n); + assert_eq!(cyrillic(24).len(), 48); + assert!(ProperName::parse(&cyrillic(24)).is_ok()); + assert!(ProperName::parse(&cyrillic(25)).is_err()); + // The 64-byte bound still counts bytes: two 48-byte tokens are 97 bytes. + assert!(ProperName::parse(&format!("{0} {0}", cyrillic(24))).is_err()); +} + +#[test] +fn proper_name_is_form_only() { + // A name is bounded text: no case rule, so lowercase, hyphenated and + // caseless-script tokens parse. + for ok in [ + "nibu", + "Nibu ivaldi", + "take-your-hand-off-that", + "Take-your-hand-off-that", + "\u{3131}-control-unless-you", + "\u{5c71}\u{7530}", + ] { + assert_eq!(ProperName::parse(ok).unwrap().as_str(), ok); + } + // The bounds still hold: 5 tokens, 65 bytes, a sentence's punctuation. + assert!(ProperName::parse("a b c d e").is_err()); + assert!(ProperName::parse(&"a".repeat(25)).is_err()); + let token = |n: usize| "a".repeat(n); + let over = [token(15), token(15), token(15), token(17)].join(" "); + assert_eq!(over.len(), 65); + assert!(ProperName::parse(&over).is_err()); + assert!(ProperName::parse("take your hand, now.").is_err()); +} diff --git a/crates/weksa/tests/split_schema.rs b/crates/weksa/tests/split_schema.rs new file mode 100644 index 0000000..d23737b --- /dev/null +++ b/crates/weksa/tests/split_schema.rs @@ -0,0 +1,150 @@ +use weksa::corpus::{CorpusItem, Source}; +use weksa::split::fnv1a64; +use weksa::{Split, SplitScheme}; + +fn item(group: &str, source: Source) -> CorpusItem { + CorpusItem { + id: "item:t:1".into(), + text: "t".into(), + language: "en-US".into(), + speaker: "s".into(), + group: group.into(), + prior_turns: vec![], + source, + admitted_on: "2026-10-29".into(), + supersedes: None, + withdrawn: false, + } +} + +fn cast() -> Source { + Source::CastAuthored { + persona: "p".into(), + fixture: "f".into(), + } +} + +fn model() -> Source { + Source::ModelWritten { + model_family: "m".into(), + prompt_digest: None, + } +} + +fn group_in_bucket(want: u64) -> String { + (0..100_000) + .map(|i| format!("g{i}")) + .find(|g| SplitScheme::V1.bucket(g) == want) + .unwrap_or_else(|| panic!("no group lands in bucket {want}")) +} + +#[test] +fn fnv1a64_matches_the_reference_vectors() { + assert_eq!(fnv1a64(b""), 0xcbf2_9ce4_8422_2325); + assert_eq!(fnv1a64(b"a"), 0xaf63_dc4c_8601_ec8c); + assert_eq!(fnv1a64(b"foobar"), 0x8594_4171_f739_67e8); +} + +#[test] +fn p1_split_deterministic() { + let a = SplitScheme::V1.split(&item("scene-9", cast())); + for _ in 0..3 { + assert_eq!(SplitScheme::V1.split(&item("scene-9", cast())), a); + } + // The split reads the group and nothing else: speaker, text and id do not move it. + let mut other = item("scene-9", cast()); + other.speaker = "someone".into(); + other.text = "different".into(); + other.id = "item:other:7".into(); + assert_eq!(SplitScheme::V1.split(&other), a); + // The bucket is the pinned hash of the prefixed group. + assert_eq!( + SplitScheme::V1.bucket("x"), + fnv1a64(b"weksa-split-v1:x") % 100 + ); +} + +#[test] +fn split_boundaries() { + let s = SplitScheme::V1; + assert_eq!(s.of_bucket(0), Split::Train); + assert_eq!(s.of_bucket(39), Split::Train); + assert_eq!(s.of_bucket(40), Split::Validation); + assert_eq!(s.of_bucket(79), Split::Validation); + assert_eq!(s.of_bucket(80), Split::Test); + assert_eq!(s.of_bucket(99), Split::Test); + for (bucket, want) in [ + (39, Split::Train), + (40, Split::Validation), + (79, Split::Validation), + (80, Split::Test), + ] { + assert_eq!( + s.split(&item(&group_in_bucket(bucket), cast())), + Some(want), + "bucket {bucket}" + ); + } +} + +#[test] +fn p2_model_written_in_a_test_group_is_excluded() { + let test_group = group_in_bucket(95); + assert_eq!( + SplitScheme::V1.split(&item(&test_group, cast())), + Some(Split::Test) + ); + // A model line of the test meaning is in no split; it is never moved to train. + assert_eq!(SplitScheme::V1.split(&item(&test_group, model())), None); + // Model-written items in the other splits keep them. + assert_eq!( + SplitScheme::V1.split(&item(&group_in_bucket(10), model())), + Some(Split::Train) + ); + assert_eq!( + SplitScheme::V1.split(&item(&group_in_bucket(60), model())), + Some(Split::Validation) + ); +} + +#[test] +fn p3_split_ratio() { + let mut counts = [0usize; 3]; + for i in 0..10_000 { + match SplitScheme::V1.split(&item(&format!("scene-{i}"), cast())) { + Some(Split::Train) => counts[0] += 1, + Some(Split::Validation) => counts[1] += 1, + Some(Split::Test) => counts[2] += 1, + None => panic!("a cast item is always in a split"), + } + } + let near = |got: usize, want_percent: usize| got.abs_diff(want_percent * 100) <= 200; + assert!( + near(counts[0], 40) && near(counts[1], 40) && near(counts[2], 20), + "{counts:?}" + ); +} + +#[test] +fn published_schemas_match() { + let dir = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../schemas/cultnet"); + let published = weksa::schema::published(); + let mut names: Vec = published.iter().map(|(n, _)| n.clone()).collect(); + names.sort(); + let mut on_disk: Vec = std::fs::read_dir(&dir) + .unwrap() + .map(|e| e.unwrap().file_name().into_string().unwrap()) + .filter(|n| n.ends_with(".schema.json")) + .collect(); + on_disk.sort(); + assert_eq!( + on_disk, names, + "schemas/cultnet holds exactly the generated files" + ); + for (name, json) in published { + let held = std::fs::read_to_string(dir.join(&name)) + .unwrap() + .replace("\r\n", "\n"); + assert_eq!(held, json, "{name} drifted; run `weksa schema`"); + } +} diff --git a/crates/weksa/tests/stores.rs b/crates/weksa/tests/stores.rs new file mode 100644 index 0000000..5366191 --- /dev/null +++ b/crates/weksa/tests/stores.rs @@ -0,0 +1,636 @@ +mod common; +use common::*; +use tempfile::TempDir; +use weksa::corpus::{CorpusItem, Source, Turn}; +use weksa::inventory::{Change, seed}; +use weksa::{ + AddOutcome, CorpusStore, Family, InventoryEntry, InventoryRevision, InventoryStore, + PacketStore, Refusal, Slot, SplitScheme, StoreError, StoreRefusal, ValueKind, +}; + +fn item(key: &str, text: &str) -> CorpusItem { + CorpusItem { + prior_turns: vec![Turn { + speaker: "player".into(), + text: "hello".into(), + }], + ..item_in(key, text, "scene-1", cast()) + } +} + +fn refusal(r: Result) -> StoreRefusal { + match r { + Err(StoreError::Refused(r)) => r, + other => panic!("expected a store refusal, got {other:?}"), + } +} + +fn dir() -> TempDir { + tempfile::tempdir().unwrap() +} + +fn empty_corpus(d: &TempDir) -> CorpusStore { + CorpusStore::open(&d.path().join("empty-corpus.cc")).unwrap() +} + +fn malformed(key: &str, why: &'static str) -> StoreRefusal { + StoreRefusal::Malformed { + key: key.into(), + why, + } +} + +#[test] +fn s1_item_text_immutable() { + let d = dir(); + let path = d.path().join("c.cc"); + let mut store = CorpusStore::open(&path).unwrap(); + assert_eq!( + store.add(item("item:nibu:1", "Mind the hatch.")).unwrap(), + AddOutcome::Added + ); + // The same text is idempotent, even from a reopened store. + let mut store = CorpusStore::open(&path).unwrap(); + assert_eq!( + store.add(item("item:nibu:1", "Mind the hatch.")).unwrap(), + AddOutcome::Unchanged + ); + assert_eq!( + refusal(store.add(item("item:nibu:1", "Mind the hatch!"))), + StoreRefusal::S1ItemTextImmutable { + key: "item:nibu:1".into() + } + ); + assert_eq!( + store.get("item:nibu:1").unwrap().unwrap().text, + "Mind the hatch." + ); +} + +#[test] +fn corpus_admission_rules() { + let d = dir(); + let mut store = CorpusStore::open(&d.path().join("c.cc")).unwrap(); + assert_eq!( + refusal(store.add(item("nibu:1", "x"))), + malformed("nibu:1", "item key is not item::") + ); + assert_eq!( + refusal(store.add(item("item:nibu:x", "x"))), + malformed("item:nibu:x", "item key is not item::") + ); + let mut born_withdrawn = item("item:nibu:1", "x"); + born_withdrawn.withdrawn = true; + assert_eq!( + refusal(store.add(born_withdrawn)), + malformed("item:nibu:1", "an item is admitted live, then withdrawn") + ); + let mut orphan = item("item:nibu:1", "x"); + orphan.supersedes = Some("item:nibu:9".into()); + assert_eq!( + refusal(store.add(orphan)), + malformed("item:nibu:1", "supersedes an item that does not exist") + ); + // Nothing was written by the refusals. + assert!(store.all().unwrap().is_empty()); + // A correction is a new item that supersedes the old one. + store.add(item("item:nibu:1", "old")).unwrap(); + let mut fix = item("item:nibu:2", "new"); + fix.supersedes = Some("item:nibu:1".into()); + assert_eq!(store.add(fix).unwrap(), AddOutcome::Added); + assert_eq!( + refusal(store.withdraw("item:nibu:7")), + malformed("item:nibu:7", "no such item") + ); +} + +#[test] +fn s5_supersede_keeps_the_group() { + let d = dir(); + let mut store = CorpusStore::open(&d.path().join("c.cc")).unwrap(); + store.add(item("item:nibu:1", "old")).unwrap(); + let mut elsewhere = item("item:nibu:2", "new"); + elsewhere.supersedes = Some("item:nibu:1".into()); + elsewhere.group = "scene-2".into(); + assert_eq!( + refusal(store.add(elsewhere)), + StoreRefusal::S5SupersedeGroup { + key: "item:nibu:2".into(), + expected: "scene-1".into(), + got: "scene-2".into() + } + ); + assert!(store.get("item:nibu:2").unwrap().is_none()); + let mut same = item("item:nibu:2", "new"); + same.supersedes = Some("item:nibu:1".into()); + assert_eq!(store.add(same).unwrap(), AddOutcome::Added); +} + +#[test] +fn withdrawing_keeps_the_text_and_the_number() { + let d = dir(); + let path = d.path().join("c.cc"); + let mut store = CorpusStore::open(&path).unwrap(); + store.add(item("item:nibu:1", "keep me")).unwrap(); + assert_eq!(store.next_key("nibu").unwrap(), "item:nibu:2"); + assert_eq!(store.next_key("other").unwrap(), "item:other:1"); + store.withdraw("item:nibu:1").unwrap(); + store.withdraw("item:nibu:1").unwrap(); + let store = CorpusStore::open(&path).unwrap(); + let got = store.get("item:nibu:1").unwrap().unwrap(); + assert!(got.withdrawn); + assert_eq!(got.text, "keep me"); + assert_eq!(store.next_key("nibu").unwrap(), "item:nibu:2"); +} + +fn entries_for(rev: u32, ids: &[&str]) -> (InventoryRevision, Vec) { + let entries = ids.iter().map(|id| entry(id, rev)).collect(); + let changes = ids.iter().map(|id| Change::Add(iv(id))).collect(); + ( + rev_with(changes, rev, if rev == 1 { None } else { Some(rev - 1) }), + entries, + ) +} + +fn seeded(path: &std::path::Path) -> InventoryStore { + let mut store = InventoryStore::open(path).unwrap(); + let (rev, entries) = seed(); + store.admit_revision(rev, entries).unwrap(); + store +} + +#[test] +fn s2_revision_chain() { + let d = dir(); + let mut store = InventoryStore::open(&d.path().join("i.cc")).unwrap(); + // The first revision is 1 with no parent. + let (rev, entries) = entries_for(2, &["concept:1"]); + assert_eq!( + refusal(store.admit_revision(rev, entries)), + StoreRefusal::S2RevisionChain { + expected: 1, + got: 2 + } + ); + let (mut rev, entries) = entries_for(1, &["concept:1"]); + rev.parent = Some(0); + assert_eq!( + refusal(store.admit_revision(rev, entries)), + StoreRefusal::S2RevisionChain { + expected: 1, + got: 1 + } + ); + let (rev, entries) = entries_for(1, &["concept:1"]); + store.admit_revision(rev, entries).unwrap(); + // Replaying n, skipping n, or naming the wrong parent is refused. + let (rev, entries) = entries_for(1, &["concept:2"]); + assert_eq!( + refusal(store.admit_revision(rev, entries)), + StoreRefusal::S2RevisionChain { + expected: 2, + got: 1 + } + ); + let (rev, entries) = entries_for(3, &["concept:2"]); + assert_eq!( + refusal(store.admit_revision(rev, entries)), + StoreRefusal::S2RevisionChain { + expected: 2, + got: 3 + } + ); + let (mut rev, entries) = entries_for(2, &["concept:2"]); + rev.parent = None; + assert_eq!( + refusal(store.admit_revision(rev, entries)), + StoreRefusal::S2RevisionChain { + expected: 2, + got: 2 + } + ); + let (rev, entries) = entries_for(2, &["concept:2"]); + store.admit_revision(rev, entries).unwrap(); + assert_eq!(store.inventory().unwrap().latest().unwrap().n, 2); +} + +#[test] +fn s3_frozen_is_final() { + let d = dir(); + let mut store = InventoryStore::open(&d.path().join("i.cc")).unwrap(); + let (mut rev, entries) = entries_for(1, &["concept:1"]); + rev.frozen = true; + store.admit_revision(rev, entries).unwrap(); + let (rev, entries) = entries_for(2, &["concept:2"]); + assert_eq!( + refusal(store.admit_revision(rev, entries)), + StoreRefusal::S3Frozen { frozen: 1 } + ); + assert_eq!(store.inventory().unwrap().latest().unwrap().n, 1); +} + +#[test] +fn s4_entry_introduced_by_its_revision() { + let d = dir(); + let mut store = InventoryStore::open(&d.path().join("i.cc")).unwrap(); + let (rev, mut entries) = entries_for(1, &["concept:1"]); + entries[0].introduced_rev = 2; + assert_eq!( + refusal(store.admit_revision(rev, entries)), + StoreRefusal::S4IntroducedRev { + id: iv("concept:1"), + introduced: 2, + revision: 1 + } + ); + assert!(store.inventory().unwrap().latest().is_none()); +} + +#[test] +fn revision_change_rules() { + let d = dir(); + let mut store = seeded(&d.path().join("i.cc")); + // An added id with no entry. + let rev = rev_with(vec![Change::Add(iv("concept:1"))], 2, Some(1)); + assert_eq!( + refusal(store.admit_revision(rev, vec![])), + malformed("concept:1", "added id has no entry") + ); + // An entry the changes do not name. + let (rev, mut entries) = entries_for(2, &["concept:1"]); + entries.push(entry("concept:2", 2)); + assert_eq!( + refusal(store.admit_revision(rev, entries)), + malformed("concept:2", "entry is not in the revision's changes") + ); + // An id already in the inventory, and one added twice. + let (rev, entries) = entries_for(2, &["role:1"]); + assert_eq!( + refusal(store.admit_revision(rev, entries)), + malformed("role:1", "id is already in the inventory") + ); + let (mut rev, entries) = entries_for(2, &["concept:1"]); + rev.changes.push(Change::Add(iv("concept:1"))); + assert_eq!( + refusal(store.admit_revision(rev, entries)), + malformed("concept:1", "id is already in the inventory") + ); + // A malformed entry: a feature with no value kind, a non-feature with one, + // an extension with no value kind or with slots, an entry born retired. + let shape = "entry shape does not match its family"; + let (rev, mut entries) = entries_for(2, &["feature:50"]); + entries[0].applies_to = vec![Slot::Discourse]; + assert_eq!( + refusal(store.admit_revision(rev, entries)), + malformed("feature:50", shape) + ); + let (rev, mut entries) = entries_for(2, &["feature:50"]); + entries[0].value_kind = Some(ValueKind::Flag); + assert_eq!( + refusal(store.admit_revision(rev, entries)), + malformed("feature:50", shape) + ); + let (rev, mut entries) = entries_for(2, &["concept:1"]); + entries[0].value_kind = Some(ValueKind::Flag); + assert_eq!( + refusal(store.admit_revision(rev, entries)), + malformed("concept:1", shape) + ); + let (rev, entries) = entries_for(2, &["extension:1"]); + assert_eq!( + refusal(store.admit_revision(rev, entries)), + malformed("extension:1", shape) + ); + let (rev, mut entries) = entries_for(2, &["extension:1"]); + entries[0].value_kind = Some(ValueKind::Flag); + entries[0].applies_to = vec![Slot::Discourse]; + assert_eq!( + refusal(store.admit_revision(rev, entries)), + malformed("extension:1", shape) + ); + let (rev, mut entries) = entries_for(2, &["concept:1"]); + entries[0].retired_rev = Some(2); + assert_eq!( + refusal(store.admit_revision(rev, entries)), + malformed("concept:1", shape) + ); + // Retire targets must exist and be live; an id changes once per revision. + let rev = rev_with(vec![Change::Retire(iv("concept:9"))], 2, Some(1)); + assert_eq!( + refusal(store.admit_revision(rev, vec![])), + malformed("concept:9", "retire target is missing or already retired") + ); + let rev = rev_with( + vec![Change::Retire(iv("role:1")), Change::Retire(iv("role:1"))], + 2, + Some(1), + ); + assert_eq!( + refusal(store.admit_revision(rev, vec![])), + malformed("role:1", "id changes twice in one revision") + ); + // Nothing above was written. + let inv = store.inventory().unwrap(); + assert_eq!(inv.latest().unwrap().n, 1); + assert!(inv.entry(&iv("concept:1")).is_none()); + assert!(inv.entry(&iv("role:1")).unwrap().retired_rev.is_none()); +} + +#[test] +fn a_well_formed_extension_entry_is_admitted() { + let d = dir(); + let mut store = seeded(&d.path().join("i.cc")); + let (rev, mut entries) = entries_for(2, &["extension:1"]); + entries[0].value_kind = Some(ValueKind::Grade); + store.admit_revision(rev, entries).unwrap(); + assert_eq!( + store + .inventory() + .unwrap() + .entry(&iv("extension:1")) + .unwrap() + .family(), + Family::Extension + ); +} + +#[test] +fn retiring_keeps_the_entry_for_earlier_revisions() { + let d = dir(); + let path = d.path().join("i.cc"); + let mut store = seeded(&path); + let (rev, entries) = entries_for(2, &["concept:1"]); + store.admit_revision(rev, entries).unwrap(); + let retire = rev_with(vec![Change::Retire(iv("concept:1"))], 3, Some(2)); + store.admit_revision(retire, vec![]).unwrap(); + let inv = InventoryStore::open(&path).unwrap().inventory().unwrap(); + let id = iv("concept:1"); + assert_eq!(inv.entry(&id).unwrap().retired_rev, Some(3)); + assert!(inv.resolve(&id, 2).is_some()); + assert!(inv.resolve(&id, 3).is_none()); + let again = rev_with(vec![Change::Retire(id)], 4, Some(3)); + assert!(matches!( + refusal(store.admit_revision(again, vec![])), + StoreRefusal::Malformed { .. } + )); +} + +#[test] +fn revision_one_survives_revision_two() { + let inv = inventory(); + let revisions: Vec<(u32, usize)> = inv.revisions().map(|r| (r.n, r.changes.len())).collect(); + assert_eq!(revisions, vec![(1, 54), (2, 6), (3, 1), (4, 1)]); + // A packet of revision 1 is judged by the inventory of revision 1: the + // revision is known (no V10), and it names nothing that came later. + let mut p = good(); + p.inventory_rev = 1; + let refusals = weksa::validate(&p, &inv).unwrap_err(); + assert!( + refusals + .iter() + .all(|r| matches!(r, Refusal::V5Unresolved { .. })), + "{refusals:?}" + ); +} + +#[test] +fn the_seed_is_the_standards_enumerations_and_the_nuance_features() { + let d = dir(); + let store = seeded(&d.path().join("i.cc")); + let inv = store.inventory().unwrap(); + let count = |f: Family| inv.entries().filter(|e| e.family() == f).count(); + assert_eq!(count(Family::ReferentKind), 11); + assert_eq!(count(Family::Relation), 16); + assert_eq!(count(Family::Role), 8); + assert_eq!(count(Family::SpeechAct), 8); + assert_eq!(count(Family::Feature), 11); + assert_eq!( + count(Family::Concept) + count(Family::Frame) + count(Family::Extension), + 0 + ); + let gloss = |id: &str| inv.entry(&iv(id)).unwrap().gloss.clone(); + assert_eq!(gloss(PERSON), "person"); + assert_eq!(gloss(AGENT), "agent"); + assert_eq!(gloss(WARN), "warn"); + assert_eq!(gloss("relation:8"), "causes"); + assert_eq!(gloss(POLARITY), "polarity"); + assert_eq!(gloss(MODALITY), "modality"); + assert_eq!(gloss(HEDGE), "hedge"); + assert_eq!(gloss(IRONY), "irony"); + assert_eq!( + inv.entry(&iv(IRONY)).unwrap().value_kind, + Some(ValueKind::Flag) + ); + assert!( + inv.entries() + .all(|e| e.well_formed() && e.introduced_rev == 1 && e.retired_rev.is_none()) + ); + assert!(inv.has_revision(1)); +} + +fn admit_one(d: &TempDir, entry: InventoryEntry) -> Result<(), StoreError> { + let mut store = seeded(&d.path().join("i.cc")); + let rev = rev_with(vec![Change::Add(entry.id)], 2, Some(1)); + store.admit_revision(rev, vec![entry]) +} + +#[test] +fn gloss_is_bounded_text() { + let why = "gloss is 1 to 120 bytes with no control character"; + let with = |gloss: &str| { + let d = dir(); + let mut e = entry("concept:1", 2); + e.gloss = gloss.into(); + admit_one(&d, e) + }; + with(&"g".repeat(120)).unwrap(); + with("two words").unwrap(); + for bad in [ + String::new(), + "g".repeat(121), + "a\nb".into(), + "a\u{7}b".into(), + ] { + assert_eq!(refusal(with(&bad)), malformed("concept:1", why), "{bad:?}"); + } +} + +#[test] +fn packet_admission_goes_through_the_validator() { + let w = world(); + let mut store = PacketStore::open(&w.dir.path().join("p.cc")).unwrap(); + let mut bad = good(); + bad.schema_version = 5; + match store.admit(bad, &w.inventory, &w.corpus) { + Err(e @ StoreError::Invalid(_)) => { + assert!(e.to_string().contains("V1"), "{e}"); + let StoreError::Invalid(r) = e else { + unreachable!() + }; + assert_eq!(r, vec![Refusal::V1SchemaVersion { found: 5 }]); + } + other => panic!("{other:?}"), + } + assert!(store.all().unwrap().is_empty()); + store.admit(good(), &w.inventory, &w.corpus).unwrap(); + // The same packet is idempotent; a different one under the same key is refused. + store.admit(good(), &w.inventory, &w.corpus).unwrap(); + let mut other = good(); + other.provenance.confidence = grade(1); + assert_eq!( + refusal(store.admit(other, &w.inventory, &w.corpus)), + StoreRefusal::PacketImmutable { + key: PACKET_ID.into() + } + ); + assert_eq!(store.all().unwrap().len(), 1); + assert_eq!(store.get(PACKET_ID).unwrap(), Some(good())); +} + +#[test] +fn admission_reads_the_store_inventory() { + // The store holds only revision 1: a revision-2 packet has no inventory to + // stand on, whatever the caller believes about revision 2. + let d = dir(); + let mut corpus = empty_corpus(&d); + corpus + .add(item_in( + "item:nibu:1", + "Nibu says mind the hatch.", + "g", + cast(), + )) + .unwrap(); + let mut inventory = seeded(&d.path().join("i.cc")); + let mut packets = PacketStore::open(&d.path().join("p.cc")).unwrap(); + assert_eq!( + match packets.admit(good(), &inventory, &corpus) { + Err(StoreError::Invalid(r)) => r, + other => panic!("{other:?}"), + }, + vec![Refusal::V10UnknownRevision { rev: 2 }] + ); + assert!(packets.all().unwrap().is_empty()); + // The same packet admits once revision 2 is admitted. + let mut ext = entry(EXT, 2); + ext.value_kind = Some(ValueKind::Grade); + admit( + &mut inventory, + 2, + vec![ + entry(CONCEPT_A, 2), + entry(CONCEPT_B, 2), + entry(FRAME, 2), + ext, + ], + ); + packets.admit(good(), &inventory, &corpus).unwrap(); +} + +#[test] +fn corpus_item_must_resolve() { + let w = world(); + let mut packets = PacketStore::open(&w.dir.path().join("p.cc")).unwrap(); + for key in ["item:nibu:9", "banana"] { + let mut p = good(); + p.provenance.corpus_item = Some(key.into()); + assert_eq!( + refusal(packets.admit(p, &w.inventory, &w.corpus)), + StoreRefusal::UnknownCorpusItem { key: key.into() } + ); + } + assert!(packets.all().unwrap().is_empty()); + // A packet minted for no item needs none. + let mut p = good(); + p.provenance.corpus_item = None; + packets.admit(p, &w.inventory, &w.corpus).unwrap(); +} + +#[test] +fn store_roundtrip() { + let w = world(); + let d = dir(); + // A packet, an entry, a revision and an item written to temp stores read back equal. + let packets = d.path().join("p.cc"); + let p = good(); + PacketStore::open(&packets) + .unwrap() + .admit(p.clone(), &w.inventory, &w.corpus) + .unwrap(); + assert_eq!( + PacketStore::open(&packets).unwrap().get(PACKET_ID).unwrap(), + Some(p) + ); + + let inventories = d.path().join("i.cc"); + let declared = feature( + "feature:1", + ValueKind::Options(weksa::OptionSet::new(vec![lab("a"), lab("b")]).unwrap()), + &[Slot::Relation, Slot::Referent], + ); + let mut concept = entry("concept:1", 1); + concept.gloss = "a gloss".into(); + let rev = rev_with( + vec![Change::Add(iv("concept:1")), Change::Add(iv("feature:1"))], + 1, + None, + ); + InventoryStore::open(&inventories) + .unwrap() + .admit_revision(rev.clone(), vec![concept.clone(), declared.clone()]) + .unwrap(); + let back = InventoryStore::open(&inventories) + .unwrap() + .inventory() + .unwrap(); + assert_eq!(back.entry(&iv("concept:1")), Some(&concept)); + assert_eq!(back.entry(&iv("feature:1")), Some(&declared)); + assert_eq!(back.latest(), Some(&rev)); + + let corpus = d.path().join("c.cc"); + let it = item("item:nibu:1", "round trip"); + CorpusStore::open(&corpus).unwrap().add(it.clone()).unwrap(); + assert_eq!( + CorpusStore::open(&corpus) + .unwrap() + .get("item:nibu:1") + .unwrap(), + Some(it) + ); +} + +#[test] +fn the_committed_stores_load_and_hold_clean_data() { + let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); + let inv = InventoryStore::open(&root.join("data/interlingua/inventory.cc")) + .unwrap() + .inventory() + .unwrap(); + assert!(inv.has_revision(1)); + assert!(inv.entry(&iv(PERSON)).is_some()); + let items = CorpusStore::open(&root.join("data/corpus/corpus.cc")) + .unwrap() + .all() + .unwrap(); + assert!(items.len() >= 3, "{} items", items.len()); + for i in &items { + assert!( + !i.text.trim().is_empty() && i.language == "en-US" && !i.withdrawn, + "{i:?}" + ); + if matches!(i.source, Source::ModelWritten { .. }) { + assert_ne!( + SplitScheme::V1.split(i), + Some(weksa::Split::Test), + "{}", + i.id + ); + } + } + assert!( + items + .iter() + .any(|i| matches!(i.source, Source::ModelWritten { .. })), + "model-written lines are labelled" + ); +} diff --git a/crates/weksa/tests/validate.rs b/crates/weksa/tests/validate.rs new file mode 100644 index 0000000..1d4eaa6 --- /dev/null +++ b/crates/weksa/tests/validate.rs @@ -0,0 +1,594 @@ +mod common; +use common::*; +use weksa::packet::*; +use weksa::{Family, Refusal, Slot, validate}; + +fn refusals(p: &Packet) -> Vec { + validate(p, &inventory()).err().unwrap_or_default() +} + +fn one(p: &Packet) -> Refusal { + let mut r = refusals(p); + assert_eq!(r.len(), 1, "{r:?}"); + r.remove(0) +} + +fn unresolved(id: &str) -> Refusal { + Refusal::V5Unresolved { id: iv(id) } +} + +fn empty_payload() -> Packet { + let mut p = good(); + p.referents.clear(); + p.predications.clear(); + p.relations.clear(); + p.discourse.addressee = None; + p.context.participants.clear(); + p.context.audience.clear(); + p.constraints.preserve.clear(); + p +} + +fn feat(id: &str, value: FeatValue) -> FeatureValue { + FeatureValue { + feature: iv(id), + value, + } +} + +#[test] +fn a_good_packet_validates() { + assert!(validate(&good(), &inventory()).is_ok()); +} + +#[test] +fn v1_schema_version() { + let mut p = good(); + p.schema_version = SCHEMA_VERSION + 1; + assert_eq!( + refusals(&p), + vec![Refusal::V1SchemaVersion { + found: SCHEMA_VERSION + 1 + }] + ); +} + +#[test] +fn v2_payload_required() { + assert_eq!(refusals(&empty_payload()), vec![Refusal::V2PayloadRequired]); + // Any one family is payload enough. + let mut only_relations = empty_payload(); + only_relations.relations = good().relations; + only_relations.relations[0].from = nd("n8"); + only_relations.relations[0].to = nd("n9"); + assert!(!refusals(&only_relations).contains(&Refusal::V2PayloadRequired)); + let mut only_predications = empty_payload(); + only_predications.predications = good().predications; + only_predications.predications[0].roles.clear(); + assert_eq!(refusals(&only_predications), vec![]); + let mut only_referents = empty_payload(); + only_referents.referents = good().referents; + assert_eq!(refusals(&only_referents), vec![]); +} + +#[test] +fn v3_unique_node_ids() { + let mut p = good(); + p.predications[0].id = nd("n1"); + p.relations[0].from = nd("n1"); + assert_eq!( + refusals(&p), + vec![Refusal::V3DuplicateNode { id: nd("n1") }] + ); + let mut twice = good(); + twice.referents.push(twice.referents[0].clone()); + assert_eq!( + refusals(&twice), + vec![Refusal::V3DuplicateNode { id: nd("n1") }] + ); +} + +#[test] +fn v4_refs_resolve() { + let dangling = |edit: fn(&mut Packet), at: &'static str| { + let mut p = good(); + edit(&mut p); + assert_eq!( + refusals(&p), + vec![Refusal::V4DanglingRef { id: nd("n99"), at }], + "{at}" + ); + }; + dangling( + |p| p.predications[0].roles[0].filler = nd("n99"), + "role filler", + ); + dangling(|p| p.relations[0].from = nd("n99"), "relation from"); + dangling(|p| p.relations[0].to = nd("n99"), "relation to"); + dangling(|p| p.discourse.addressee = Some(nd("n99")), "addressee"); + dangling(|p| p.context.participants.push(nd("n99")), "participant"); + dangling(|p| p.context.audience.push(nd("n99")), "audience"); + dangling(|p| p.constraints.preserve.push(nd("n99")), "preserve"); +} + +#[test] +fn v5_inventory_resolves_at_rev() { + let with_concept = |rev: u32, id: &str| { + let mut p = good(); + p.inventory_rev = rev; + p.referents[0].concept = iv(id); + p + }; + // Introduced after the packet's rev. + assert_eq!( + one(&with_concept(2, CONCEPT_LATE)), + unresolved(CONCEPT_LATE) + ); + // Introduced at the packet's rev. + assert_eq!(refusals(&with_concept(4, CONCEPT_LATE)), vec![]); + // Retired later than the packet's rev still resolves. + assert_eq!(refusals(&with_concept(2, CONCEPT_RETIRED)), vec![]); + // Retired at or before the packet's rev does not. + assert_eq!( + one(&with_concept(3, CONCEPT_RETIRED)), + unresolved(CONCEPT_RETIRED) + ); + assert_eq!( + one(&with_concept(4, CONCEPT_RETIRED)), + unresolved(CONCEPT_RETIRED) + ); + // An id no revision holds. + assert_eq!( + one(&with_concept(2, "concept:99")), + unresolved("concept:99") + ); + // An extension is resolved like any inventory id. + let mut ext = good(); + ext.extensions[0].key = iv("extension:99"); + assert_eq!(one(&ext), unresolved("extension:99")); +} + +#[test] +fn v6_family_matches_slot() { + let wrong = |edit: fn(&mut Packet), id: &str, expected: Family, at: &'static str| { + let mut p = good(); + edit(&mut p); + assert_eq!( + refusals(&p), + vec![Refusal::V6WrongFamily { + id: iv(id), + expected, + at + }], + "{at}" + ); + }; + wrong( + |p| p.discourse.features[0].feature = iv(AGENT), + AGENT, + Family::Feature, + "feature", + ); + wrong( + |p| p.discourse.speech_act = Some(iv("relation:1")), + "relation:1", + Family::SpeechAct, + "speech act", + ); + wrong( + |p| p.referents[0].kind = iv(CONCEPT_A), + CONCEPT_A, + Family::ReferentKind, + "referent kind", + ); + wrong( + |p| p.referents[0].concept = iv(PERSON), + PERSON, + Family::Concept, + "referent concept", + ); + wrong( + |p| p.predications[0].concept = iv(FRAME), + FRAME, + Family::Concept, + "predication concept", + ); + wrong( + |p| p.predications[0].frame = iv(CONCEPT_A), + CONCEPT_A, + Family::Frame, + "frame", + ); + wrong( + |p| p.predications[0].roles[0].role = iv(HEDGE), + HEDGE, + Family::Role, + "role", + ); + wrong( + |p| p.relations[0].relation = iv(AGENT), + AGENT, + Family::Relation, + "relation", + ); + wrong( + |p| p.context.setting = Some(iv(FRAME)), + FRAME, + Family::Concept, + "context concept", + ); + wrong( + |p| p.context.medium = Some(iv(FRAME)), + FRAME, + Family::Concept, + "context concept", + ); + wrong( + |p| p.extensions[0].key = iv(CONCEPT_A), + CONCEPT_A, + Family::Extension, + "extension", + ); +} + +#[test] +fn v7_value_kind() { + let hedge = |v: FeatValue| { + let mut p = good(); + p.discourse.features[0].value = v; + refusals(&p) + }; + let bad = vec![Refusal::V7ValueKind { what: HEDGE.into() }]; + assert_eq!(hedge(FeatValue::Grade(grade(1000))), vec![]); + assert_eq!(hedge(FeatValue::Flag(true)), bad); + assert_eq!(hedge(FeatValue::Option(lab("x"))), bad); + let modality = |v: FeatValue| { + let mut p = good(); + p.predications[0].features[0].value = v; + refusals(&p) + }; + let bad_modality = vec![Refusal::V7ValueKind { + what: MODALITY.into(), + }]; + assert_eq!(modality(FeatValue::Option(lab("necessary"))), vec![]); + assert_eq!(modality(FeatValue::Option(lab("banana"))), bad_modality); + assert_eq!(modality(FeatValue::Grade(grade(1))), bad_modality); + let mut flag = good(); + flag.discourse.features = vec![feat(IRONY, FeatValue::Flag(false))]; + assert_eq!(refusals(&flag), vec![]); + flag.discourse.features[0].value = FeatValue::Grade(grade(5)); + assert_eq!( + refusals(&flag), + vec![Refusal::V7ValueKind { what: IRONY.into() }] + ); +} + +#[test] +fn extension_key_is_an_extension_entry() { + // A concept id is not an extension; an extension entry fits its value kind. + let mut by_concept = good(); + by_concept.extensions[0].key = iv(CONCEPT_A); + assert_eq!( + one(&by_concept), + Refusal::V6WrongFamily { + id: iv(CONCEPT_A), + expected: Family::Extension, + at: "extension" + } + ); + let mut flagged = good(); + flagged.extensions[0].value = FeatValue::Flag(true); + assert_eq!(one(&flagged), Refusal::V7ValueKind { what: EXT.into() }); + let mut optioned = good(); + optioned.extensions[0].value = FeatValue::Option(lab("x")); + assert_eq!(one(&optioned), Refusal::V7ValueKind { what: EXT.into() }); + let mut unknown = good(); + unknown.extensions[0].key = iv("extension:2"); + assert_eq!(one(&unknown), unresolved("extension:2")); + // No extension means none to check. + let mut none = good(); + none.extensions.clear(); + assert_eq!(refusals(&none), vec![]); +} + +#[test] +fn v8_feature_applies_to() { + let mut p = good(); + // polarity applies to predications only. + p.discourse + .features + .push(feat(POLARITY, FeatValue::Option(lab("negative")))); + assert_eq!( + refusals(&p), + vec![Refusal::V8NotApplicable { + feature: iv(POLARITY), + slot: Slot::Discourse + }] + ); + let mut referent = good(); + referent.referents[0] + .features + .push(feat(HEDGE, FeatValue::Grade(grade(1)))); + assert_eq!( + refusals(&referent), + vec![Refusal::V8NotApplicable { + feature: iv(HEDGE), + slot: Slot::Referent + }] + ); + let mut relation = good(); + relation.relations[0] + .features + .push(feat(HEDGE, FeatValue::Grade(grade(1)))); + assert_eq!( + refusals(&relation), + vec![Refusal::V8NotApplicable { + feature: iv(HEDGE), + slot: Slot::Relation + }] + ); + let mut pred = good(); + pred.predications[0] + .features + .push(feat(HEDGE, FeatValue::Grade(grade(1)))); + assert_eq!(refusals(&pred), vec![]); +} + +#[test] +fn v10_rev_exists() { + let mut p = good(); + p.inventory_rev = 9; + assert_eq!(refusals(&p), vec![Refusal::V10UnknownRevision { rev: 9 }]); + p.inventory_rev = 0; + assert_eq!(refusals(&p), vec![Refusal::V10UnknownRevision { rev: 0 }]); +} + +#[test] +fn every_broken_rule_is_reported() { + let mut p = good(); + p.schema_version = 7; + p.predications[0].roles[0].filler = nd("n99"); + p.extensions[0].key = iv(CONCEPT_A); + let rules: Vec<&str> = refusals(&p).iter().map(Refusal::rule).collect(); + assert_eq!(rules, vec!["V1", "V4", "V6"]); +} + +#[test] +fn lowering_input_carries_no_trace_id_or_provenance() { + let mut p = good(); + p.trace.notes = vec!["SECRET-NOTE".into()]; + p.trace.rejected_readings = vec!["SECRET-READING".into()]; + let input = validate(&p, &inventory()).unwrap(); + let seen = format!("{input:?}"); + assert!(seen.contains("Nibu"), "the input carries the name: {seen}"); + for hidden in [PACKET_ID, ENCODER_ID, "item:nibu:1", "SECRET"] { + assert!(!seen.contains(hidden), "{hidden} reached the input: {seen}"); + } + // The view is the packet's meaning, readable through getters. + assert_eq!(input.inventory_rev(), 2); + assert_eq!(input.kind(), PacketKind::DialogueLine); + assert_eq!(input.referents(), p.referents.as_slice()); + assert_eq!(input.predications(), p.predications.as_slice()); + assert_eq!(input.relations(), p.relations.as_slice()); + assert_eq!(input.extensions(), p.extensions.as_slice()); + assert_eq!(input.context(), &p.context); + assert_eq!(input.discourse(), &p.discourse); + assert_eq!(input.constraints(), &p.constraints); +} + +#[test] +fn invalid_packet_has_no_lowering_input() { + let mut p = good(); + p.schema_version = 99; + p.referents.clear(); + let refused = validate(&p, &inventory()).unwrap_err(); + let rules: Vec<&str> = refused.iter().map(Refusal::rule).collect(); + assert_eq!(rules[0], "V1"); + assert!(rules.contains(&"V4"), "{rules:?}"); +} + +fn grade_feature(grade_value: u16) -> FeatureValue { + feat(FEAT_ALL, FeatValue::Grade(grade(grade_value))) +} + +fn referent(id: &str) -> Referent { + Referent { + id: nd(id), + kind: iv(PERSON), + concept: iv(CONCEPT_A), + name: None, + features: vec![], + } +} + +fn v11(slot: Option, feature: &str) -> Refusal { + Refusal::V11DuplicateFeature { + feature: iv(feature), + slot, + } +} + +#[test] +fn v11_duplicate_feature_each_slot() { + let mut discourse = good(); + discourse.discourse.features = vec![grade_feature(1), grade_feature(2)]; + assert_eq!( + refusals(&discourse), + vec![v11(Some(Slot::Discourse), FEAT_ALL)] + ); + let mut referent = good(); + referent.referents[0].features = vec![grade_feature(1), grade_feature(2)]; + assert_eq!( + refusals(&referent), + vec![v11(Some(Slot::Referent), FEAT_ALL)] + ); + let mut predication = good(); + predication.predications[0] + .features + .extend([grade_feature(1), grade_feature(2)]); + assert_eq!( + refusals(&predication), + vec![v11(Some(Slot::Predication), FEAT_ALL)] + ); + let mut relation = good(); + relation.relations[0].features = vec![grade_feature(1), grade_feature(2)]; + assert_eq!( + refusals(&relation), + vec![v11(Some(Slot::Relation), FEAT_ALL)] + ); + let mut extension = good(); + extension.extensions.push(Extension { + key: iv(EXT), + value: FeatValue::Grade(grade(4)), + }); + assert_eq!(refusals(&extension), vec![v11(None, EXT)]); +} + +#[test] +fn v11_counts_per_list_not_per_packet() { + // The same feature once in each list, and different features in one list, are fine. + let mut p = good(); + p.discourse.features = vec![grade_feature(1), feat(HEDGE, FeatValue::Grade(grade(2)))]; + p.referents[0].features = vec![grade_feature(1)]; + p.predications[0].features.push(grade_feature(1)); + p.relations[0].features = vec![grade_feature(1)]; + assert_eq!(refusals(&p), vec![]); +} + +#[test] +fn v11_grade_bytes_packet_refused() { + // Soul s2: 113 hedge grades in one list spell a line byte by byte. + let mut p = good(); + p.discourse.features = (0..113) + .map(|i| feat(HEDGE, FeatValue::Grade(grade(i)))) + .collect(); + let got = refusals(&p); + assert_eq!(got.len(), 112, "{got:?}"); + assert!(got.iter().all(|r| *r == v11(Some(Slot::Discourse), HEDGE))); +} + +#[test] +fn v12_node_bound() { + // 62 referents, the predication n2 and the relation make 64 nodes. + let with_referents = |last: u32| { + let mut p = good(); + let ids = std::iter::once(1).chain(3..=last); + p.referents = ids.map(|k| referent(&format!("n{k}"))).collect(); + p + }; + let at_bound = with_referents(63); + assert_eq!( + at_bound.referents.len() + at_bound.predications.len() + at_bound.relations.len(), + Packet::MAX_NODES + ); + assert_eq!(refusals(&at_bound), vec![]); + assert_eq!( + refusals(&with_referents(64)), + vec![Refusal::V12TooManyNodes { + count: 65, + max: Packet::MAX_NODES + }] + ); +} + +// Soul s3: a repeated node id in a node list, or a repeated role in one +// predication, spelled a line one bit per entry. Each list now holds a node or +// a role at most once, so the channel is bounded. + +const LINE: &str = "Take your hand off the hatch, Captain."; + +fn bits(s: &str) -> Vec { + s.bytes() + .flat_map(|b| (0..8).rev().map(move |i| (b >> i) & 1 == 1)) + .collect() +} + +fn two_referents() -> Packet { + let mut p = good(); + let mut r = p.referents[0].clone(); + r.id = nd("n3"); + r.name = None; + p.referents.push(r); + p +} + +fn bit_ids(s: &str) -> Vec { + bits(s) + .into_iter() + .map(|b| nd(if b { "n3" } else { "n1" })) + .collect() +} + +fn dup_node(id: &str, at: &'static str) -> Refusal { + Refusal::V11DuplicateNodeRef { id: nd(id), at } +} + +#[test] +fn v11_node_lists_refuse_repeats_and_allow_each_node_once() { + let mut ok = two_referents(); + ok.context.participants = vec![nd("n1"), nd("n3")]; + ok.context.audience = vec![nd("n3"), nd("n1")]; + ok.constraints.preserve = vec![nd("n3"), nd("n1")]; + assert_eq!(refusals(&ok), vec![]); + + let mut p = two_referents(); + p.context.participants = vec![nd("n1"), nd("n1")]; + assert_eq!(refusals(&p), vec![dup_node("n1", "participant")]); + let mut p = two_referents(); + p.context.audience = vec![nd("n3"), nd("n1"), nd("n3")]; + assert_eq!(refusals(&p), vec![dup_node("n3", "audience")]); + let mut p = two_referents(); + p.constraints.preserve = vec![nd("n1"), nd("n1")]; + assert_eq!(refusals(&p), vec![dup_node("n1", "preserve")]); +} + +#[test] +fn v11_bit_channel_in_each_node_list_refused() { + let n = bits(LINE).len(); + let mut p = two_referents(); + p.context.participants = bit_ids(LINE); + let got = refusals(&p); + assert_eq!(got.len(), n - 2, "{got:?}"); + assert!(got.iter().all(|r| r.rule() == "V11")); + let mut p = two_referents(); + p.context.audience = bit_ids(LINE); + assert_eq!(refusals(&p).len(), n - 2); + let mut p = two_referents(); + p.constraints.preserve = bit_ids(LINE); + assert_eq!(refusals(&p).len(), n - 2); +} + +#[test] +fn v11_unbounded_participants_refused() { + let mut p = two_referents(); + p.context.participants = (0..100_000) + .map(|i| nd(if i % 3 == 0 { "n3" } else { "n1" })) + .collect(); + assert_eq!(refusals(&p).len(), 100_000 - 2); +} + +#[test] +fn v11_role_repeat_in_one_predication_refused() { + let dup = |role: &str| Refusal::V11DuplicateRole { + role: iv(role), + predication: nd("n2"), + }; + let mut p = good(); + p.predications[0].roles = bits(LINE) + .into_iter() + .map(|b| RoleFill { + role: iv(if b { AGENT } else { PATIENT }), + filler: nd("n1"), + }) + .collect(); + let got = refusals(&p); + assert_eq!(got.len(), bits(LINE).len() - 2, "{got:?}"); + assert!(got.contains(&dup(AGENT)) && got.contains(&dup(PATIENT))); + + // The same role in two different predications is not a repeat. + let mut ok = good(); + let mut second = ok.predications[0].clone(); + second.id = nd("n4"); + ok.predications.push(second); + assert_eq!(refusals(&ok), vec![]); +} diff --git a/data/corpus/corpus.cc b/data/corpus/corpus.cc new file mode 100644 index 0000000..803fc17 --- /dev/null +++ b/data/corpus/corpus.cc @@ -0,0 +1 @@ +“²cultcache.store.v1‘—±weksa.corpus_item±weksa.corpus_item“weksa.corpus_item.v1±weksa.corpus_itemŁV{"schemaName":"weksa.corpus_item","schemaVersion":"weksa.corpus_item.v1","members":[]}‘±weksa.corpus_item“”­item:brokkr:1±weksa.corpus_item“2026-10-29T09:44:22Zēš­item:brokkr:1Ł,The command has no receipt. It is not truth.„en-US¦brokkr±brokkr-edda-voice¬ModelWritten’ølocal-source-aware-draftĄŖ2026-10-29ĄĀ”«item:nibu:1±weksa.corpus_item“2026-10-29T09:44:22ZÄ÷š«item:nibu:1ŁnTake your hand off that control unless you want your nervous system to become my least useful circuit diagram.„en-US¤nibu®nibu-proof-001¬CastAuthored’¤nibuŁ:` with `n` a canonical decimal of at +least 1 (`role:1`; `role:01` is not an id), a node id is `n` with `k` in +1..=9999, a packet id is `packet:<16 hex>` and an encoder id is +`encoder:<16 hex>`. The free text a packet may hold is a referent's `name`, +a `ProperName` of 1 to 4 space-joined tokens of letters, apostrophes and +hyphens within 64 bytes, and the `trace`. A name is bounded text, not a judged +one: no door compares packet text with the corpus. Leakage of scored text is +owned outside the validator, by the loop's context builder (what each model +call sees) and the scorer (names are masked before scoring); the validator +owns form and capacity. The trace is stripped before lowering. Concept ids are opaque and carry a gloss in the inventory; concepts +are open-class and grow, while the other families are closed-class. + +```text +Packet + id, schema_version, inventory_rev, kind + provenance { source, corpus_item, encoder, confidence } + context { setting, medium, participants, audience } + discourse { speech_act, addressee, features } + referents [ { id, kind, concept, name, features } ] + predications [ { id, concept, frame, roles: [ { role, filler } ], features } ] + relations [ { relation, from, to, features } ] + constraints { preserve, deterministic, max_latency_ms } + extensions [ { key: InvId (family extension), value: Grade | Option | Flag } ] + trace { notes, rejected_readings } ``` -Only `interlingua_version`, `packet_id`, `kind`, and at least one semantic -payload family are required. The rest should be present when known. +`schema_version` and `inventory_rev` name the exact schema and inventory +revision a packet was written against. A grade is an integer permille, +`0..=1000`, and the published schema says so. There are no floats. -## Top-Level Fields +### Validation + +A packet exists only if `validate` passes; every path that admits or lowers a +packet calls it. A refusal names the rule. + +- V1: `schema_version` is the version these types define. +- V2: at least one of `referents`, `predications`, `relations` is present. +- V3: a node id appears once across referents and predications. +- V4: every role filler, relation end, addressee, participant, audience member + and preserved node names a node. +- V5: every inventory id resolves at the packet's `inventory_rev`: introduced + at or before it, and not retired at or before it. +- V6: an inventory id is of the family its slot takes. +- V7: a feature or extension value fits its entry's value kind (a grade is a + grade, an option is one the entry declares, a flag is a flag). +- V8: a feature sits only in a slot its entry declares. +- V10: `inventory_rev` has a revision record. +- V11: a feature id appears at most once in one slot's feature list, an + extension id at most once per packet, a node id at most once in each of + `participants`, `audience` and `preserve`, and a role id at most once in one + predication's roles. +- V12: a packet has at most 64 referents, predications and relations together + (a calibration constant, capacity and not meaning). + +V9 is retired and not renumbered: the form of a key or a grade is its type, and +an extension key's family is V6. `validate` returns the lowering view of a +packet that passes; it carries no packet id, no provenance and no trace, and is +built nowhere else. A packet is admitted against the inventory the packet store +reads at the packet's revision, never one the caller supplies. ### `kind` -The packet's broad use case. - -Initial values: - -- `utterance` -- `name_request` -- `commentary_intent` -- `narration` -- `dialogue_line` -- `corpus_example` -- `debug_probe` +`Utterance`, `NameRequest`, `CommentaryIntent`, `Narration`, `DialogueLine`, +`CorpusExample`, `DebugProbe`. ### `provenance` -Where the packet came from. - -```yaml -provenance: - source_type: authored | model_decomposition | game_telemetry | imported_corpus - source_ref: - author: - created_at: - confidence: -``` +`source` is `Authored`, `ModelDecomposition`, `GameTelemetry` or +`ImportedCorpus`. `corpus_item` is the key of the corpus item the packet was +minted for, `encoder` names what wrote it, and `confidence` is a grade. ### `context` -World and situation information that is not itself the main semantic claim. - -```yaml -context: - setting: - scene: - time: - location: - participants: - audience: - medium: -``` +`setting` and `medium` are concept ids. `participants` and `audience` are node +ids. ### `discourse` -Why this packet is being said, rendered, or named. - -```yaml -discourse: - speech_act: - intent: - tone: - register_hint: - addressee: - audience_function: -``` - -This is not target-language register. It is source-side communicative pressure. +Why the packet is said: `speech_act` is a speech-act id, `addressee` a node id, +and `features` carry source-side nuance (hedge, certainty, warmth, formality, +intensity, irony and whatever the inventory grows). This is source-side +communicative pressure, not target-language register. ### `referents` -Entities, groups, locations, abstract objects, institutions, ecological actors, -routes, memories, artifacts, and other nodes that can participate in meaning. - -```yaml -referents: - - id: - kind: - label: - ontology_hint: - animacy: - agency: - number: - definiteness: - salience: - attributes: - extensions: -``` - -Initial broad kinds: - -- `person` -- `group` -- `organism` -- `place` -- `artifact` -- `institution` -- `ecology` -- `route` -- `memory` -- `event` -- `abstract` +Entities, groups, places, abstractions and the rest. `kind` is a referent-kind +id (the inventory seeds `person`, `group`, `organism`, `place`, `artifact`, +`institution`, `ecology`, `route`, `memory`, `event`, `abstract`), `concept` is +a concept id, and `name` is a proper name when the referent has one. ### `predications` -Events, states, processes, properties, and frame-like scenes. - -```yaml -predications: - - id: - predicate: - frame: - aspect: - tense: - polarity: - modality: - evidentiality: - roles: - agent: - patient: - experiencer: - instrument: - source: - goal: - location: - beneficiary: - attributes: - extensions: -``` - -`predicate` is a broad source-side claim. `frame` is optional but recommended -when a scene model is known. Roles may use common names or namespaced frame -roles. +Events, states, processes and scenes. `concept` and `frame` are ids; `roles` +pair a role id with a filler node (the inventory seeds `agent`, `patient`, +`experiencer`, `instrument`, `source`, `goal`, `location`, `beneficiary`). Tense, +aspect, polarity, modality and evidentiality are features with declared option +sets, not fields. ### `relations` -Edges that are not naturally owned by one predication. - -```yaml -relations: - - type: - from: - to: - attributes: - extensions: -``` - -Initial relation families: - -- `coreference` -- `part_of` -- `member_of` -- `located_at` -- `near` -- `before` -- `after` -- `causes` -- `enables` -- `prevents` -- `owns` -- `owes` -- `knows` -- `names` -- `represents` -- `contrasts_with` +Edges not owned by one predication. `relation` is a relation id (the inventory +seeds `coreference`, `part_of`, `member_of`, `located_at`, `near`, `before`, +`after`, `causes`, `enables`, `prevents`, `owns`, `owes`, `knows`, `names`, +`represents`, `contrasts_with`), and `from` and `to` are node ids. ### `constraints` -Hard requirements the downstream renderer must respect. - -```yaml -constraints: - preserve_referents: - preserve_relations: - required_outputs: - forbidden_outputs: - max_latency_ms: - deterministic: -``` - -These are engineering and content constraints, not ontology rules. +`preserve` lists nodes the lowering must keep, `deterministic` and +`max_latency_ms` bound the renderer. Requirements about which outputs a +renderer must or must not produce belong to the target profile, not the packet. ### `extensions` -Namespaced setting or client data. - -```yaml -extensions: - zyphos.memory_relation: - zyphos.boundary_permeability: - rtd.aesthetic_reading: - game.priority: -``` - -Extension rules: - -- Namespaces must be explicit. -- Extensions must not replace common fields when common fields are sufficient. -- A downstream language project may choose to ignore an extension. -- If an extension becomes broadly required across clients, promote it into the - standard in a later version. +Setting or client data. An extension key is an inventory entry of family +`extension` (`extension:`) whose entry declares the value kind, so a key is +an id and not a string. Extensions must not replace common fields when common +fields suffice, a language project may ignore them, and one that becomes +broadly required is promoted into the inventory or the schema. ### `trace` -Optional authoring/debug data. - -```yaml -trace: - notes: - decomposition_steps: - uncertainty: - rejected_readings: -``` - -Runtime may strip `trace` from compiled packets when latency or size matters. +Authoring and debug notes. The trace is stripped before lowering: the lowering +input type has no trace field. ## Minimal Examples +These examples illustrate meaning in the 0.1 shape, with free-text labels and +predicates. Those fields no longer exist; the examples are not valid packets. + ### Name Request ```yaml diff --git a/schemas/cultnet/weksa.corpus_item.schema.json b/schemas/cultnet/weksa.corpus_item.schema.json new file mode 100644 index 0000000..1359deb --- /dev/null +++ b/schemas/cultnet/weksa.corpus_item.schema.json @@ -0,0 +1,184 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "CorpusItem", + "description": "Key: `item::`.", + "type": "object", + "properties": { + "admitted_on": { + "type": "string" + }, + "group": { + "description": "The unit the split hashes: a scene, a fixture, a work.", + "type": "string" + }, + "id": { + "type": "string" + }, + "language": { + "type": "string" + }, + "prior_turns": { + "type": "array", + "items": { + "$ref": "#/$defs/Turn" + } + }, + "source": { + "$ref": "#/$defs/Source" + }, + "speaker": { + "type": "string" + }, + "supersedes": { + "description": "Key of the item this one corrects.", + "type": [ + "string", + "null" + ] + }, + "text": { + "type": "string" + }, + "withdrawn": { + "type": "boolean" + } + }, + "required": [ + "id", + "text", + "language", + "speaker", + "group", + "prior_turns", + "source", + "admitted_on", + "withdrawn" + ], + "$defs": { + "Source": { + "oneOf": [ + { + "type": "string", + "enum": [ + "OperatorWritten" + ] + }, + { + "description": "A line from a cast member's hand-authored fixture.", + "type": "object", + "properties": { + "CastAuthored": { + "type": "object", + "properties": { + "fixture": { + "type": "string" + }, + "persona": { + "type": "string" + } + }, + "required": [ + "persona", + "fixture" + ] + } + }, + "additionalProperties": false, + "required": [ + "CastAuthored" + ] + }, + { + "type": "object", + "properties": { + "OperatorApproved": { + "type": "object", + "properties": { + "note": { + "type": "string" + } + }, + "required": [ + "note" + ] + } + }, + "additionalProperties": false, + "required": [ + "OperatorApproved" + ] + }, + { + "type": "object", + "properties": { + "PublicDomain": { + "type": "object", + "properties": { + "author": { + "type": "string" + }, + "edition_url": { + "type": "string" + }, + "work": { + "type": "string" + } + }, + "required": [ + "work", + "author", + "edition_url" + ] + } + }, + "additionalProperties": false, + "required": [ + "PublicDomain" + ] + }, + { + "description": "Labelled model output; never in the test split, never judged by its own family.", + "type": "object", + "properties": { + "ModelWritten": { + "type": "object", + "properties": { + "model_family": { + "type": "string" + }, + "prompt_digest": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "model_family" + ] + } + }, + "additionalProperties": false, + "required": [ + "ModelWritten" + ] + } + ] + }, + "Turn": { + "type": "object", + "properties": { + "speaker": { + "type": "string" + }, + "text": { + "type": "string" + } + }, + "required": [ + "speaker", + "text" + ] + } + } +} diff --git a/schemas/cultnet/weksa.interlingua_packet.schema.json b/schemas/cultnet/weksa.interlingua_packet.schema.json new file mode 100644 index 0000000..634d0d5 --- /dev/null +++ b/schemas/cultnet/weksa.interlingua_packet.schema.json @@ -0,0 +1,472 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Packet", + "type": "object", + "properties": { + "constraints": { + "$ref": "#/$defs/Constraints" + }, + "context": { + "$ref": "#/$defs/Context" + }, + "discourse": { + "$ref": "#/$defs/Discourse" + }, + "extensions": { + "type": "array", + "items": { + "$ref": "#/$defs/Extension" + } + }, + "id": { + "$ref": "#/$defs/PacketId" + }, + "inventory_rev": { + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "kind": { + "$ref": "#/$defs/PacketKind" + }, + "predications": { + "type": "array", + "items": { + "$ref": "#/$defs/Predication" + } + }, + "provenance": { + "$ref": "#/$defs/Provenance" + }, + "referents": { + "type": "array", + "items": { + "$ref": "#/$defs/Referent" + } + }, + "relations": { + "type": "array", + "items": { + "$ref": "#/$defs/Relation" + } + }, + "schema_version": { + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "trace": { + "$ref": "#/$defs/Trace" + } + }, + "required": [ + "id", + "schema_version", + "inventory_rev", + "kind", + "provenance", + "context", + "discourse", + "referents", + "predications", + "relations", + "constraints", + "extensions", + "trace" + ], + "$defs": { + "Constraints": { + "type": "object", + "properties": { + "deterministic": { + "type": "boolean" + }, + "max_latency_ms": { + "type": [ + "integer", + "null" + ], + "format": "uint32", + "minimum": 0 + }, + "preserve": { + "type": "array", + "items": { + "$ref": "#/$defs/NodeId" + } + } + }, + "required": [ + "preserve", + "deterministic" + ] + }, + "Context": { + "type": "object", + "properties": { + "audience": { + "type": "array", + "items": { + "$ref": "#/$defs/NodeId" + } + }, + "medium": { + "anyOf": [ + { + "$ref": "#/$defs/InvId" + }, + { + "type": "null" + } + ] + }, + "participants": { + "type": "array", + "items": { + "$ref": "#/$defs/NodeId" + } + }, + "setting": { + "description": "Concept ids.", + "anyOf": [ + { + "$ref": "#/$defs/InvId" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "participants", + "audience" + ] + }, + "Discourse": { + "type": "object", + "properties": { + "addressee": { + "anyOf": [ + { + "$ref": "#/$defs/NodeId" + }, + { + "type": "null" + } + ] + }, + "features": { + "type": "array", + "items": { + "$ref": "#/$defs/FeatureValue" + } + }, + "speech_act": { + "description": "Speech-act id.", + "anyOf": [ + { + "$ref": "#/$defs/InvId" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "features" + ] + }, + "EncoderId": { + "type": "string", + "pattern": "^encoder:[0-9a-f]{16}$" + }, + "Extension": { + "type": "object", + "properties": { + "key": { + "description": "Extension-entry id; the inventory declares its value kind.", + "$ref": "#/$defs/InvId" + }, + "value": { + "$ref": "#/$defs/FeatValue" + } + }, + "required": [ + "key", + "value" + ] + }, + "FeatValue": { + "oneOf": [ + { + "type": "object", + "properties": { + "Grade": { + "$ref": "#/$defs/Grade" + } + }, + "additionalProperties": false, + "required": [ + "Grade" + ] + }, + { + "description": "A label the feature's inventory entry declares.", + "type": "object", + "properties": { + "Option": { + "$ref": "#/$defs/OptionLabel" + } + }, + "additionalProperties": false, + "required": [ + "Option" + ] + }, + { + "type": "object", + "properties": { + "Flag": { + "type": "boolean" + } + }, + "additionalProperties": false, + "required": [ + "Flag" + ] + } + ] + }, + "FeatureValue": { + "type": "object", + "properties": { + "feature": { + "description": "Feature id.", + "$ref": "#/$defs/InvId" + }, + "value": { + "$ref": "#/$defs/FeatValue" + } + }, + "required": [ + "feature", + "value" + ] + }, + "Grade": { + "type": "integer", + "maximum": 1000, + "minimum": 0 + }, + "InvId": { + "type": "string", + "pattern": "^(concept|frame|role|relation|speech_act|feature|referent_kind|extension):[1-9][0-9]*$" + }, + "NodeId": { + "type": "string", + "pattern": "^n[1-9][0-9]{0,3}$" + }, + "OptionLabel": { + "type": "string", + "pattern": "^[a-z][a-z0-9_]{0,23}$" + }, + "PacketId": { + "type": "string", + "pattern": "^packet:[0-9a-f]{16}$" + }, + "PacketKind": { + "type": "string", + "enum": [ + "Utterance", + "NameRequest", + "CommentaryIntent", + "Narration", + "DialogueLine", + "CorpusExample", + "DebugProbe" + ] + }, + "Predication": { + "type": "object", + "properties": { + "concept": { + "description": "Concept id.", + "$ref": "#/$defs/InvId" + }, + "features": { + "type": "array", + "items": { + "$ref": "#/$defs/FeatureValue" + } + }, + "frame": { + "description": "Frame id.", + "$ref": "#/$defs/InvId" + }, + "id": { + "$ref": "#/$defs/NodeId" + }, + "roles": { + "type": "array", + "items": { + "$ref": "#/$defs/RoleFill" + } + } + }, + "required": [ + "id", + "concept", + "frame", + "roles", + "features" + ] + }, + "ProperName": { + "description": "1 to 4 space-joined tokens of letters, apostrophes and hyphens.", + "type": "string", + "maxLength": 64, + "pattern": "^\\p{L}([\\p{L}'ā€™-]{0,22}\\p{L})?( \\p{L}([\\p{L}'ā€™-]{0,22}\\p{L})?){0,3}$" + }, + "Provenance": { + "type": "object", + "properties": { + "confidence": { + "$ref": "#/$defs/Grade" + }, + "corpus_item": { + "description": "Key of the corpus item this packet was minted for.", + "type": [ + "string", + "null" + ] + }, + "encoder": { + "$ref": "#/$defs/EncoderId" + }, + "source": { + "$ref": "#/$defs/ProvenanceSource" + } + }, + "required": [ + "source", + "encoder", + "confidence" + ] + }, + "ProvenanceSource": { + "type": "string", + "enum": [ + "Authored", + "ModelDecomposition", + "GameTelemetry", + "ImportedCorpus" + ] + }, + "Referent": { + "type": "object", + "properties": { + "concept": { + "description": "Concept id.", + "$ref": "#/$defs/InvId" + }, + "features": { + "type": "array", + "items": { + "$ref": "#/$defs/FeatureValue" + } + }, + "id": { + "$ref": "#/$defs/NodeId" + }, + "kind": { + "description": "Referent-kind id.", + "$ref": "#/$defs/InvId" + }, + "name": { + "description": "A proper name, the one place a referent carries wording. It is\nform-bounded text; the scorer masks it.", + "anyOf": [ + { + "$ref": "#/$defs/ProperName" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "id", + "kind", + "concept", + "features" + ] + }, + "Relation": { + "type": "object", + "properties": { + "features": { + "type": "array", + "items": { + "$ref": "#/$defs/FeatureValue" + } + }, + "from": { + "$ref": "#/$defs/NodeId" + }, + "relation": { + "description": "Relation id.", + "$ref": "#/$defs/InvId" + }, + "to": { + "$ref": "#/$defs/NodeId" + } + }, + "required": [ + "relation", + "from", + "to", + "features" + ] + }, + "RoleFill": { + "type": "object", + "properties": { + "filler": { + "$ref": "#/$defs/NodeId" + }, + "role": { + "description": "Role id.", + "$ref": "#/$defs/InvId" + } + }, + "required": [ + "role", + "filler" + ] + }, + "Trace": { + "description": "Authoring and debug notes. Stripped before lowering.", + "type": "object", + "properties": { + "notes": { + "type": "array", + "items": { + "type": "string" + } + }, + "rejected_readings": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "notes", + "rejected_readings" + ] + } + } +} diff --git a/schemas/cultnet/weksa.inventory_entry.schema.json b/schemas/cultnet/weksa.inventory_entry.schema.json new file mode 100644 index 0000000..6187f02 --- /dev/null +++ b/schemas/cultnet/weksa.inventory_entry.schema.json @@ -0,0 +1,104 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "InventoryEntry", + "description": "Key: the entry's id, `:`.", + "type": "object", + "properties": { + "applies_to": { + "description": "Non-empty exactly on features.", + "type": "array", + "items": { + "$ref": "#/$defs/Slot" + } + }, + "gloss": { + "description": "1 to 120 bytes, no control character (checked at admission).", + "type": "string" + }, + "id": { + "$ref": "#/$defs/InvId" + }, + "introduced_rev": { + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "retired_rev": { + "type": [ + "integer", + "null" + ], + "format": "uint32", + "minimum": 0 + }, + "value_kind": { + "description": "Present exactly on features and extensions.", + "anyOf": [ + { + "$ref": "#/$defs/ValueKind" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "id", + "gloss", + "applies_to", + "introduced_rev" + ], + "$defs": { + "InvId": { + "type": "string", + "pattern": "^(concept|frame|role|relation|speech_act|feature|referent_kind|extension):[1-9][0-9]*$" + }, + "OptionLabel": { + "type": "string", + "pattern": "^[a-z][a-z0-9_]{0,23}$" + }, + "OptionSet": { + "type": "array", + "items": { + "$ref": "#/$defs/OptionLabel" + }, + "maxItems": 16, + "minItems": 1, + "uniqueItems": true + }, + "Slot": { + "description": "Where a feature may sit in a packet.", + "type": "string", + "enum": [ + "Discourse", + "Referent", + "Predication", + "Relation" + ] + }, + "ValueKind": { + "oneOf": [ + { + "type": "string", + "enum": [ + "Grade", + "Flag" + ] + }, + { + "type": "object", + "properties": { + "Options": { + "$ref": "#/$defs/OptionSet" + } + }, + "additionalProperties": false, + "required": [ + "Options" + ] + } + ] + } + } +} diff --git a/schemas/cultnet/weksa.inventory_revision.schema.json b/schemas/cultnet/weksa.inventory_revision.schema.json new file mode 100644 index 0000000..97e78be --- /dev/null +++ b/schemas/cultnet/weksa.inventory_revision.schema.json @@ -0,0 +1,69 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "InventoryRevision", + "description": "Key: `rev:`.", + "type": "object", + "properties": { + "changes": { + "type": "array", + "items": { + "$ref": "#/$defs/Change" + } + }, + "frozen": { + "type": "boolean" + }, + "n": { + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "parent": { + "type": [ + "integer", + "null" + ], + "format": "uint32", + "minimum": 0 + } + }, + "required": [ + "n", + "changes", + "frozen" + ], + "$defs": { + "Change": { + "oneOf": [ + { + "type": "object", + "properties": { + "Add": { + "$ref": "#/$defs/InvId" + } + }, + "additionalProperties": false, + "required": [ + "Add" + ] + }, + { + "type": "object", + "properties": { + "Retire": { + "$ref": "#/$defs/InvId" + } + }, + "additionalProperties": false, + "required": [ + "Retire" + ] + } + ] + }, + "InvId": { + "type": "string", + "pattern": "^(concept|frame|role|relation|speech_act|feature|referent_kind|extension):[1-9][0-9]*$" + } + } +} diff --git a/schemas/interlingua-packet.md b/schemas/interlingua-packet.md index b92df21..d02c527 100644 --- a/schemas/interlingua-packet.md +++ b/schemas/interlingua-packet.md @@ -2,9 +2,11 @@ This is the human-facing schema receipt for Weksa interlingua packets. -Executable schema code does not exist yet. When it does, it must reconcile with -this document and [Interlingua Standard](../docs/interlingua-standard.md) in the -same pass. +The executable schema is [`crates/weksa/src/packet.rs`](../crates/weksa/src/packet.rs). +Its generated JSON Schemas are in [`schemas/cultnet/`](../schemas/cultnet/), and +a test fails when they drift. This document and the +[Interlingua Standard](../docs/interlingua-standard.md) describe it; where they +disagree with the types, the types win. ## Authority @@ -16,22 +18,26 @@ phonology, diachrony, and renderer rules. ## Required Fields -- `interlingua_version`: version string. -- `packet_id`: stable local identifier. -- `kind`: broad use case. +- `id`: stable local identifier, the packet's store key. +- `schema_version` and `inventory_rev`: the schema and inventory revision the + packet was written against. +- `kind`: one of the seven use cases in the standard. - at least one semantic payload family: - `referents` - `predications` - `relations` -## Optional Families +## Families -- `provenance`: source and confidence. -- `context`: world, scene, medium, participants, audience. -- `discourse`: speech act, intent, tone, source-side register hint. -- `constraints`: downstream requirements. +- `provenance`: source, corpus item, encoder and confidence. +- `context`: setting, medium, participants, audience. +- `discourse`: speech act, addressee and source-side nuance features. +- `constraints`: nodes to preserve, determinism, latency bound. - `extensions`: namespaced client or setting pressure. -- `trace`: debug and authoring notes. +- `trace`: debug and authoring notes, stripped before lowering. + +Every open slot is an inventory id or a node id. The only free text is a +referent's proper `name` and the trace. ## Extension Contract @@ -46,18 +52,12 @@ Extensions are not a junk drawer. They are provisional fields awaiting evidence. If a field becomes common across clients, promote it into the standard with a version bump. -## Validation Goals - -The eventual executable schema should validate: - -- required top-level fields -- referential integrity between roles and referents -- known `kind` values -- namespace syntax for extensions -- deterministic JSON/YAML serialization -- stable packet IDs inside a corpus or run -- absence of target-language surface forms in source meaning fields - -It should warn, not fail, on unknown extension namespaces unless a runtime bundle -declares a stricter policy. +## Validation +`validate` in `crates/weksa/src/validate.rs` enforces rules V1 to V8 and V10 to V12 +(V9 is retired), listed in the standard: schema version, payload present, +unique node ids, references that resolve, inventory ids that resolve at the +packet's revision, family matching the slot, feature value kind and slot, an +existing inventory revision, no repeated feature, extension, node reference or +role, and at most 64 nodes. A packet that breaks any rule is refused with a typed +refusal naming it, and never reaches a store.