packet-store: weksa crate with typed packet, inventory and corpus stores, validator V1-V8 and V10-V12, derived split, published schemas and CLI (squash of weksa-lowering/packet-store 35fa979..c6190a6; spec r3, Soul s1-s4)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
b7aa687d13
commit
28284cc8db
30 changed files with 6215 additions and 225 deletions
2
.gitignore
vendored
2
.gitignore
vendored
|
|
@ -10,6 +10,7 @@ state/scratch-runtime/
|
|||
.weksa-runs/
|
||||
.weksa-smoke/
|
||||
.weksa/*.cc
|
||||
*.cc.lock
|
||||
.weksa/*.json
|
||||
.weksa/*.log
|
||||
.weksa/*.pid
|
||||
|
|
@ -20,3 +21,4 @@ dist/
|
|||
build/
|
||||
.venv/
|
||||
venv/
|
||||
target/
|
||||
|
|
|
|||
|
|
@ -46,7 +46,7 @@ The live state architecture is ported from EpiphanyAgent in reduced form:
|
|||
- `state/map.yaml` is canonical and slow-changing.
|
||||
- `state/scratch.md` is temporary and disposable.
|
||||
- `schemas/` explains state contracts in human language.
|
||||
- `packages/weksa-state-model` owns executable typed state.
|
||||
- `crates/weksa` owns executable typed state.
|
||||
|
||||
Project truth does not belong in agent personality memory. If Weksa later grows
|
||||
role dossiers or heartbeat scheduling, those surfaces must remain separate from
|
||||
|
|
|
|||
910
Cargo.lock
generated
Normal file
910
Cargo.lock
generated
Normal file
|
|
@ -0,0 +1,910 @@
|
|||
# This file is automatically @generated by Cargo.
|
||||
# It is not intended for manual editing.
|
||||
version = 4
|
||||
|
||||
[[package]]
|
||||
name = "android_system_properties"
|
||||
version = "0.1.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc"
|
||||
dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anstream"
|
||||
version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d"
|
||||
dependencies = [
|
||||
"anstyle",
|
||||
"anstyle-parse",
|
||||
"anstyle-query",
|
||||
"anstyle-wincon",
|
||||
"colorchoice",
|
||||
"is_terminal_polyfill",
|
||||
"utf8parse",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anstyle"
|
||||
version = "1.0.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000"
|
||||
|
||||
[[package]]
|
||||
name = "anstyle-parse"
|
||||
version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e"
|
||||
dependencies = [
|
||||
"utf8parse",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anstyle-query"
|
||||
version = "1.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc"
|
||||
dependencies = [
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anstyle-wincon"
|
||||
version = "3.0.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d"
|
||||
dependencies = [
|
||||
"anstyle",
|
||||
"once_cell_polyfill",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anyhow"
|
||||
version = "1.0.104"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470"
|
||||
|
||||
[[package]]
|
||||
name = "ar_archive_writer"
|
||||
version = "0.5.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "73cd58deff2140a0a8eae87e417bd01db68a33e148aa93d1e8cd837e55e312b6"
|
||||
dependencies = [
|
||||
"object",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "autocfg"
|
||||
version = "1.5.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"
|
||||
|
||||
[[package]]
|
||||
name = "bitflags"
|
||||
version = "2.13.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06"
|
||||
|
||||
[[package]]
|
||||
name = "bumpalo"
|
||||
version = "3.20.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649"
|
||||
|
||||
[[package]]
|
||||
name = "cc"
|
||||
version = "1.10.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "86e055fa6e3e40ad6187ba25b6855eeb635d5271f8e0f0e1e9a5826b8d75bea0"
|
||||
dependencies = [
|
||||
"find-msvc-tools",
|
||||
"shlex",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cfg-if"
|
||||
version = "1.0.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600"
|
||||
|
||||
[[package]]
|
||||
name = "chrono"
|
||||
version = "0.4.45"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327"
|
||||
dependencies = [
|
||||
"iana-time-zone",
|
||||
"js-sys",
|
||||
"num-traits",
|
||||
"serde",
|
||||
"wasm-bindgen",
|
||||
"windows-link",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "clap"
|
||||
version = "4.6.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "aa8876b300ab35ba921adea3dfd70157a46249b33f95c9084ae5709785478946"
|
||||
dependencies = [
|
||||
"clap_builder",
|
||||
"clap_derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "clap_builder"
|
||||
version = "4.6.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ec0797fb7aeb1406c84efac526901f7ec3ead2124f946b494e72879d4b54704d"
|
||||
dependencies = [
|
||||
"anstream",
|
||||
"anstyle",
|
||||
"clap_lex",
|
||||
"strsim",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "clap_derive"
|
||||
version = "4.6.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f9c751b79415d4e559e3d1fcf128e09e720eb673a06d26cf6f392d37d75b66e0"
|
||||
dependencies = [
|
||||
"heck",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.9",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "clap_lex"
|
||||
version = "1.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1c133bc6a41be0d194c306b5506d15e6feeea7b1d6604bd3f8310dfb2ca96486"
|
||||
|
||||
[[package]]
|
||||
name = "colorchoice"
|
||||
version = "1.0.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570"
|
||||
|
||||
[[package]]
|
||||
name = "core-foundation-sys"
|
||||
version = "0.8.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"
|
||||
|
||||
[[package]]
|
||||
name = "cultcache-rs"
|
||||
version = "0.3.1"
|
||||
source = "git+https://github.com/GameCult/CultLib.git?rev=64b86e0a273a3936939357e9deb186b7bf3bc417#64b86e0a273a3936939357e9deb186b7bf3bc417"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"chrono",
|
||||
"cultcache-rs-derive",
|
||||
"fs2",
|
||||
"redb",
|
||||
"rmp-serde",
|
||||
"serde",
|
||||
"serde_bytes",
|
||||
"serde_json",
|
||||
"serde_stacker",
|
||||
"stacker",
|
||||
"uuid",
|
||||
"windows-sys 0.59.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cultcache-rs-derive"
|
||||
version = "0.2.0"
|
||||
source = "git+https://github.com/GameCult/CultLib.git?rev=64b86e0a273a3936939357e9deb186b7bf3bc417#64b86e0a273a3936939357e9deb186b7bf3bc417"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "dyn-clone"
|
||||
version = "1.0.20"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555"
|
||||
|
||||
[[package]]
|
||||
name = "errno"
|
||||
version = "0.3.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "fastrand"
|
||||
version = "2.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223"
|
||||
|
||||
[[package]]
|
||||
name = "find-msvc-tools"
|
||||
version = "0.1.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1c66915cc5391097de7a2b0b972e8505aa33c0df401fd3a6811dbdb32881ba48"
|
||||
|
||||
[[package]]
|
||||
name = "fs2"
|
||||
version = "0.4.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "futures-core"
|
||||
version = "0.3.34"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e"
|
||||
|
||||
[[package]]
|
||||
name = "futures-task"
|
||||
version = "0.3.34"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd"
|
||||
|
||||
[[package]]
|
||||
name = "futures-util"
|
||||
version = "0.3.34"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc"
|
||||
dependencies = [
|
||||
"futures-core",
|
||||
"futures-task",
|
||||
"pin-project-lite",
|
||||
"slab",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "getrandom"
|
||||
version = "0.4.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"libc",
|
||||
"r-efi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "heck"
|
||||
version = "0.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
|
||||
|
||||
[[package]]
|
||||
name = "iana-time-zone"
|
||||
version = "0.1.65"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470"
|
||||
dependencies = [
|
||||
"android_system_properties",
|
||||
"core-foundation-sys",
|
||||
"iana-time-zone-haiku",
|
||||
"js-sys",
|
||||
"log",
|
||||
"wasm-bindgen",
|
||||
"windows-core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "iana-time-zone-haiku"
|
||||
version = "0.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f"
|
||||
dependencies = [
|
||||
"cc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "is_terminal_polyfill"
|
||||
version = "1.70.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695"
|
||||
|
||||
[[package]]
|
||||
name = "itoa"
|
||||
version = "1.0.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
|
||||
|
||||
[[package]]
|
||||
name = "js-sys"
|
||||
version = "0.3.106"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7883d941dae510fb2d978fc3fe018c71c9e2892fd38854de3e8b92c2e5ad9cc5"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"futures-util",
|
||||
"wasm-bindgen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "libc"
|
||||
version = "0.2.191"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a"
|
||||
|
||||
[[package]]
|
||||
name = "linux-raw-sys"
|
||||
version = "0.12.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53"
|
||||
|
||||
[[package]]
|
||||
name = "log"
|
||||
version = "0.4.34"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6"
|
||||
|
||||
[[package]]
|
||||
name = "memchr"
|
||||
version = "2.8.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"
|
||||
|
||||
[[package]]
|
||||
name = "num-traits"
|
||||
version = "0.2.19"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "object"
|
||||
version = "0.39.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2e5a6c098c7a3b6547378093f5cc30bc54fd361ce711e05293a5cc589562739b"
|
||||
dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "once_cell"
|
||||
version = "1.21.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
|
||||
|
||||
[[package]]
|
||||
name = "once_cell_polyfill"
|
||||
version = "1.70.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe"
|
||||
|
||||
[[package]]
|
||||
name = "pin-project-lite"
|
||||
version = "0.2.17"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd"
|
||||
|
||||
[[package]]
|
||||
name = "proc-macro2"
|
||||
version = "1.0.107"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9"
|
||||
dependencies = [
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "psm"
|
||||
version = "0.1.32"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4dcd034599e63b970727f70d79e02d62390a4a84f7c6b827c27c46d5ac3fa622"
|
||||
dependencies = [
|
||||
"ar_archive_writer",
|
||||
"cc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quote"
|
||||
version = "1.0.47"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "r-efi"
|
||||
version = "6.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf"
|
||||
|
||||
[[package]]
|
||||
name = "redb"
|
||||
version = "4.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "39178e0a867f9e3eaa6292a4ab1dd3613f2519c5b564341e1e7b29b270736cc4"
|
||||
dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ref-cast"
|
||||
version = "1.0.27"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7e440fb4e4b4147295338efb76001ab9e4efc0e5839df2c47fc5ac2381d365c3"
|
||||
dependencies = [
|
||||
"ref-cast-impl",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ref-cast-impl"
|
||||
version = "1.0.27"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "92ecd8964f8453721699a1ed72037b0db49ce2f5a5138486ee89bed6f67cdf3a"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.9",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rmp"
|
||||
version = "0.8.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4ba8be72d372b2c9b35542551678538b562e7cf86c3315773cae48dfbfe7790c"
|
||||
dependencies = [
|
||||
"num-traits",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rmp-serde"
|
||||
version = "1.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "72f81bee8c8ef9b577d1681a70ebbc962c232461e397b22c208c43c04b67a155"
|
||||
dependencies = [
|
||||
"rmp",
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustix"
|
||||
version = "1.1.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e5e76742e5294153d4dde1207dbec5bdd81eef6fabdc0d2118babadabfd858aa"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
"errno",
|
||||
"libc",
|
||||
"linux-raw-sys",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustversion"
|
||||
version = "1.0.23"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f"
|
||||
|
||||
[[package]]
|
||||
name = "schemars"
|
||||
version = "1.2.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ab508826f74a77ca9d5aba6ff19b522583ee3eaf28a19384ff3d0e5835fadf6e"
|
||||
dependencies = [
|
||||
"dyn-clone",
|
||||
"ref-cast",
|
||||
"schemars_derive",
|
||||
"serde",
|
||||
"serde_json",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "schemars_derive"
|
||||
version = "1.2.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e1c3a92094fa7d61aa124645844facb6b554dfc797136d0f5fd1f890e2bffc69"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"serde_derive_internals",
|
||||
"syn 3.0.9",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde"
|
||||
version = "1.0.229"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba"
|
||||
dependencies = [
|
||||
"serde_core",
|
||||
"serde_derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_bytes"
|
||||
version = "0.11.19"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a5d440709e79d88e51ac01c4b72fc6cb7314017bb7da9eeff678aa94c10e3ea8"
|
||||
dependencies = [
|
||||
"serde",
|
||||
"serde_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_core"
|
||||
version = "1.0.229"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48"
|
||||
dependencies = [
|
||||
"serde_derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_derive"
|
||||
version = "1.0.229"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.9",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_derive_internals"
|
||||
version = "0.30.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f852137cce035d6a4df67ccce505ff6b3e9fd3a10e3e52b24dc71e650bb1a9bd"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.9",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_json"
|
||||
version = "1.0.154"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6"
|
||||
dependencies = [
|
||||
"itoa",
|
||||
"memchr",
|
||||
"serde",
|
||||
"serde_core",
|
||||
"zmij",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_stacker"
|
||||
version = "0.1.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d4936375d50c4be7eff22293a9344f8e46f323ed2b3c243e52f89138d9bb0f4a"
|
||||
dependencies = [
|
||||
"serde",
|
||||
"serde_core",
|
||||
"stacker",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "shlex"
|
||||
version = "2.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba"
|
||||
|
||||
[[package]]
|
||||
name = "slab"
|
||||
version = "0.4.12"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5"
|
||||
|
||||
[[package]]
|
||||
name = "stacker"
|
||||
version = "0.1.25"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "707f49d46706bacf8a2b00d51dace3f9de527c13eec3778f570c411f89e69967"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"cfg-if",
|
||||
"libc",
|
||||
"psm",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "strsim"
|
||||
version = "0.11.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "2.0.119"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "3.0.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tempfile"
|
||||
version = "3.27.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
|
||||
dependencies = [
|
||||
"fastrand",
|
||||
"getrandom",
|
||||
"once_cell",
|
||||
"rustix",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "unicode-ident"
|
||||
version = "1.0.27"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563"
|
||||
|
||||
[[package]]
|
||||
name = "utf8parse"
|
||||
version = "0.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
|
||||
|
||||
[[package]]
|
||||
name = "uuid"
|
||||
version = "1.29.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4a0e2d11620c50695fe12bbbfde96b3477fe606e7284c22d505c12ce51214514"
|
||||
dependencies = [
|
||||
"getrandom",
|
||||
"js-sys",
|
||||
"wasm-bindgen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen"
|
||||
version = "0.2.129"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9bb54f33acc68fd454578d9820b0bde1a1a3d17aa17bb7b6595806d02886d409"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"once_cell",
|
||||
"rustversion",
|
||||
"wasm-bindgen-macro",
|
||||
"wasm-bindgen-shared",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen-macro"
|
||||
version = "0.2.129"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2e29d0c35b16e224a7eeb5cd2d25e3e1968fbd65604117b44d3b789d00ee8535"
|
||||
dependencies = [
|
||||
"quote",
|
||||
"wasm-bindgen-macro-support",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen-macro-support"
|
||||
version = "0.2.129"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6f501a8bc3719dba86ef8ae4728879c08001bea749eb1333ac5b91e040e2a6b7"
|
||||
dependencies = [
|
||||
"bumpalo",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.9",
|
||||
"wasm-bindgen-shared",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen-shared"
|
||||
version = "0.2.129"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "23f0c9c52aa7cd7d77769a4cfe2a9adb1b331f489a41d912ce14513d5ab995c6"
|
||||
dependencies = [
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "weksa"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"clap",
|
||||
"cultcache-rs",
|
||||
"schemars",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"tempfile",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "winapi"
|
||||
version = "0.3.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
|
||||
dependencies = [
|
||||
"winapi-i686-pc-windows-gnu",
|
||||
"winapi-x86_64-pc-windows-gnu",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "winapi-i686-pc-windows-gnu"
|
||||
version = "0.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
|
||||
|
||||
[[package]]
|
||||
name = "winapi-x86_64-pc-windows-gnu"
|
||||
version = "0.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
|
||||
|
||||
[[package]]
|
||||
name = "windows-core"
|
||||
version = "0.62.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb"
|
||||
dependencies = [
|
||||
"windows-implement",
|
||||
"windows-interface",
|
||||
"windows-link",
|
||||
"windows-result",
|
||||
"windows-strings",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-implement"
|
||||
version = "0.60.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-interface"
|
||||
version = "0.59.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-link"
|
||||
version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"
|
||||
|
||||
[[package]]
|
||||
name = "windows-result"
|
||||
version = "0.4.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5"
|
||||
dependencies = [
|
||||
"windows-link",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-strings"
|
||||
version = "0.5.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091"
|
||||
dependencies = [
|
||||
"windows-link",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-sys"
|
||||
version = "0.59.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b"
|
||||
dependencies = [
|
||||
"windows-targets",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-sys"
|
||||
version = "0.61.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc"
|
||||
dependencies = [
|
||||
"windows-link",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-targets"
|
||||
version = "0.52.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973"
|
||||
dependencies = [
|
||||
"windows_aarch64_gnullvm",
|
||||
"windows_aarch64_msvc",
|
||||
"windows_i686_gnu",
|
||||
"windows_i686_gnullvm",
|
||||
"windows_i686_msvc",
|
||||
"windows_x86_64_gnu",
|
||||
"windows_x86_64_gnullvm",
|
||||
"windows_x86_64_msvc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows_aarch64_gnullvm"
|
||||
version = "0.52.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"
|
||||
|
||||
[[package]]
|
||||
name = "windows_aarch64_msvc"
|
||||
version = "0.52.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"
|
||||
|
||||
[[package]]
|
||||
name = "windows_i686_gnu"
|
||||
version = "0.52.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b"
|
||||
|
||||
[[package]]
|
||||
name = "windows_i686_gnullvm"
|
||||
version = "0.52.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"
|
||||
|
||||
[[package]]
|
||||
name = "windows_i686_msvc"
|
||||
version = "0.52.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"
|
||||
|
||||
[[package]]
|
||||
name = "windows_x86_64_gnu"
|
||||
version = "0.52.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"
|
||||
|
||||
[[package]]
|
||||
name = "windows_x86_64_gnullvm"
|
||||
version = "0.52.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"
|
||||
|
||||
[[package]]
|
||||
name = "windows_x86_64_msvc"
|
||||
version = "0.52.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
|
||||
|
||||
[[package]]
|
||||
name = "zmij"
|
||||
version = "1.0.23"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b"
|
||||
3
Cargo.toml
Normal file
3
Cargo.toml
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
[workspace]
|
||||
resolver = "3"
|
||||
members = ["crates/weksa"]
|
||||
13
README.md
13
README.md
|
|
@ -17,14 +17,20 @@ measured.
|
|||
**Entry points.**
|
||||
|
||||
- [Interlingua Standard](./docs/interlingua-standard.md): the packet contract.
|
||||
- `crates/weksa`: the packet, inventory and corpus types, `validate`, the
|
||||
derived split and the `weksa` CLI (`weksa validate`, `corpus`, `inventory`,
|
||||
`split`, `schema`); published schemas in `schemas/cultnet/`.
|
||||
- `data/corpus/corpus.cc` and `data/interlingua/inventory.cc`: the English
|
||||
corpus and the inventory, read with `weksa corpus dump` and
|
||||
`weksa inventory dump`.
|
||||
- [Lowering target](./docs/targets/interlingua-lowering.md) and
|
||||
[its map](./docs/targets/interlingua-lowering-map.md): the design, scoring
|
||||
and cut sequence.
|
||||
- `data/`: the cultural ontology and target-language profile data.
|
||||
- `settings/`: setting-specific proving grounds for language projects.
|
||||
|
||||
**Live versus planned.** Live: docs, fixtures, examples and ontology data.
|
||||
Planned: the packet crate `crates/weksa` (cut `packet-store`), lowering, and
|
||||
**Live versus planned.** Live: docs, fixtures, examples, ontology data and the
|
||||
packet crate `crates/weksa`. Planned: scoring, lowering, and
|
||||
the golf loop that scores lowered English against an expressive corpus. No
|
||||
daemon runs; the earlier Node daemon is parked at tag `parked/weksa-node-body`.
|
||||
|
||||
|
|
@ -85,7 +91,8 @@ AquaSynth or expanding pronunciation coverage.
|
|||
## Implementation Status
|
||||
|
||||
The persistent state spine is live: map, scratch, schemas, docs, examples and
|
||||
data. Packet storage, lowering and the service are planned (see the cut
|
||||
data. Packet types, validation and the corpus and inventory stores are live in
|
||||
`crates/weksa`. Scoring, lowering and the service are planned (see the cut
|
||||
sequence in [the lowering map](./docs/targets/interlingua-lowering-map.md)).
|
||||
Writing and agent state will enter Weksa as intent pressure, and typed utterance
|
||||
documents will leave for AquaSynth.
|
||||
|
|
|
|||
24
crates/weksa/Cargo.toml
Normal file
24
crates/weksa/Cargo.toml
Normal file
|
|
@ -0,0 +1,24 @@
|
|||
[package]
|
||||
name = "weksa"
|
||||
version = "0.1.0"
|
||||
edition = "2024"
|
||||
publish = false
|
||||
|
||||
[lib]
|
||||
name = "weksa"
|
||||
path = "src/lib.rs"
|
||||
|
||||
[[bin]]
|
||||
name = "weksa"
|
||||
path = "src/main.rs"
|
||||
|
||||
[dependencies]
|
||||
anyhow = "1"
|
||||
clap = { version = "4", features = ["derive"] }
|
||||
cultcache-rs = { git = "https://github.com/GameCult/CultLib.git", rev = "64b86e0a273a3936939357e9deb186b7bf3bc417", package = "cultcache-rs" }
|
||||
schemars = "1"
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
serde_json = "1"
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile = "3"
|
||||
77
crates/weksa/src/corpus.rs
Normal file
77
crates/weksa/src/corpus.rs
Normal file
|
|
@ -0,0 +1,77 @@
|
|||
//! The English target corpus. An item's text is immutable; a correction is a new
|
||||
//! item that supersedes the old, and an item is withdrawn, never deleted. The
|
||||
//! source enum has no variant for a source whose terms forbid training.
|
||||
|
||||
use cultcache_rs::DatabaseEntry;
|
||||
use schemars::JsonSchema;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct Turn {
|
||||
pub speaker: String,
|
||||
pub text: String,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||
pub enum Source {
|
||||
/// A line from a cast member's hand-authored fixture.
|
||||
CastAuthored {
|
||||
persona: String,
|
||||
fixture: String,
|
||||
},
|
||||
OperatorWritten,
|
||||
OperatorApproved {
|
||||
note: String,
|
||||
},
|
||||
PublicDomain {
|
||||
work: String,
|
||||
author: String,
|
||||
edition_url: String,
|
||||
},
|
||||
/// Labelled model output; never in the test split, never judged by its own family.
|
||||
ModelWritten {
|
||||
model_family: String,
|
||||
prompt_digest: Option<String>,
|
||||
},
|
||||
}
|
||||
|
||||
/// Key: `item:<source-slug>:<n>`.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, JsonSchema, DatabaseEntry)]
|
||||
#[cultcache(type = "weksa.corpus_item")]
|
||||
pub struct CorpusItem {
|
||||
#[cultcache(key = 0)]
|
||||
pub id: String,
|
||||
#[cultcache(key = 1)]
|
||||
pub text: String,
|
||||
#[cultcache(key = 2)]
|
||||
pub language: String,
|
||||
#[cultcache(key = 3)]
|
||||
pub speaker: String,
|
||||
/// The unit the split hashes: a scene, a fixture, a work.
|
||||
#[cultcache(key = 4)]
|
||||
pub group: String,
|
||||
#[cultcache(key = 5)]
|
||||
pub prior_turns: Vec<Turn>,
|
||||
#[cultcache(key = 6)]
|
||||
pub source: Source,
|
||||
#[cultcache(key = 7)]
|
||||
pub admitted_on: String,
|
||||
/// Key of the item this one corrects.
|
||||
#[cultcache(key = 8)]
|
||||
pub supersedes: Option<String>,
|
||||
#[cultcache(key = 9)]
|
||||
pub withdrawn: bool,
|
||||
}
|
||||
|
||||
/// `item:<slug>:<n>`.
|
||||
pub fn item_key(slug: &str, n: u32) -> String {
|
||||
format!("item:{slug}:{n}")
|
||||
}
|
||||
|
||||
/// The slug of an item key, if it is `item:<slug>:<n>`.
|
||||
pub fn key_slug(key: &str) -> Option<&str> {
|
||||
let rest = key.strip_prefix("item:")?;
|
||||
let (slug, n) = rest.rsplit_once(':')?;
|
||||
n.parse::<u32>().ok()?;
|
||||
(!slug.is_empty()).then_some(slug)
|
||||
}
|
||||
62
crates/weksa/src/form.rs
Normal file
62
crates/weksa/src/form.rs
Normal file
|
|
@ -0,0 +1,62 @@
|
|||
//! Form types. A form lives in a newtype whose only constructor is a fallible
|
||||
//! `parse`; deserializing goes through the same parse, so a malformed value can
|
||||
//! neither be built in Rust nor read back from a store.
|
||||
|
||||
use std::fmt;
|
||||
|
||||
/// A value that is not of the form its type names.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct FormError {
|
||||
pub form: &'static str,
|
||||
pub got: String,
|
||||
}
|
||||
|
||||
impl FormError {
|
||||
pub fn new(form: &'static str, got: impl fmt::Display) -> Self {
|
||||
Self {
|
||||
form,
|
||||
got: got.to_string(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for FormError {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
write!(f, "{:?} is not a {}", self.got, self.form)
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for FormError {}
|
||||
|
||||
/// Wires a string form type (with `parse` and `Display`) to FromStr, serde's
|
||||
/// `try_from`/`into` String and a published schema.
|
||||
macro_rules! string_form {
|
||||
($t:ident, $schema:tt) => {
|
||||
impl std::str::FromStr for $t {
|
||||
type Err = $crate::form::FormError;
|
||||
fn from_str(s: &str) -> Result<Self, Self::Err> {
|
||||
Self::parse(s)
|
||||
}
|
||||
}
|
||||
impl TryFrom<String> for $t {
|
||||
type Error = $crate::form::FormError;
|
||||
fn try_from(s: String) -> Result<Self, Self::Error> {
|
||||
Self::parse(&s)
|
||||
}
|
||||
}
|
||||
impl From<$t> for String {
|
||||
fn from(v: $t) -> String {
|
||||
v.to_string()
|
||||
}
|
||||
}
|
||||
impl schemars::JsonSchema for $t {
|
||||
fn schema_name() -> std::borrow::Cow<'static, str> {
|
||||
stringify!($t).into()
|
||||
}
|
||||
fn json_schema(_: &mut schemars::SchemaGenerator) -> schemars::Schema {
|
||||
schemars::json_schema!($schema)
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
pub(crate) use string_form;
|
||||
456
crates/weksa/src/inventory.rs
Normal file
456
crates/weksa/src/inventory.rs
Normal file
|
|
@ -0,0 +1,456 @@
|
|||
//! The inventory: the ids packets name, as CultCache documents under appended
|
||||
//! revisions. An entry is never edited in place; a changed meaning is a retire
|
||||
//! plus an add, and a retired id still resolves for packets of earlier revisions.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::fmt;
|
||||
use std::num::NonZeroU32;
|
||||
|
||||
use cultcache_rs::DatabaseEntry;
|
||||
use schemars::JsonSchema;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::form::{FormError, string_form};
|
||||
|
||||
#[derive(
|
||||
Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize, JsonSchema,
|
||||
)]
|
||||
pub enum Family {
|
||||
Concept,
|
||||
Frame,
|
||||
Role,
|
||||
Relation,
|
||||
SpeechAct,
|
||||
Feature,
|
||||
ReferentKind,
|
||||
/// A game- or loop-declared packet field; the entry carries its value kind.
|
||||
Extension,
|
||||
}
|
||||
|
||||
impl Family {
|
||||
pub const ALL: [Family; 8] = [
|
||||
Family::Concept,
|
||||
Family::Frame,
|
||||
Family::Role,
|
||||
Family::Relation,
|
||||
Family::SpeechAct,
|
||||
Family::Feature,
|
||||
Family::ReferentKind,
|
||||
Family::Extension,
|
||||
];
|
||||
|
||||
pub fn prefix(self) -> &'static str {
|
||||
match self {
|
||||
Family::Concept => "concept",
|
||||
Family::Frame => "frame",
|
||||
Family::Role => "role",
|
||||
Family::Relation => "relation",
|
||||
Family::SpeechAct => "speech_act",
|
||||
Family::Feature => "feature",
|
||||
Family::ReferentKind => "referent_kind",
|
||||
Family::Extension => "extension",
|
||||
}
|
||||
}
|
||||
|
||||
/// The id of the `n`th entry of this family.
|
||||
pub fn id(self, n: NonZeroU32) -> InvId {
|
||||
InvId { family: self, n }
|
||||
}
|
||||
}
|
||||
|
||||
/// Id of an inventory entry: `<family prefix>:<n>`, `n` a canonical decimal
|
||||
/// (no sign, no leading zero) of at least 1.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
|
||||
#[serde(try_from = "String", into = "String")]
|
||||
pub struct InvId {
|
||||
family: Family,
|
||||
n: NonZeroU32,
|
||||
}
|
||||
|
||||
impl InvId {
|
||||
pub fn parse(s: &str) -> Result<Self, FormError> {
|
||||
let bad = || FormError::new("InvId", s);
|
||||
let (prefix, n) = s.split_once(':').ok_or_else(bad)?;
|
||||
let family = Family::ALL
|
||||
.into_iter()
|
||||
.find(|f| f.prefix() == prefix)
|
||||
.ok_or_else(bad)?;
|
||||
if n.starts_with('0') || !n.bytes().all(|b| b.is_ascii_digit()) {
|
||||
return Err(bad());
|
||||
}
|
||||
let n = n.parse::<NonZeroU32>().map_err(|_| bad())?;
|
||||
Ok(Self { family, n })
|
||||
}
|
||||
|
||||
pub fn family(self) -> Family {
|
||||
self.family
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for InvId {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
write!(f, "{}:{}", self.family.prefix(), self.n)
|
||||
}
|
||||
}
|
||||
|
||||
string_form!(InvId, { "type": "string", "pattern": "^(concept|frame|role|relation|speech_act|feature|referent_kind|extension):[1-9][0-9]*$" });
|
||||
|
||||
/// A value label of an options feature: `^[a-z][a-z0-9_]{0,23}$`.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
|
||||
#[serde(try_from = "String", into = "String")]
|
||||
pub struct OptionLabel(String);
|
||||
|
||||
impl OptionLabel {
|
||||
pub fn parse(s: &str) -> Result<Self, FormError> {
|
||||
let mut bytes = s.bytes();
|
||||
let ok = s.len() <= 24
|
||||
&& bytes.next().is_some_and(|b| b.is_ascii_lowercase())
|
||||
&& bytes.all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'_');
|
||||
if ok {
|
||||
Ok(Self(s.to_string()))
|
||||
} else {
|
||||
Err(FormError::new("OptionLabel", s))
|
||||
}
|
||||
}
|
||||
|
||||
pub fn as_str(&self) -> &str {
|
||||
&self.0
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for OptionLabel {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.write_str(&self.0)
|
||||
}
|
||||
}
|
||||
|
||||
string_form!(OptionLabel, { "type": "string", "pattern": "^[a-z][a-z0-9_]{0,23}$" });
|
||||
|
||||
/// 1 to 16 distinct labels.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(try_from = "Vec<OptionLabel>", into = "Vec<OptionLabel>")]
|
||||
pub struct OptionSet(Vec<OptionLabel>);
|
||||
|
||||
impl OptionSet {
|
||||
pub const MAX: usize = 16;
|
||||
|
||||
pub fn new(labels: Vec<OptionLabel>) -> Result<Self, FormError> {
|
||||
let mut distinct: Vec<&OptionLabel> = labels.iter().collect();
|
||||
distinct.sort();
|
||||
distinct.dedup();
|
||||
if (1..=Self::MAX).contains(&labels.len()) && distinct.len() == labels.len() {
|
||||
Ok(Self(labels))
|
||||
} else {
|
||||
Err(FormError::new(
|
||||
"OptionSet",
|
||||
format!("{} labels", labels.len()),
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
pub fn contains(&self, label: &OptionLabel) -> bool {
|
||||
self.0.contains(label)
|
||||
}
|
||||
}
|
||||
|
||||
impl TryFrom<Vec<OptionLabel>> for OptionSet {
|
||||
type Error = FormError;
|
||||
fn try_from(v: Vec<OptionLabel>) -> Result<Self, FormError> {
|
||||
Self::new(v)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<OptionSet> for Vec<OptionLabel> {
|
||||
fn from(s: OptionSet) -> Self {
|
||||
s.0
|
||||
}
|
||||
}
|
||||
|
||||
impl JsonSchema for OptionSet {
|
||||
fn schema_name() -> std::borrow::Cow<'static, str> {
|
||||
"OptionSet".into()
|
||||
}
|
||||
fn json_schema(g: &mut schemars::SchemaGenerator) -> schemars::Schema {
|
||||
schemars::json_schema!({
|
||||
"type": "array",
|
||||
"items": g.subschema_for::<OptionLabel>(),
|
||||
"minItems": 1,
|
||||
"maxItems": 16,
|
||||
"uniqueItems": true
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Where a feature may sit in a packet.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||
pub enum Slot {
|
||||
Discourse,
|
||||
Referent,
|
||||
Predication,
|
||||
Relation,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||
pub enum ValueKind {
|
||||
Grade,
|
||||
Options(OptionSet),
|
||||
Flag,
|
||||
}
|
||||
|
||||
/// Key: the entry's id, `<family>:<n>`.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, JsonSchema, DatabaseEntry)]
|
||||
#[cultcache(type = "weksa.inventory_entry")]
|
||||
pub struct InventoryEntry {
|
||||
#[cultcache(key = 0)]
|
||||
pub id: InvId,
|
||||
/// 1 to 120 bytes, no control character (checked at admission).
|
||||
#[cultcache(key = 1)]
|
||||
pub gloss: String,
|
||||
/// Present exactly on features and extensions.
|
||||
#[cultcache(key = 2)]
|
||||
pub value_kind: Option<ValueKind>,
|
||||
/// Non-empty exactly on features.
|
||||
#[cultcache(key = 3)]
|
||||
pub applies_to: Vec<Slot>,
|
||||
#[cultcache(key = 4)]
|
||||
pub introduced_rev: u32,
|
||||
#[cultcache(key = 5)]
|
||||
pub retired_rev: Option<u32>,
|
||||
}
|
||||
|
||||
impl InventoryEntry {
|
||||
pub fn family(&self) -> Family {
|
||||
self.id.family()
|
||||
}
|
||||
|
||||
/// Whether the entry's shape matches its family: features carry a value kind
|
||||
/// and slots, extensions a value kind and no slots, nothing else either.
|
||||
pub fn well_formed(&self) -> bool {
|
||||
match self.family() {
|
||||
Family::Feature => self.value_kind.is_some() && !self.applies_to.is_empty(),
|
||||
Family::Extension => self.value_kind.is_some() && self.applies_to.is_empty(),
|
||||
_ => self.value_kind.is_none() && self.applies_to.is_empty(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether a packet of inventory revision `rev` may name this entry.
|
||||
pub fn live_at(&self, rev: u32) -> bool {
|
||||
self.introduced_rev <= rev && self.retired_rev.is_none_or(|r| r > rev)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||
pub enum Change {
|
||||
Add(InvId),
|
||||
Retire(InvId),
|
||||
}
|
||||
|
||||
/// Key: `rev:<n>`.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, JsonSchema, DatabaseEntry)]
|
||||
#[cultcache(type = "weksa.inventory_revision")]
|
||||
pub struct InventoryRevision {
|
||||
#[cultcache(key = 0)]
|
||||
pub n: u32,
|
||||
#[cultcache(key = 1)]
|
||||
pub parent: Option<u32>,
|
||||
#[cultcache(key = 2)]
|
||||
pub changes: Vec<Change>,
|
||||
#[cultcache(key = 3)]
|
||||
pub frozen: bool,
|
||||
}
|
||||
|
||||
pub fn revision_key(n: u32) -> String {
|
||||
format!("rev:{n}")
|
||||
}
|
||||
|
||||
/// The whole inventory as the store holds it.
|
||||
#[derive(Clone, Debug, Default)]
|
||||
pub struct Inventory {
|
||||
entries: BTreeMap<InvId, InventoryEntry>,
|
||||
revisions: BTreeMap<u32, InventoryRevision>,
|
||||
}
|
||||
|
||||
impl Inventory {
|
||||
/// An inventory exists only as `InventoryStore::inventory` returns it, so
|
||||
/// no caller can validate or admit against a fabricated one.
|
||||
///
|
||||
/// ```compile_fail,E0624
|
||||
/// // `Inventory::new` is not callable outside the crate.
|
||||
/// let _ = weksa::Inventory::new(vec![], vec![]);
|
||||
/// ```
|
||||
pub(crate) fn new(entries: Vec<InventoryEntry>, revisions: Vec<InventoryRevision>) -> Self {
|
||||
Self {
|
||||
entries: entries.into_iter().map(|e| (e.id, e)).collect(),
|
||||
revisions: revisions.into_iter().map(|r| (r.n, r)).collect(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn entries(&self) -> impl Iterator<Item = &InventoryEntry> {
|
||||
self.entries.values()
|
||||
}
|
||||
|
||||
pub fn entry(&self, id: &InvId) -> Option<&InventoryEntry> {
|
||||
self.entries.get(id)
|
||||
}
|
||||
|
||||
pub fn revisions(&self) -> impl Iterator<Item = &InventoryRevision> {
|
||||
self.revisions.values()
|
||||
}
|
||||
|
||||
pub fn has_revision(&self, n: u32) -> bool {
|
||||
self.revisions.contains_key(&n)
|
||||
}
|
||||
|
||||
pub fn latest(&self) -> Option<&InventoryRevision> {
|
||||
self.revisions.values().next_back()
|
||||
}
|
||||
|
||||
/// The entry `id`, if a packet of revision `rev` may name it.
|
||||
pub fn resolve(&self, id: &InvId, rev: u32) -> Option<&InventoryEntry> {
|
||||
self.entry(id).filter(|e| e.live_at(rev))
|
||||
}
|
||||
}
|
||||
|
||||
/// Revision 1: the standard's enumerated values and the nuance features the
|
||||
/// loop starts from. Concepts start empty.
|
||||
pub fn seed() -> (InventoryRevision, Vec<InventoryEntry>) {
|
||||
let mut entries: Vec<InventoryEntry> = Vec::new();
|
||||
let mut add = |family: Family, gloss: &str, kind: Option<ValueKind>, slots: &[Slot]| {
|
||||
let taken = entries.iter().filter(|e| e.family() == family).count() as u32;
|
||||
entries.push(InventoryEntry {
|
||||
id: family.id(NonZeroU32::MIN.saturating_add(taken)),
|
||||
gloss: gloss.to_string(),
|
||||
value_kind: kind,
|
||||
applies_to: slots.to_vec(),
|
||||
introduced_rev: 1,
|
||||
retired_rev: None,
|
||||
});
|
||||
};
|
||||
for g in [
|
||||
"person",
|
||||
"group",
|
||||
"organism",
|
||||
"place",
|
||||
"artifact",
|
||||
"institution",
|
||||
"ecology",
|
||||
"route",
|
||||
"memory",
|
||||
"event",
|
||||
"abstract",
|
||||
] {
|
||||
add(Family::ReferentKind, g, None, &[]);
|
||||
}
|
||||
for g in [
|
||||
"coreference",
|
||||
"part_of",
|
||||
"member_of",
|
||||
"located_at",
|
||||
"near",
|
||||
"before",
|
||||
"after",
|
||||
"causes",
|
||||
"enables",
|
||||
"prevents",
|
||||
"owns",
|
||||
"owes",
|
||||
"knows",
|
||||
"names",
|
||||
"represents",
|
||||
"contrasts_with",
|
||||
] {
|
||||
add(Family::Relation, g, None, &[]);
|
||||
}
|
||||
for g in [
|
||||
"agent",
|
||||
"patient",
|
||||
"experiencer",
|
||||
"instrument",
|
||||
"source",
|
||||
"goal",
|
||||
"location",
|
||||
"beneficiary",
|
||||
] {
|
||||
add(Family::Role, g, None, &[]);
|
||||
}
|
||||
for g in [
|
||||
"assert",
|
||||
"ask",
|
||||
"request",
|
||||
"promise",
|
||||
"warn",
|
||||
"greet",
|
||||
"thank",
|
||||
"apologise",
|
||||
] {
|
||||
add(Family::SpeechAct, g, None, &[]);
|
||||
}
|
||||
let options = |labels: &[&str]| {
|
||||
let labels = labels
|
||||
.iter()
|
||||
.map(|l| OptionLabel::parse(l).expect("seed labels are well formed"))
|
||||
.collect();
|
||||
Some(ValueKind::Options(
|
||||
OptionSet::new(labels).expect("seed option sets are well formed"),
|
||||
))
|
||||
};
|
||||
let pred = [Slot::Predication];
|
||||
add(
|
||||
Family::Feature,
|
||||
"polarity",
|
||||
options(&["positive", "negative"]),
|
||||
&pred,
|
||||
);
|
||||
add(
|
||||
Family::Feature,
|
||||
"tense",
|
||||
options(&["past", "present", "future", "timeless"]),
|
||||
&pred,
|
||||
);
|
||||
add(
|
||||
Family::Feature,
|
||||
"aspect",
|
||||
options(&[
|
||||
"perfective",
|
||||
"imperfective",
|
||||
"progressive",
|
||||
"habitual",
|
||||
"perfect",
|
||||
]),
|
||||
&pred,
|
||||
);
|
||||
add(
|
||||
Family::Feature,
|
||||
"modality",
|
||||
options(&[
|
||||
"actual",
|
||||
"potential",
|
||||
"possible",
|
||||
"necessary",
|
||||
"desired",
|
||||
"obligated",
|
||||
]),
|
||||
&pred,
|
||||
);
|
||||
add(
|
||||
Family::Feature,
|
||||
"evidentiality",
|
||||
options(&["direct", "inferred", "reported", "assumed"]),
|
||||
&pred,
|
||||
);
|
||||
let nuance = [Slot::Discourse, Slot::Predication];
|
||||
for g in ["hedge", "certainty", "warmth", "formality", "intensity"] {
|
||||
add(Family::Feature, g, Some(ValueKind::Grade), &nuance);
|
||||
}
|
||||
add(Family::Feature, "irony", Some(ValueKind::Flag), &nuance);
|
||||
let changes = entries.iter().map(|e| Change::Add(e.id)).collect();
|
||||
(
|
||||
InventoryRevision {
|
||||
n: 1,
|
||||
parent: None,
|
||||
changes,
|
||||
frozen: false,
|
||||
},
|
||||
entries,
|
||||
)
|
||||
}
|
||||
25
crates/weksa/src/lib.rs
Normal file
25
crates/weksa/src/lib.rs
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
//! Weksa's typed interlingua state: packet, inventory and corpus documents in
|
||||
//! CultCache, the validator that decides whether a packet exists, the derived
|
||||
//! train/validation/test split, and the stores that admit them.
|
||||
|
||||
pub mod corpus;
|
||||
pub mod form;
|
||||
pub mod inventory;
|
||||
pub mod packet;
|
||||
pub mod schema;
|
||||
pub mod split;
|
||||
pub mod store;
|
||||
pub mod validate;
|
||||
|
||||
pub use corpus::{CorpusItem, Source, Turn};
|
||||
pub use form::FormError;
|
||||
pub use inventory::{
|
||||
Change, Family, InvId, Inventory, InventoryEntry, InventoryRevision, OptionLabel, OptionSet,
|
||||
Slot, ValueKind,
|
||||
};
|
||||
pub use packet::{
|
||||
EncoderId, Grade, NodeId, Packet, PacketId, PacketKind, ProperName, SCHEMA_VERSION,
|
||||
};
|
||||
pub use split::{Split, SplitScheme};
|
||||
pub use store::{AddOutcome, CorpusStore, InventoryStore, PacketStore, StoreError, StoreRefusal};
|
||||
pub use validate::{LoweringInput, Refusal, validate};
|
||||
272
crates/weksa/src/main.rs
Normal file
272
crates/weksa/src/main.rs
Normal file
|
|
@ -0,0 +1,272 @@
|
|||
//! The `weksa` command line: the same validator and store admission the library
|
||||
//! exposes, plus text projections (`dump`) that are for review and are not stores.
|
||||
|
||||
use std::path::PathBuf;
|
||||
|
||||
use anyhow::{Context, Result, bail};
|
||||
use clap::{Args, Parser, Subcommand, ValueEnum};
|
||||
use weksa::corpus::{CorpusItem, Source, Turn};
|
||||
use weksa::inventory::seed;
|
||||
use weksa::{CorpusStore, InventoryStore, PacketStore, SplitScheme, StoreError, validate};
|
||||
|
||||
#[derive(Parser)]
|
||||
#[command(
|
||||
name = "weksa",
|
||||
about = "Weksa interlingua packet, inventory and corpus tools"
|
||||
)]
|
||||
struct Cli {
|
||||
/// The corpus store.
|
||||
#[arg(long, global = true, default_value = "data/corpus/corpus.cc")]
|
||||
corpus: PathBuf,
|
||||
/// The inventory store.
|
||||
#[arg(long, global = true, default_value = "data/interlingua/inventory.cc")]
|
||||
inventory: PathBuf,
|
||||
#[command(subcommand)]
|
||||
command: Command,
|
||||
}
|
||||
|
||||
#[derive(Subcommand)]
|
||||
enum Command {
|
||||
/// Validate the packets in a store (or one packet) against the inventory.
|
||||
Validate { store: PathBuf, key: Option<String> },
|
||||
#[command(subcommand)]
|
||||
Corpus(CorpusCommand),
|
||||
#[command(subcommand)]
|
||||
Inventory(InventoryCommand),
|
||||
/// Print the split of a corpus item.
|
||||
Split { item_key: String },
|
||||
/// Write the published JSON Schemas.
|
||||
Schema {
|
||||
#[arg(long, default_value = "schemas/cultnet")]
|
||||
out: PathBuf,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Subcommand)]
|
||||
enum CorpusCommand {
|
||||
Add(Box<AddArgs>),
|
||||
/// Text projection of the corpus for review; not a store.
|
||||
Dump,
|
||||
Withdraw {
|
||||
key: String,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Subcommand)]
|
||||
enum InventoryCommand {
|
||||
/// Text projection of the inventory for review; not a store.
|
||||
Dump,
|
||||
/// Admit revision 1, the standard's enumerated values and the nuance features.
|
||||
Seed,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, ValueEnum)]
|
||||
enum SourceKind {
|
||||
CastAuthored,
|
||||
OperatorWritten,
|
||||
OperatorApproved,
|
||||
PublicDomain,
|
||||
ModelWritten,
|
||||
}
|
||||
|
||||
#[derive(Args)]
|
||||
struct AddArgs {
|
||||
#[arg(long)]
|
||||
slug: String,
|
||||
#[arg(long)]
|
||||
text: String,
|
||||
#[arg(long, default_value = "en-US")]
|
||||
language: String,
|
||||
#[arg(long)]
|
||||
speaker: String,
|
||||
#[arg(long)]
|
||||
group: String,
|
||||
/// `speaker=text`, repeated, oldest first.
|
||||
#[arg(long = "prior-turn")]
|
||||
prior_turns: Vec<String>,
|
||||
#[arg(long, value_enum)]
|
||||
source: SourceKind,
|
||||
#[arg(long)]
|
||||
persona: Option<String>,
|
||||
#[arg(long)]
|
||||
fixture: Option<String>,
|
||||
#[arg(long)]
|
||||
note: Option<String>,
|
||||
#[arg(long)]
|
||||
work: Option<String>,
|
||||
#[arg(long)]
|
||||
author: Option<String>,
|
||||
#[arg(long)]
|
||||
edition_url: Option<String>,
|
||||
#[arg(long)]
|
||||
model_family: Option<String>,
|
||||
#[arg(long)]
|
||||
prompt_digest: Option<String>,
|
||||
#[arg(long)]
|
||||
admitted_on: String,
|
||||
#[arg(long)]
|
||||
supersedes: Option<String>,
|
||||
}
|
||||
|
||||
fn need(value: Option<String>, flag: &str) -> Result<String> {
|
||||
value.with_context(|| format!("this source needs --{flag}"))
|
||||
}
|
||||
|
||||
impl AddArgs {
|
||||
fn source(&self) -> Result<Source> {
|
||||
let get = |v: &Option<String>, flag: &str| need(v.clone(), flag);
|
||||
Ok(match self.source {
|
||||
SourceKind::CastAuthored => Source::CastAuthored {
|
||||
persona: get(&self.persona, "persona")?,
|
||||
fixture: get(&self.fixture, "fixture")?,
|
||||
},
|
||||
SourceKind::OperatorWritten => Source::OperatorWritten,
|
||||
SourceKind::OperatorApproved => Source::OperatorApproved {
|
||||
note: get(&self.note, "note")?,
|
||||
},
|
||||
SourceKind::PublicDomain => Source::PublicDomain {
|
||||
work: get(&self.work, "work")?,
|
||||
author: get(&self.author, "author")?,
|
||||
edition_url: get(&self.edition_url, "edition-url")?,
|
||||
},
|
||||
SourceKind::ModelWritten => Source::ModelWritten {
|
||||
model_family: get(&self.model_family, "model-family")?,
|
||||
prompt_digest: self.prompt_digest.clone(),
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
fn item(self, id: String) -> Result<CorpusItem> {
|
||||
let prior_turns = self
|
||||
.prior_turns
|
||||
.iter()
|
||||
.map(|t| {
|
||||
let (speaker, text) = t
|
||||
.split_once('=')
|
||||
.with_context(|| format!("--prior-turn {t:?} is not speaker=text"))?;
|
||||
Ok(Turn {
|
||||
speaker: speaker.to_string(),
|
||||
text: text.to_string(),
|
||||
})
|
||||
})
|
||||
.collect::<Result<Vec<_>>>()?;
|
||||
Ok(CorpusItem {
|
||||
id,
|
||||
source: self.source()?,
|
||||
text: self.text,
|
||||
language: self.language,
|
||||
speaker: self.speaker,
|
||||
group: self.group,
|
||||
prior_turns,
|
||||
admitted_on: self.admitted_on,
|
||||
supersedes: self.supersedes,
|
||||
withdrawn: false,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// The split of an item as `Train`, `Validation`, `Test` or `excluded`.
|
||||
fn split_text(item: &CorpusItem) -> String {
|
||||
SplitScheme::V1
|
||||
.split(item)
|
||||
.map_or_else(|| "excluded".to_string(), |s| format!("{s:?}"))
|
||||
}
|
||||
|
||||
fn refused(e: StoreError) -> anyhow::Error {
|
||||
anyhow::anyhow!("{e}")
|
||||
}
|
||||
|
||||
fn main() -> Result<()> {
|
||||
let cli = Cli::parse();
|
||||
match cli.command {
|
||||
Command::Validate { store, key } => {
|
||||
let inventory = InventoryStore::open(&cli.inventory)?.inventory()?;
|
||||
let packets = PacketStore::open(&store)?;
|
||||
let packets = match key {
|
||||
Some(key) => vec![
|
||||
packets
|
||||
.get(&key)?
|
||||
.with_context(|| format!("no packet {key}"))?,
|
||||
],
|
||||
None => packets.all()?,
|
||||
};
|
||||
let mut bad = 0;
|
||||
for p in &packets {
|
||||
match validate(p, &inventory) {
|
||||
Ok(_) => println!("ok {}", p.id),
|
||||
Err(refusals) => {
|
||||
bad += 1;
|
||||
for r in refusals {
|
||||
println!("refused {} {r}", p.id);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if bad > 0 {
|
||||
bail!("{bad} of {} packets refused", packets.len());
|
||||
}
|
||||
}
|
||||
Command::Corpus(CorpusCommand::Add(args)) => {
|
||||
let mut store = CorpusStore::open(&cli.corpus)?;
|
||||
let key = store.next_key(&args.slug)?;
|
||||
let outcome = store.add((*args).item(key.clone())?).map_err(refused)?;
|
||||
println!("{outcome:?} {key}");
|
||||
}
|
||||
Command::Corpus(CorpusCommand::Dump) => {
|
||||
for i in CorpusStore::open(&cli.corpus)?.all()? {
|
||||
let split = split_text(&i);
|
||||
let withdrawn = if i.withdrawn { " withdrawn" } else { "" };
|
||||
println!(
|
||||
"{} [{} {} group={} {split}{withdrawn}]\n {:?}\n source: {:?}",
|
||||
i.id, i.language, i.speaker, i.group, i.text, i.source
|
||||
);
|
||||
}
|
||||
}
|
||||
Command::Corpus(CorpusCommand::Withdraw { key }) => {
|
||||
CorpusStore::open(&cli.corpus)?
|
||||
.withdraw(&key)
|
||||
.map_err(refused)?;
|
||||
println!("withdrawn {key}");
|
||||
}
|
||||
Command::Inventory(InventoryCommand::Dump) => {
|
||||
let inv = InventoryStore::open(&cli.inventory)?.inventory()?;
|
||||
for r in inv.revisions() {
|
||||
println!(
|
||||
"rev:{} parent={:?} frozen={} changes={}",
|
||||
r.n,
|
||||
r.parent,
|
||||
r.frozen,
|
||||
r.changes.len()
|
||||
);
|
||||
}
|
||||
for e in inv.entries() {
|
||||
println!(
|
||||
"{} introduced={} retired={:?} {:?} {:?} {}",
|
||||
e.id, e.introduced_rev, e.retired_rev, e.value_kind, e.applies_to, e.gloss
|
||||
);
|
||||
}
|
||||
}
|
||||
Command::Inventory(InventoryCommand::Seed) => {
|
||||
let (rev, entries) = seed();
|
||||
let count = entries.len();
|
||||
InventoryStore::open(&cli.inventory)?
|
||||
.admit_revision(rev, entries)
|
||||
.map_err(refused)?;
|
||||
println!("seeded rev:1 with {count} entries");
|
||||
}
|
||||
Command::Split { item_key } => {
|
||||
let item = CorpusStore::open(&cli.corpus)?
|
||||
.get(&item_key)?
|
||||
.with_context(|| format!("no item {item_key}"))?;
|
||||
println!("{}", split_text(&item));
|
||||
}
|
||||
Command::Schema { out } => {
|
||||
std::fs::create_dir_all(&out)?;
|
||||
for (name, json) in weksa::schema::published() {
|
||||
std::fs::write(out.join(&name), json)?;
|
||||
println!("wrote {name}");
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
365
crates/weksa/src/packet.rs
Normal file
365
crates/weksa/src/packet.rs
Normal file
|
|
@ -0,0 +1,365 @@
|
|||
//! The interlingua packet. Every id is an opaque form type: a node id `n<k>`, a
|
||||
//! packet id `packet:<hex16>`, an encoder id `encoder:<hex16>`, an inventory id
|
||||
//! `<family>:<n>`. The free text is a bounded [`ProperName`] and the trace, which
|
||||
//! is stripped before lowering. A malformed id or name cannot be built, and
|
||||
//! cannot be read back from a store, so a packet cannot carry target-language
|
||||
//! wording through an id slot.
|
||||
|
||||
use std::fmt;
|
||||
|
||||
use cultcache_rs::DatabaseEntry;
|
||||
use schemars::JsonSchema;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::form::{FormError, string_form};
|
||||
pub use crate::inventory::InvId;
|
||||
use crate::inventory::OptionLabel;
|
||||
|
||||
/// Bumped by a commit that changes these types; the published schema moves with it.
|
||||
pub const SCHEMA_VERSION: u32 = 1;
|
||||
|
||||
/// Id of a referent or predication within one packet: `n<k>`, `k` a canonical
|
||||
/// decimal 1..=9999.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
|
||||
#[serde(try_from = "String", into = "String")]
|
||||
pub struct NodeId(String);
|
||||
|
||||
impl NodeId {
|
||||
pub fn parse(s: &str) -> Result<Self, FormError> {
|
||||
let k = s
|
||||
.strip_prefix('n')
|
||||
.filter(|d| !d.starts_with('0') && d.bytes().all(|b| b.is_ascii_digit()))
|
||||
.and_then(|d| d.parse::<u16>().ok())
|
||||
.filter(|k| (1..=9999).contains(k));
|
||||
k.map(|_| Self(s.to_string()))
|
||||
.ok_or_else(|| FormError::new("NodeId", s))
|
||||
}
|
||||
|
||||
pub fn as_str(&self) -> &str {
|
||||
&self.0
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for NodeId {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.write_str(&self.0)
|
||||
}
|
||||
}
|
||||
|
||||
string_form!(NodeId, { "type": "string", "pattern": "^n[1-9][0-9]{0,3}$" });
|
||||
|
||||
/// `<prefix>:<16 lowercase hex>`.
|
||||
fn hex16(prefix: &str, s: &str) -> bool {
|
||||
s.strip_prefix(prefix)
|
||||
.is_some_and(|h| h.len() == 16 && h.bytes().all(|b| matches!(b, b'0'..=b'9' | b'a'..=b'f')))
|
||||
}
|
||||
|
||||
/// Key of a packet: `packet:<hex16>`, a digest the loop derives from the item,
|
||||
/// the inventory revision and the encoder.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
|
||||
#[serde(try_from = "String", into = "String")]
|
||||
pub struct PacketId(String);
|
||||
|
||||
impl PacketId {
|
||||
pub fn parse(s: &str) -> Result<Self, FormError> {
|
||||
if hex16("packet:", s) {
|
||||
Ok(Self(s.to_string()))
|
||||
} else {
|
||||
Err(FormError::new("PacketId", s))
|
||||
}
|
||||
}
|
||||
|
||||
pub fn as_str(&self) -> &str {
|
||||
&self.0
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for PacketId {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.write_str(&self.0)
|
||||
}
|
||||
}
|
||||
|
||||
string_form!(PacketId, { "type": "string", "pattern": "^packet:[0-9a-f]{16}$" });
|
||||
|
||||
/// `encoder:<hex16>`, a digest of the model id and the prompt digest.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
|
||||
#[serde(try_from = "String", into = "String")]
|
||||
pub struct EncoderId(String);
|
||||
|
||||
impl EncoderId {
|
||||
pub fn parse(s: &str) -> Result<Self, FormError> {
|
||||
if hex16("encoder:", s) {
|
||||
Ok(Self(s.to_string()))
|
||||
} else {
|
||||
Err(FormError::new("EncoderId", s))
|
||||
}
|
||||
}
|
||||
|
||||
pub fn as_str(&self) -> &str {
|
||||
&self.0
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for EncoderId {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.write_str(&self.0)
|
||||
}
|
||||
}
|
||||
|
||||
string_form!(EncoderId, { "type": "string", "pattern": "^encoder:[0-9a-f]{16}$" });
|
||||
|
||||
/// Permille, 0..=1000.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
|
||||
#[serde(try_from = "u16", into = "u16")]
|
||||
pub struct Grade(u16);
|
||||
|
||||
impl Grade {
|
||||
pub const MAX: u16 = 1000;
|
||||
|
||||
pub fn new(permille: u16) -> Result<Self, FormError> {
|
||||
if permille <= Self::MAX {
|
||||
Ok(Self(permille))
|
||||
} else {
|
||||
Err(FormError::new("Grade", permille))
|
||||
}
|
||||
}
|
||||
|
||||
pub fn get(self) -> u16 {
|
||||
self.0
|
||||
}
|
||||
}
|
||||
|
||||
impl TryFrom<u16> for Grade {
|
||||
type Error = FormError;
|
||||
fn try_from(v: u16) -> Result<Self, FormError> {
|
||||
Self::new(v)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<Grade> for u16 {
|
||||
fn from(g: Grade) -> u16 {
|
||||
g.0
|
||||
}
|
||||
}
|
||||
|
||||
impl JsonSchema for Grade {
|
||||
fn schema_name() -> std::borrow::Cow<'static, str> {
|
||||
"Grade".into()
|
||||
}
|
||||
fn json_schema(_: &mut schemars::SchemaGenerator) -> schemars::Schema {
|
||||
schemars::json_schema!({ "type": "integer", "minimum": 0, "maximum": 1000 })
|
||||
}
|
||||
}
|
||||
|
||||
/// A proper name: 1 to 4 tokens joined by single spaces, 64 bytes at most. A
|
||||
/// token is 1 to 24 letters, apostrophes (`'` or U+2019) and hyphens, beginning
|
||||
/// and ending with a letter. The form bounds capacity only: names are text the
|
||||
/// scorer masks, and no door judges them for leakage.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
|
||||
#[serde(try_from = "String", into = "String")]
|
||||
pub struct ProperName(String);
|
||||
|
||||
impl ProperName {
|
||||
pub const MAX_BYTES: usize = 64;
|
||||
pub const MAX_TOKENS: usize = 4;
|
||||
pub const MAX_TOKEN_CHARS: usize = 24;
|
||||
|
||||
pub fn parse(s: &str) -> Result<Self, FormError> {
|
||||
let tokens: Vec<&str> = s.split(' ').collect();
|
||||
if s.len() <= Self::MAX_BYTES
|
||||
&& tokens.len() <= Self::MAX_TOKENS
|
||||
&& tokens.iter().all(|t| Self::token_ok(t))
|
||||
{
|
||||
Ok(Self(s.to_string()))
|
||||
} else {
|
||||
Err(FormError::new("ProperName", s))
|
||||
}
|
||||
}
|
||||
|
||||
fn token_ok(t: &str) -> bool {
|
||||
let (Some(first), Some(last)) = (t.chars().next(), t.chars().next_back()) else {
|
||||
return false;
|
||||
};
|
||||
t.chars().count() <= Self::MAX_TOKEN_CHARS
|
||||
&& first.is_alphabetic()
|
||||
&& last.is_alphabetic()
|
||||
&& t.chars()
|
||||
.all(|c| c.is_alphabetic() || matches!(c, '\'' | '\u{2019}' | '-'))
|
||||
}
|
||||
|
||||
pub fn as_str(&self) -> &str {
|
||||
&self.0
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for ProperName {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.write_str(&self.0)
|
||||
}
|
||||
}
|
||||
|
||||
string_form!(ProperName, {
|
||||
"type": "string",
|
||||
"maxLength": 64,
|
||||
"pattern": "^\\p{L}([\\p{L}'\u{2019}-]{0,22}\\p{L})?( \\p{L}([\\p{L}'\u{2019}-]{0,22}\\p{L})?){0,3}$",
|
||||
"description": "1 to 4 space-joined tokens of letters, apostrophes and hyphens."
|
||||
});
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, JsonSchema, DatabaseEntry)]
|
||||
#[cultcache(type = "weksa.interlingua_packet")]
|
||||
pub struct Packet {
|
||||
#[cultcache(key = 0)]
|
||||
pub id: PacketId,
|
||||
#[cultcache(key = 1)]
|
||||
pub schema_version: u32,
|
||||
#[cultcache(key = 2)]
|
||||
pub inventory_rev: u32,
|
||||
#[cultcache(key = 3)]
|
||||
pub kind: PacketKind,
|
||||
#[cultcache(key = 4)]
|
||||
pub provenance: Provenance,
|
||||
#[cultcache(key = 5)]
|
||||
pub context: Context,
|
||||
#[cultcache(key = 6)]
|
||||
pub discourse: Discourse,
|
||||
#[cultcache(key = 7)]
|
||||
pub referents: Vec<Referent>,
|
||||
#[cultcache(key = 8)]
|
||||
pub predications: Vec<Predication>,
|
||||
#[cultcache(key = 9)]
|
||||
pub relations: Vec<Relation>,
|
||||
#[cultcache(key = 10)]
|
||||
pub constraints: Constraints,
|
||||
#[cultcache(key = 11)]
|
||||
pub extensions: Vec<Extension>,
|
||||
#[cultcache(key = 12)]
|
||||
pub trace: Trace,
|
||||
}
|
||||
|
||||
impl Packet {
|
||||
/// The most referents, predications and relations one packet holds: a
|
||||
/// calibration constant of ours, capacity and not meaning.
|
||||
pub const MAX_NODES: usize = 64;
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||
pub enum PacketKind {
|
||||
Utterance,
|
||||
NameRequest,
|
||||
CommentaryIntent,
|
||||
Narration,
|
||||
DialogueLine,
|
||||
CorpusExample,
|
||||
DebugProbe,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||
pub enum ProvenanceSource {
|
||||
Authored,
|
||||
ModelDecomposition,
|
||||
GameTelemetry,
|
||||
ImportedCorpus,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct Provenance {
|
||||
pub source: ProvenanceSource,
|
||||
/// Key of the corpus item this packet was minted for.
|
||||
pub corpus_item: Option<String>,
|
||||
pub encoder: EncoderId,
|
||||
pub confidence: Grade,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct Context {
|
||||
/// Concept ids.
|
||||
pub setting: Option<InvId>,
|
||||
pub medium: Option<InvId>,
|
||||
pub participants: Vec<NodeId>,
|
||||
pub audience: Vec<NodeId>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct Discourse {
|
||||
/// Speech-act id.
|
||||
pub speech_act: Option<InvId>,
|
||||
pub addressee: Option<NodeId>,
|
||||
pub features: Vec<FeatureValue>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct FeatureValue {
|
||||
/// Feature id.
|
||||
pub feature: InvId,
|
||||
pub value: FeatValue,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||
pub enum FeatValue {
|
||||
Grade(Grade),
|
||||
/// A label the feature's inventory entry declares.
|
||||
Option(OptionLabel),
|
||||
Flag(bool),
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct Referent {
|
||||
pub id: NodeId,
|
||||
/// Referent-kind id.
|
||||
pub kind: InvId,
|
||||
/// Concept id.
|
||||
pub concept: InvId,
|
||||
/// A proper name, the one place a referent carries wording. It is
|
||||
/// form-bounded text; the scorer masks it.
|
||||
pub name: Option<ProperName>,
|
||||
pub features: Vec<FeatureValue>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct Predication {
|
||||
pub id: NodeId,
|
||||
/// Concept id.
|
||||
pub concept: InvId,
|
||||
/// Frame id.
|
||||
pub frame: InvId,
|
||||
pub roles: Vec<RoleFill>,
|
||||
pub features: Vec<FeatureValue>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct RoleFill {
|
||||
/// Role id.
|
||||
pub role: InvId,
|
||||
pub filler: NodeId,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct Relation {
|
||||
/// Relation id.
|
||||
pub relation: InvId,
|
||||
pub from: NodeId,
|
||||
pub to: NodeId,
|
||||
pub features: Vec<FeatureValue>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct Constraints {
|
||||
pub preserve: Vec<NodeId>,
|
||||
pub deterministic: bool,
|
||||
pub max_latency_ms: Option<u32>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct Extension {
|
||||
/// Extension-entry id; the inventory declares its value kind.
|
||||
pub key: InvId,
|
||||
pub value: FeatValue,
|
||||
}
|
||||
|
||||
/// Authoring and debug notes. Stripped before lowering.
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct Trace {
|
||||
pub notes: Vec<String>,
|
||||
pub rejected_readings: Vec<String>,
|
||||
}
|
||||
23
crates/weksa/src/schema.rs
Normal file
23
crates/weksa/src/schema.rs
Normal file
|
|
@ -0,0 +1,23 @@
|
|||
//! JSON Schema publication. JSON is only the xenos-boundary export of the typed
|
||||
//! documents; nothing in this crate reads JSON.
|
||||
|
||||
use crate::corpus::CorpusItem;
|
||||
use crate::inventory::{InventoryEntry, InventoryRevision};
|
||||
use crate::packet::Packet;
|
||||
use cultcache_rs::DatabaseEntry;
|
||||
use schemars::{JsonSchema, schema_for};
|
||||
|
||||
fn published_for<T: DatabaseEntry + JsonSchema>() -> (String, String) {
|
||||
let json = serde_json::to_string_pretty(&schema_for!(T)).expect("a schema serializes");
|
||||
(format!("{}.schema.json", T::TYPE), format!("{json}\n"))
|
||||
}
|
||||
|
||||
/// `(file name, contents)` for every document type, as `schemas/cultnet/` holds them.
|
||||
pub fn published() -> Vec<(String, String)> {
|
||||
vec![
|
||||
published_for::<Packet>(),
|
||||
published_for::<InventoryEntry>(),
|
||||
published_for::<InventoryRevision>(),
|
||||
published_for::<CorpusItem>(),
|
||||
]
|
||||
}
|
||||
54
crates/weksa/src/split.rs
Normal file
54
crates/weksa/src/split.rs
Normal file
|
|
@ -0,0 +1,54 @@
|
|||
//! The train/validation/test split is derived from an item's group and nothing
|
||||
//! else. No stored assignment exists, so no writer can reassign one.
|
||||
|
||||
use crate::corpus::{CorpusItem, Source};
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum Split {
|
||||
Train,
|
||||
Validation,
|
||||
Test,
|
||||
}
|
||||
|
||||
/// A scheme is a constant; a new scheme is a new variant, never a reassignment.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum SplitScheme {
|
||||
V1,
|
||||
}
|
||||
|
||||
pub fn fnv1a64(bytes: &[u8]) -> u64 {
|
||||
let mut h: u64 = 0xcbf2_9ce4_8422_2325;
|
||||
for b in bytes {
|
||||
h ^= u64::from(*b);
|
||||
h = h.wrapping_mul(0x0000_0100_0000_01b3);
|
||||
}
|
||||
h
|
||||
}
|
||||
|
||||
impl SplitScheme {
|
||||
/// 0..100, FNV-1a 64 of `weksa-split-v1:` + group, mod 100.
|
||||
pub fn bucket(self, group: &str) -> u64 {
|
||||
match self {
|
||||
SplitScheme::V1 => fnv1a64(format!("weksa-split-v1:{group}").as_bytes()) % 100,
|
||||
}
|
||||
}
|
||||
|
||||
/// <40 Train, <80 Validation, else Test.
|
||||
pub fn of_bucket(self, bucket: u64) -> Split {
|
||||
match self {
|
||||
SplitScheme::V1 if bucket < 40 => Split::Train,
|
||||
SplitScheme::V1 if bucket < 80 => Split::Validation,
|
||||
SplitScheme::V1 => Split::Test,
|
||||
}
|
||||
}
|
||||
|
||||
/// The split of an item: its group's, except that a model-written item of a
|
||||
/// test group is in no split. It is never moved to train: a model line of
|
||||
/// the test meaning would teach the test.
|
||||
pub fn split(self, item: &CorpusItem) -> Option<Split> {
|
||||
match self.of_bucket(self.bucket(&item.group)) {
|
||||
Split::Test if matches!(item.source, Source::ModelWritten { .. }) => None,
|
||||
split => Some(split),
|
||||
}
|
||||
}
|
||||
}
|
||||
361
crates/weksa/src/store.rs
Normal file
361
crates/weksa/src/store.rs
Normal file
|
|
@ -0,0 +1,361 @@
|
|||
//! Admission to the corpus, inventory and packet stores. Each store is a
|
||||
//! CultCache over one `SingleFileMessagePackBackingStore` file. These functions
|
||||
//! are the only writers, so the rules S1..S4 hold for every caller.
|
||||
|
||||
use std::collections::BTreeSet;
|
||||
use std::fmt;
|
||||
use std::path::Path;
|
||||
|
||||
use cultcache_rs::{CultCache, SingleFileMessagePackBackingStore};
|
||||
|
||||
use crate::corpus::{CorpusItem, item_key, key_slug};
|
||||
use crate::inventory::{Change, InvId, Inventory, InventoryEntry, InventoryRevision, revision_key};
|
||||
use crate::packet::Packet;
|
||||
use crate::validate::{Refusal, validate};
|
||||
|
||||
/// A broken store rule, named.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub enum StoreRefusal {
|
||||
/// S1: an existing item key re-added with different text.
|
||||
S1ItemTextImmutable { key: String },
|
||||
/// S2: a revision that is not latest+1 with parent latest.
|
||||
S2RevisionChain { expected: u32, got: u32 },
|
||||
/// S3: a revision after a frozen one.
|
||||
S3Frozen { frozen: u32 },
|
||||
/// S4: an entry whose introduced_rev is not the revision adding it.
|
||||
S4IntroducedRev {
|
||||
id: InvId,
|
||||
introduced: u32,
|
||||
revision: u32,
|
||||
},
|
||||
/// S5: a superseding item's group differs from its predecessor's.
|
||||
S5SupersedeGroup {
|
||||
key: String,
|
||||
expected: String,
|
||||
got: String,
|
||||
},
|
||||
/// A `provenance.corpus_item` key names no item.
|
||||
UnknownCorpusItem { key: String },
|
||||
/// A packet key already holds a different packet.
|
||||
PacketImmutable { key: String },
|
||||
/// Any other malformed document or change.
|
||||
Malformed { key: String, why: &'static str },
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub enum StoreError {
|
||||
Refused(StoreRefusal),
|
||||
/// The packet broke validator rules.
|
||||
Invalid(Vec<Refusal>),
|
||||
Io(anyhow::Error),
|
||||
}
|
||||
|
||||
impl fmt::Display for StoreError {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
match self {
|
||||
StoreError::Refused(r) => write!(f, "refused: {r:?}"),
|
||||
StoreError::Invalid(rs) => {
|
||||
write!(f, "invalid packet:")?;
|
||||
rs.iter().try_for_each(|r| write!(f, " [{r}]"))
|
||||
}
|
||||
StoreError::Io(e) => write!(f, "{e:#}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for StoreError {}
|
||||
|
||||
impl From<anyhow::Error> for StoreError {
|
||||
fn from(e: anyhow::Error) -> Self {
|
||||
StoreError::Io(e)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<StoreRefusal> for StoreError {
|
||||
fn from(r: StoreRefusal) -> Self {
|
||||
StoreError::Refused(r)
|
||||
}
|
||||
}
|
||||
|
||||
fn malformed(key: &str, why: &'static str) -> StoreError {
|
||||
StoreRefusal::Malformed {
|
||||
key: key.to_string(),
|
||||
why,
|
||||
}
|
||||
.into()
|
||||
}
|
||||
|
||||
fn open_cache(
|
||||
path: &Path,
|
||||
register: impl FnOnce(&mut CultCache) -> anyhow::Result<()>,
|
||||
) -> anyhow::Result<CultCache> {
|
||||
if let Some(dir) = path.parent() {
|
||||
std::fs::create_dir_all(dir)?;
|
||||
}
|
||||
let mut cache = CultCache::new();
|
||||
register(&mut cache)?;
|
||||
cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(path))?;
|
||||
cache.pull_all_backing_stores()?;
|
||||
Ok(cache)
|
||||
}
|
||||
|
||||
#[derive(Debug, PartialEq, Eq)]
|
||||
pub enum AddOutcome {
|
||||
Added,
|
||||
/// The key already held this text; nothing was written.
|
||||
Unchanged,
|
||||
}
|
||||
|
||||
pub struct CorpusStore {
|
||||
cache: CultCache,
|
||||
}
|
||||
|
||||
impl CorpusStore {
|
||||
pub fn open(path: &Path) -> anyhow::Result<Self> {
|
||||
Ok(Self {
|
||||
cache: open_cache(path, |c| c.register_entry_type::<CorpusItem>())?,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn all(&self) -> anyhow::Result<Vec<CorpusItem>> {
|
||||
let mut items = self.cache.get_all::<CorpusItem>()?;
|
||||
items.sort_by(|a, b| a.id.cmp(&b.id));
|
||||
Ok(items)
|
||||
}
|
||||
|
||||
pub fn get(&self, key: &str) -> anyhow::Result<Option<CorpusItem>> {
|
||||
self.cache.get::<CorpusItem>(key)
|
||||
}
|
||||
|
||||
/// The next free key for `slug`; withdrawn items keep their numbers.
|
||||
pub fn next_key(&self, slug: &str) -> anyhow::Result<String> {
|
||||
let highest = self
|
||||
.all()?
|
||||
.iter()
|
||||
.filter(|i| key_slug(&i.id) == Some(slug))
|
||||
.filter_map(|i| i.id.rsplit_once(':')?.1.parse::<u32>().ok())
|
||||
.max()
|
||||
.unwrap_or(0);
|
||||
Ok(item_key(slug, highest + 1))
|
||||
}
|
||||
|
||||
/// The item `key` names, or `UnknownCorpusItem` when the key is not
|
||||
/// `item:<slug>:<n>` or holds nothing.
|
||||
fn resolve(&self, key: &str) -> Result<CorpusItem, StoreError> {
|
||||
let unknown = || StoreRefusal::UnknownCorpusItem {
|
||||
key: key.to_string(),
|
||||
};
|
||||
if key_slug(key).is_none() {
|
||||
return Err(unknown().into());
|
||||
}
|
||||
self.get(key)?.ok_or_else(|| unknown().into())
|
||||
}
|
||||
|
||||
/// S1: an item key holds its text for good. S5: a correction stays in its
|
||||
/// predecessor's group, so it cannot move across the split.
|
||||
pub fn add(&mut self, item: CorpusItem) -> Result<AddOutcome, StoreError> {
|
||||
if key_slug(&item.id).is_none() {
|
||||
return Err(malformed(&item.id, "item key is not item:<slug>:<n>"));
|
||||
}
|
||||
if item.withdrawn {
|
||||
return Err(malformed(
|
||||
&item.id,
|
||||
"an item is admitted live, then withdrawn",
|
||||
));
|
||||
}
|
||||
if let Some(existing) = self.get(&item.id)? {
|
||||
if existing.text == item.text {
|
||||
return Ok(AddOutcome::Unchanged);
|
||||
}
|
||||
return Err(StoreRefusal::S1ItemTextImmutable { key: item.id }.into());
|
||||
}
|
||||
if let Some(old) = &item.supersedes {
|
||||
let Some(old) = self.get(old)? else {
|
||||
return Err(malformed(
|
||||
&item.id,
|
||||
"supersedes an item that does not exist",
|
||||
));
|
||||
};
|
||||
if old.group != item.group {
|
||||
return Err(StoreRefusal::S5SupersedeGroup {
|
||||
key: item.id,
|
||||
expected: old.group,
|
||||
got: item.group,
|
||||
}
|
||||
.into());
|
||||
}
|
||||
}
|
||||
self.cache.put(item.id.clone(), &item)?;
|
||||
Ok(AddOutcome::Added)
|
||||
}
|
||||
|
||||
/// Marks an item withdrawn; its text stays.
|
||||
pub fn withdraw(&mut self, key: &str) -> Result<(), StoreError> {
|
||||
let Some(mut item) = self.get(key)? else {
|
||||
return Err(malformed(key, "no such item"));
|
||||
};
|
||||
if !item.withdrawn {
|
||||
item.withdrawn = true;
|
||||
self.cache.put(key, &item)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
pub struct InventoryStore {
|
||||
cache: CultCache,
|
||||
}
|
||||
|
||||
impl InventoryStore {
|
||||
pub fn open(path: &Path) -> anyhow::Result<Self> {
|
||||
let cache = open_cache(path, |c| {
|
||||
c.register_entry_type::<InventoryEntry>()?;
|
||||
c.register_entry_type::<InventoryRevision>()
|
||||
})?;
|
||||
Ok(Self { cache })
|
||||
}
|
||||
|
||||
pub fn inventory(&self) -> anyhow::Result<Inventory> {
|
||||
Ok(Inventory::new(
|
||||
self.cache.get_all::<InventoryEntry>()?,
|
||||
self.cache.get_all::<InventoryRevision>()?,
|
||||
))
|
||||
}
|
||||
|
||||
/// Admits revision `rev` and the entries it adds, whole or not at all.
|
||||
/// S2: latest+1 with parent latest. S3: nothing follows a frozen revision.
|
||||
/// S4: an added entry's introduced_rev is this revision.
|
||||
/// An added entry's gloss is 1 to 120 bytes without a control character.
|
||||
/// No door judges text against the corpus: leak is owned by the loop's
|
||||
/// context builder and the scorer.
|
||||
pub fn admit_revision(
|
||||
&mut self,
|
||||
rev: InventoryRevision,
|
||||
entries: Vec<InventoryEntry>,
|
||||
) -> Result<(), StoreError> {
|
||||
let inv = self.inventory()?;
|
||||
let latest = inv.latest();
|
||||
if let Some(l) = latest.filter(|l| l.frozen) {
|
||||
return Err(StoreRefusal::S3Frozen { frozen: l.n }.into());
|
||||
}
|
||||
let (expected, parent) = latest.map_or((1, None), |l| (l.n + 1, Some(l.n)));
|
||||
if rev.n != expected || rev.parent != parent {
|
||||
return Err(StoreRefusal::S2RevisionChain {
|
||||
expected,
|
||||
got: rev.n,
|
||||
}
|
||||
.into());
|
||||
}
|
||||
let mut touched: BTreeSet<InvId> = BTreeSet::new();
|
||||
let mut staged = Vec::new();
|
||||
for change in &rev.changes {
|
||||
match change {
|
||||
Change::Add(id) => {
|
||||
let Some(entry) = entries.iter().find(|e| &e.id == id) else {
|
||||
return Err(malformed(&id.to_string(), "added id has no entry"));
|
||||
};
|
||||
if entry.introduced_rev != rev.n {
|
||||
let (id, introduced, revision) = (*id, entry.introduced_rev, rev.n);
|
||||
return Err(StoreRefusal::S4IntroducedRev {
|
||||
id,
|
||||
introduced,
|
||||
revision,
|
||||
}
|
||||
.into());
|
||||
}
|
||||
let key = id.to_string();
|
||||
if !entry.well_formed() || entry.retired_rev.is_some() {
|
||||
return Err(malformed(&key, "entry shape does not match its family"));
|
||||
}
|
||||
if inv.entry(id).is_some() || !touched.insert(*id) {
|
||||
return Err(malformed(&key, "id is already in the inventory"));
|
||||
}
|
||||
check_entry_text(entry)?;
|
||||
staged.push(self.cache.prepare_entry_named(key, entry)?.0);
|
||||
}
|
||||
Change::Retire(id) => {
|
||||
let key = id.to_string();
|
||||
let Some(current) = inv.entry(id).filter(|e| e.retired_rev.is_none()) else {
|
||||
return Err(malformed(
|
||||
&key,
|
||||
"retire target is missing or already retired",
|
||||
));
|
||||
};
|
||||
if !touched.insert(*id) {
|
||||
return Err(malformed(&key, "id changes twice in one revision"));
|
||||
}
|
||||
let retired = InventoryEntry {
|
||||
retired_rev: Some(rev.n),
|
||||
..current.clone()
|
||||
};
|
||||
staged.push(self.cache.prepare_entry_named(key, &retired)?.0);
|
||||
}
|
||||
}
|
||||
}
|
||||
if let Some(stray) = entries.iter().find(|e| !touched.contains(&e.id)) {
|
||||
return Err(malformed(
|
||||
&stray.id.to_string(),
|
||||
"entry is not in the revision's changes",
|
||||
));
|
||||
}
|
||||
staged.push(self.cache.prepare_entry_named(revision_key(rev.n), &rev)?.0);
|
||||
self.cache.put_prepared_batch(staged)?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn check_entry_text(entry: &InventoryEntry) -> Result<(), StoreError> {
|
||||
if !(1..=120).contains(&entry.gloss.len()) || entry.gloss.chars().any(char::is_control) {
|
||||
return Err(malformed(
|
||||
&entry.id.to_string(),
|
||||
"gloss is 1 to 120 bytes with no control character",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub struct PacketStore {
|
||||
cache: CultCache,
|
||||
}
|
||||
|
||||
impl PacketStore {
|
||||
pub fn open(path: &Path) -> anyhow::Result<Self> {
|
||||
Ok(Self {
|
||||
cache: open_cache(path, |c| c.register_entry_type::<Packet>())?,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn all(&self) -> anyhow::Result<Vec<Packet>> {
|
||||
let mut packets = self.cache.get_all::<Packet>()?;
|
||||
packets.sort_by(|a, b| a.id.cmp(&b.id));
|
||||
Ok(packets)
|
||||
}
|
||||
|
||||
pub fn get(&self, key: &str) -> anyhow::Result<Option<Packet>> {
|
||||
self.cache.get::<Packet>(key)
|
||||
}
|
||||
|
||||
/// The only door: a packet exists only if `validate` passes against the
|
||||
/// inventory the store holds at the packet's revision, its `corpus_item`
|
||||
/// (when set) resolves, and a key holds its packet for good.
|
||||
pub fn admit(
|
||||
&mut self,
|
||||
packet: Packet,
|
||||
inventory: &InventoryStore,
|
||||
corpus: &CorpusStore,
|
||||
) -> Result<(), StoreError> {
|
||||
validate(&packet, &inventory.inventory()?).map_err(StoreError::Invalid)?;
|
||||
if let Some(item_key) = &packet.provenance.corpus_item {
|
||||
corpus.resolve(item_key)?;
|
||||
}
|
||||
let key = packet.id.to_string();
|
||||
if let Some(existing) = self.get(&key)? {
|
||||
if existing == packet {
|
||||
return Ok(());
|
||||
}
|
||||
return Err(StoreRefusal::PacketImmutable { key }.into());
|
||||
}
|
||||
self.cache.put(key, &packet)?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
475
crates/weksa/src/validate.rs
Normal file
475
crates/weksa/src/validate.rs
Normal file
|
|
@ -0,0 +1,475 @@
|
|||
//! The validator owns whether a packet exists. Every path that admits or lowers a
|
||||
//! packet calls `validate`, and its Ok value, [`LoweringInput`], is the only
|
||||
//! thing a lowerer sees. The rules are V1..V8 and V10..V12: the shape of an id
|
||||
//! or a grade is its type's parse, not a rule, so V9 is retired and not
|
||||
//! renumbered.
|
||||
//!
|
||||
//! `validate` owns form and capacity and makes no leak decision. A packet's
|
||||
//! free text is form-bounded and every list is bounded, so a packet has no
|
||||
//! unbounded channel. What bounds each list (V11, V12): a feature list holds at
|
||||
//! most one value per feature id and `extensions` at most one per extension id,
|
||||
//! so each is at most the inventory's ids of that kind; `participants`,
|
||||
//! `audience` and `preserve` hold each node id at most once and every id names
|
||||
//! a node (V4), so each is at most `Packet::MAX_NODES`; a predication's `roles`
|
||||
//! hold each role id at most once, so at most the inventory's roles; and the
|
||||
//! nodes themselves are at most `Packet::MAX_NODES` (V12). Whether a score may
|
||||
//! rise because corpus text reached a packet is owned by the golf loop's
|
||||
//! context builder and the scorer (ruling `leak-owned-by-flow-and-scorer`),
|
||||
//! never by a text rule here, in `admit_revision` or in `PacketStore::admit`.
|
||||
|
||||
use std::collections::BTreeSet;
|
||||
use std::fmt;
|
||||
|
||||
use crate::inventory::{Family, InvId, Inventory, InventoryEntry, Slot, ValueKind};
|
||||
use crate::packet::{
|
||||
Constraints, Context, Discourse, Extension, FeatValue, FeatureValue, NodeId, Packet,
|
||||
PacketKind, Predication, Referent, Relation, SCHEMA_VERSION,
|
||||
};
|
||||
|
||||
/// A broken rule, named.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub enum Refusal {
|
||||
/// V1: the packet's schema_version is not the one these types define.
|
||||
V1SchemaVersion { found: u32 },
|
||||
/// V2: no referents, predications or relations.
|
||||
V2PayloadRequired,
|
||||
/// V3: a node id repeats across referents and predications.
|
||||
V3DuplicateNode { id: NodeId },
|
||||
/// V4: a reference names no node.
|
||||
V4DanglingRef { id: NodeId, at: &'static str },
|
||||
/// V5: an inventory id does not resolve at the packet's revision.
|
||||
V5Unresolved { id: InvId },
|
||||
/// V6: an inventory id is of the wrong family for its slot.
|
||||
V6WrongFamily {
|
||||
id: InvId,
|
||||
expected: Family,
|
||||
at: &'static str,
|
||||
},
|
||||
/// V7: a feature or extension value does not fit its value kind.
|
||||
V7ValueKind { what: String },
|
||||
/// V8: a feature is not declared for the slot it sits on.
|
||||
V8NotApplicable { feature: InvId, slot: Slot },
|
||||
/// V10: the packet's inventory revision has no record.
|
||||
V10UnknownRevision { rev: u32 },
|
||||
/// V11: a feature id repeats in one slot's list (`slot` is `None` for an
|
||||
/// extension id repeated in the packet).
|
||||
V11DuplicateFeature { feature: InvId, slot: Option<Slot> },
|
||||
/// V11: a node id repeats in `participants`, `audience` or `preserve`
|
||||
/// (`at` names the list).
|
||||
V11DuplicateNodeRef { id: NodeId, at: &'static str },
|
||||
/// V11: a role id repeats in one predication's roles.
|
||||
V11DuplicateRole { role: InvId, predication: NodeId },
|
||||
/// V12: more referents, predications and relations than `Packet::MAX_NODES`.
|
||||
V12TooManyNodes { count: usize, max: usize },
|
||||
}
|
||||
|
||||
impl Refusal {
|
||||
pub fn rule(&self) -> &'static str {
|
||||
match self {
|
||||
Refusal::V1SchemaVersion { .. } => "V1",
|
||||
Refusal::V2PayloadRequired => "V2",
|
||||
Refusal::V3DuplicateNode { .. } => "V3",
|
||||
Refusal::V4DanglingRef { .. } => "V4",
|
||||
Refusal::V5Unresolved { .. } => "V5",
|
||||
Refusal::V6WrongFamily { .. } => "V6",
|
||||
Refusal::V7ValueKind { .. } => "V7",
|
||||
Refusal::V8NotApplicable { .. } => "V8",
|
||||
Refusal::V10UnknownRevision { .. } => "V10",
|
||||
Refusal::V11DuplicateFeature { .. }
|
||||
| Refusal::V11DuplicateNodeRef { .. }
|
||||
| Refusal::V11DuplicateRole { .. } => "V11",
|
||||
Refusal::V12TooManyNodes { .. } => "V12",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for Refusal {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
write!(f, "{}: {self:?}", self.rule())
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether `packet` may exist under `inventory`. All broken rules are returned;
|
||||
/// a packet that passes yields the lowering view, which only this function builds.
|
||||
pub fn validate(packet: &Packet, inventory: &Inventory) -> Result<LoweringInput, Vec<Refusal>> {
|
||||
let mut out = Vec::new();
|
||||
if packet.schema_version != SCHEMA_VERSION {
|
||||
out.push(Refusal::V1SchemaVersion {
|
||||
found: packet.schema_version,
|
||||
});
|
||||
}
|
||||
if packet.referents.is_empty() && packet.predications.is_empty() && packet.relations.is_empty()
|
||||
{
|
||||
out.push(Refusal::V2PayloadRequired);
|
||||
}
|
||||
let count = packet.referents.len() + packet.predications.len() + packet.relations.len();
|
||||
if count > Packet::MAX_NODES {
|
||||
out.push(Refusal::V12TooManyNodes {
|
||||
count,
|
||||
max: Packet::MAX_NODES,
|
||||
});
|
||||
}
|
||||
let nodes = check_nodes(packet, &mut out);
|
||||
check_refs(packet, &nodes, &mut out);
|
||||
check_repeats(packet, &mut out);
|
||||
if inventory.has_revision(packet.inventory_rev) {
|
||||
Inventoried {
|
||||
inv: inventory,
|
||||
rev: packet.inventory_rev,
|
||||
out: &mut out,
|
||||
}
|
||||
.check(packet);
|
||||
} else {
|
||||
out.push(Refusal::V10UnknownRevision {
|
||||
rev: packet.inventory_rev,
|
||||
});
|
||||
}
|
||||
if out.is_empty() {
|
||||
Ok(LoweringInput::of(packet))
|
||||
} else {
|
||||
Err(out)
|
||||
}
|
||||
}
|
||||
|
||||
fn check_nodes(packet: &Packet, out: &mut Vec<Refusal>) -> BTreeSet<NodeId> {
|
||||
let mut nodes = BTreeSet::new();
|
||||
let ids = packet
|
||||
.referents
|
||||
.iter()
|
||||
.map(|r| &r.id)
|
||||
.chain(packet.predications.iter().map(|p| &p.id));
|
||||
for id in ids {
|
||||
if !nodes.insert(id.clone()) {
|
||||
out.push(Refusal::V3DuplicateNode { id: id.clone() });
|
||||
}
|
||||
}
|
||||
nodes
|
||||
}
|
||||
|
||||
fn check_refs(packet: &Packet, nodes: &BTreeSet<NodeId>, out: &mut Vec<Refusal>) {
|
||||
let mut need = |id: &NodeId, at: &'static str| {
|
||||
if !nodes.contains(id) {
|
||||
out.push(Refusal::V4DanglingRef { id: id.clone(), at });
|
||||
}
|
||||
};
|
||||
for p in &packet.predications {
|
||||
for r in &p.roles {
|
||||
need(&r.filler, "role filler");
|
||||
}
|
||||
}
|
||||
for r in &packet.relations {
|
||||
need(&r.from, "relation from");
|
||||
need(&r.to, "relation to");
|
||||
}
|
||||
if let Some(a) = &packet.discourse.addressee {
|
||||
need(a, "addressee");
|
||||
}
|
||||
for (list, at) in node_lists(packet) {
|
||||
for id in list {
|
||||
need(id, at);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn node_lists(packet: &Packet) -> [(&Vec<NodeId>, &'static str); 3] {
|
||||
[
|
||||
(&packet.context.participants, "participant"),
|
||||
(&packet.context.audience, "audience"),
|
||||
(&packet.constraints.preserve, "preserve"),
|
||||
]
|
||||
}
|
||||
|
||||
/// V11 for the lists that hold nodes and roles: each node id once per node list,
|
||||
/// each role id once per predication.
|
||||
fn check_repeats(packet: &Packet, out: &mut Vec<Refusal>) {
|
||||
for (list, at) in node_lists(packet) {
|
||||
let mut seen = BTreeSet::new();
|
||||
for id in list {
|
||||
if !seen.insert(id) {
|
||||
out.push(Refusal::V11DuplicateNodeRef { id: id.clone(), at });
|
||||
}
|
||||
}
|
||||
}
|
||||
for p in &packet.predications {
|
||||
let mut seen = BTreeSet::new();
|
||||
for r in &p.roles {
|
||||
if !seen.insert(r.role) {
|
||||
out.push(Refusal::V11DuplicateRole {
|
||||
role: r.role,
|
||||
predication: p.id.clone(),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The inventory-dependent rules V5..V8 and V11, read at the packet's revision.
|
||||
struct Inventoried<'a> {
|
||||
inv: &'a Inventory,
|
||||
rev: u32,
|
||||
out: &'a mut Vec<Refusal>,
|
||||
}
|
||||
|
||||
impl<'a> Inventoried<'a> {
|
||||
fn check(&mut self, p: &Packet) {
|
||||
let Packet {
|
||||
context,
|
||||
discourse,
|
||||
referents,
|
||||
predications,
|
||||
relations,
|
||||
extensions,
|
||||
..
|
||||
} = p;
|
||||
let Context {
|
||||
setting, medium, ..
|
||||
} = context;
|
||||
for id in setting.iter().chain(medium) {
|
||||
self.slot(id, Family::Concept, "context concept");
|
||||
}
|
||||
let Discourse {
|
||||
speech_act,
|
||||
features,
|
||||
..
|
||||
} = discourse;
|
||||
if let Some(id) = speech_act {
|
||||
self.slot(id, Family::SpeechAct, "speech act");
|
||||
}
|
||||
self.features(features, Slot::Discourse);
|
||||
for Referent {
|
||||
kind,
|
||||
concept,
|
||||
features,
|
||||
..
|
||||
} in referents
|
||||
{
|
||||
self.slot(kind, Family::ReferentKind, "referent kind");
|
||||
self.slot(concept, Family::Concept, "referent concept");
|
||||
self.features(features, Slot::Referent);
|
||||
}
|
||||
for Predication {
|
||||
concept,
|
||||
frame,
|
||||
roles,
|
||||
features,
|
||||
..
|
||||
} in predications
|
||||
{
|
||||
self.slot(concept, Family::Concept, "predication concept");
|
||||
self.slot(frame, Family::Frame, "frame");
|
||||
for r in roles {
|
||||
self.slot(&r.role, Family::Role, "role");
|
||||
}
|
||||
self.features(features, Slot::Predication);
|
||||
}
|
||||
for Relation {
|
||||
relation, features, ..
|
||||
} in relations
|
||||
{
|
||||
self.slot(relation, Family::Relation, "relation");
|
||||
self.features(features, Slot::Relation);
|
||||
}
|
||||
let mut seen = BTreeSet::new();
|
||||
for Extension { key, value } in extensions {
|
||||
if !seen.insert(*key) {
|
||||
self.out.push(Refusal::V11DuplicateFeature {
|
||||
feature: *key,
|
||||
slot: None,
|
||||
});
|
||||
}
|
||||
if let Some(entry) = self.slot(key, Family::Extension, "extension")
|
||||
&& !fits(entry.value_kind.as_ref(), value)
|
||||
{
|
||||
self.out.push(Refusal::V7ValueKind {
|
||||
what: key.to_string(),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// V5, then V6 when it resolved.
|
||||
fn slot(&mut self, id: &InvId, want: Family, at: &'static str) -> Option<&'a InventoryEntry> {
|
||||
let entry = self.resolve(id)?;
|
||||
if entry.family() != want {
|
||||
self.out.push(Refusal::V6WrongFamily {
|
||||
id: *id,
|
||||
expected: want,
|
||||
at,
|
||||
});
|
||||
return None;
|
||||
}
|
||||
Some(entry)
|
||||
}
|
||||
|
||||
fn resolve(&mut self, id: &InvId) -> Option<&'a InventoryEntry> {
|
||||
let found = self.inv.resolve(id, self.rev);
|
||||
if found.is_none() {
|
||||
self.out.push(Refusal::V5Unresolved { id: *id });
|
||||
}
|
||||
found
|
||||
}
|
||||
|
||||
fn features(&mut self, features: &[FeatureValue], slot: Slot) {
|
||||
let mut seen = BTreeSet::new();
|
||||
for f in features {
|
||||
if !seen.insert(f.feature) {
|
||||
self.out.push(Refusal::V11DuplicateFeature {
|
||||
feature: f.feature,
|
||||
slot: Some(slot),
|
||||
});
|
||||
}
|
||||
let Some(entry) = self.slot(&f.feature, Family::Feature, "feature") else {
|
||||
continue;
|
||||
};
|
||||
if !entry.applies_to.contains(&slot) {
|
||||
self.out.push(Refusal::V8NotApplicable {
|
||||
feature: f.feature,
|
||||
slot,
|
||||
});
|
||||
}
|
||||
if !fits(entry.value_kind.as_ref(), &f.value) {
|
||||
self.out.push(Refusal::V7ValueKind {
|
||||
what: f.feature.to_string(),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn fits(kind: Option<&ValueKind>, value: &FeatValue) -> bool {
|
||||
match (kind, value) {
|
||||
(Some(ValueKind::Grade), FeatValue::Grade(_)) => true,
|
||||
(Some(ValueKind::Options(labels)), FeatValue::Option(l)) => labels.contains(l),
|
||||
(Some(ValueKind::Flag), FeatValue::Flag(_)) => true,
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
/// What the lowerer sees: a validated packet's meaning, with no packet id, no
|
||||
/// provenance (so no corpus item key, encoder or source) and no trace. Its
|
||||
/// fields are private and `validate` is the only constructor, so a lowerer
|
||||
/// cannot be handed an unvalidated packet or the key to an answer text.
|
||||
///
|
||||
/// Each field has a doctest that reads it from outside the crate. A read
|
||||
/// compiles exactly when the field is `pub`, so the doctest (which must fail to
|
||||
/// compile) fails as soon as any one field is made public. A struct literal
|
||||
/// cannot pin this: it fails to compile while any other field stays private,
|
||||
/// and stable rustdoc does not check the error code.
|
||||
///
|
||||
/// ```compile_fail
|
||||
/// fn read(l: weksa::LoweringInput) { let _ = l.inventory_rev; }
|
||||
/// ```
|
||||
///
|
||||
/// ```compile_fail
|
||||
/// fn read(l: weksa::LoweringInput) { let _ = l.kind; }
|
||||
/// ```
|
||||
///
|
||||
/// ```compile_fail
|
||||
/// fn read(l: weksa::LoweringInput) { let _ = l.context; }
|
||||
/// ```
|
||||
///
|
||||
/// ```compile_fail
|
||||
/// fn read(l: weksa::LoweringInput) { let _ = l.discourse; }
|
||||
/// ```
|
||||
///
|
||||
/// ```compile_fail
|
||||
/// fn read(l: weksa::LoweringInput) { let _ = l.referents; }
|
||||
/// ```
|
||||
///
|
||||
/// ```compile_fail
|
||||
/// fn read(l: weksa::LoweringInput) { let _ = l.predications; }
|
||||
/// ```
|
||||
///
|
||||
/// ```compile_fail
|
||||
/// fn read(l: weksa::LoweringInput) { let _ = l.relations; }
|
||||
/// ```
|
||||
///
|
||||
/// ```compile_fail
|
||||
/// fn read(l: weksa::LoweringInput) { let _ = l.constraints; }
|
||||
/// ```
|
||||
///
|
||||
/// ```compile_fail
|
||||
/// fn read(l: weksa::LoweringInput) { let _ = l.extensions; }
|
||||
/// ```
|
||||
///
|
||||
/// The constructor is private: a call from outside the crate compiles only if
|
||||
/// `LoweringInput::of` is `pub`. The first block is the same code without the
|
||||
/// call, so the second fails for that call and nothing else.
|
||||
///
|
||||
/// ```
|
||||
/// fn build(p: &weksa::packet::Packet) { let _ = p; }
|
||||
/// ```
|
||||
///
|
||||
/// ```compile_fail
|
||||
/// fn build(p: &weksa::packet::Packet) { let _ = weksa::LoweringInput::of(p); }
|
||||
/// ```
|
||||
///
|
||||
/// A lowering view is never read from bytes: `Deserialize` would let a caller
|
||||
/// build one without `validate`. The first block proves the bound resolves for
|
||||
/// a type that does implement it.
|
||||
///
|
||||
/// ```
|
||||
/// fn de<T: serde::de::DeserializeOwned>() {}
|
||||
/// de::<weksa::packet::Packet>();
|
||||
/// ```
|
||||
///
|
||||
/// ```compile_fail
|
||||
/// fn de<T: serde::de::DeserializeOwned>() {}
|
||||
/// de::<weksa::LoweringInput>();
|
||||
/// ```
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct LoweringInput {
|
||||
inventory_rev: u32,
|
||||
kind: PacketKind,
|
||||
context: Context,
|
||||
discourse: Discourse,
|
||||
referents: Vec<Referent>,
|
||||
predications: Vec<Predication>,
|
||||
relations: Vec<Relation>,
|
||||
constraints: Constraints,
|
||||
extensions: Vec<Extension>,
|
||||
}
|
||||
|
||||
impl LoweringInput {
|
||||
fn of(p: &Packet) -> Self {
|
||||
Self {
|
||||
inventory_rev: p.inventory_rev,
|
||||
kind: p.kind,
|
||||
context: p.context.clone(),
|
||||
discourse: p.discourse.clone(),
|
||||
referents: p.referents.clone(),
|
||||
predications: p.predications.clone(),
|
||||
relations: p.relations.clone(),
|
||||
constraints: p.constraints.clone(),
|
||||
extensions: p.extensions.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn inventory_rev(&self) -> u32 {
|
||||
self.inventory_rev
|
||||
}
|
||||
pub fn kind(&self) -> PacketKind {
|
||||
self.kind
|
||||
}
|
||||
pub fn context(&self) -> &Context {
|
||||
&self.context
|
||||
}
|
||||
pub fn discourse(&self) -> &Discourse {
|
||||
&self.discourse
|
||||
}
|
||||
pub fn referents(&self) -> &[Referent] {
|
||||
&self.referents
|
||||
}
|
||||
pub fn predications(&self) -> &[Predication] {
|
||||
&self.predications
|
||||
}
|
||||
pub fn relations(&self) -> &[Relation] {
|
||||
&self.relations
|
||||
}
|
||||
pub fn constraints(&self) -> &Constraints {
|
||||
&self.constraints
|
||||
}
|
||||
pub fn extensions(&self) -> &[Extension] {
|
||||
&self.extensions
|
||||
}
|
||||
}
|
||||
227
crates/weksa/tests/cli.rs
Normal file
227
crates/weksa/tests/cli.rs
Normal file
|
|
@ -0,0 +1,227 @@
|
|||
mod common;
|
||||
use std::process::{Command, Output};
|
||||
use weksa::{InventoryStore, PacketStore};
|
||||
|
||||
fn weksa(dir: &std::path::Path, args: &[&str]) -> Output {
|
||||
Command::new(env!("CARGO_BIN_EXE_weksa"))
|
||||
.current_dir(dir)
|
||||
.args(["--corpus", "./c.cc", "--inventory", "./i.cc"])
|
||||
.args(args)
|
||||
.output()
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn out(o: &Output) -> String {
|
||||
String::from_utf8_lossy(&o.stdout).into_owned() + &String::from_utf8_lossy(&o.stderr)
|
||||
}
|
||||
|
||||
const ADD: [&str; 14] = [
|
||||
"corpus",
|
||||
"add",
|
||||
"--slug",
|
||||
"nibu",
|
||||
"--text",
|
||||
"Mind the hatch.",
|
||||
"--speaker",
|
||||
"nibu",
|
||||
"--group",
|
||||
"scene-1",
|
||||
"--admitted-on",
|
||||
"2026-10-29",
|
||||
"--source",
|
||||
"cast-authored",
|
||||
];
|
||||
|
||||
#[test]
|
||||
fn seed_add_dump_split_withdraw() {
|
||||
let d = tempfile::tempdir().unwrap();
|
||||
let seeded = weksa(d.path(), &["inventory", "seed"]);
|
||||
assert!(seeded.status.success(), "{}", out(&seeded));
|
||||
assert!(out(&seeded).contains("seeded rev:1"));
|
||||
let again = weksa(d.path(), &["inventory", "seed"]);
|
||||
assert!(
|
||||
!again.status.success(),
|
||||
"a second seed is refused: {}",
|
||||
out(&again)
|
||||
);
|
||||
let dump = out(&weksa(d.path(), &["inventory", "dump"]));
|
||||
assert!(
|
||||
dump.contains("rev:1") && dump.contains("speech_act:5") && dump.contains("warn"),
|
||||
"{dump}"
|
||||
);
|
||||
|
||||
// A cast-authored item needs its persona and fixture.
|
||||
let missing = weksa(d.path(), &ADD);
|
||||
assert!(
|
||||
!missing.status.success() && out(&missing).contains("--persona"),
|
||||
"{}",
|
||||
out(&missing)
|
||||
);
|
||||
let mut add = ADD.to_vec();
|
||||
add.extend(["--persona", "nibu", "--fixture", "f.yaml"]);
|
||||
let added = weksa(d.path(), &add);
|
||||
assert!(
|
||||
added.status.success() && out(&added).contains("Added item:nibu:1"),
|
||||
"{}",
|
||||
out(&added)
|
||||
);
|
||||
let second = weksa(d.path(), &add);
|
||||
assert!(
|
||||
out(&second).contains("Added item:nibu:2"),
|
||||
"{}",
|
||||
out(&second)
|
||||
);
|
||||
|
||||
let dump = out(&weksa(d.path(), &["corpus", "dump"]));
|
||||
assert!(
|
||||
dump.contains("item:nibu:1")
|
||||
&& dump.contains("Mind the hatch.")
|
||||
&& dump.contains("CastAuthored"),
|
||||
"{dump}"
|
||||
);
|
||||
let split = out(&weksa(d.path(), &["split", "item:nibu:1"]));
|
||||
assert!(
|
||||
["Train", "Validation", "Test"]
|
||||
.iter()
|
||||
.any(|s| split.trim() == *s),
|
||||
"{split}"
|
||||
);
|
||||
assert!(!weksa(d.path(), &["split", "item:nibu:9"]).status.success());
|
||||
|
||||
let withdrawn = weksa(d.path(), &["corpus", "withdraw", "item:nibu:1"]);
|
||||
assert!(withdrawn.status.success());
|
||||
assert!(out(&weksa(d.path(), &["corpus", "dump"])).contains("withdrawn"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn model_written_and_public_domain_sources_name_their_provenance() {
|
||||
let d = tempfile::tempdir().unwrap();
|
||||
let base = [
|
||||
"corpus",
|
||||
"add",
|
||||
"--slug",
|
||||
"x",
|
||||
"--text",
|
||||
"t",
|
||||
"--speaker",
|
||||
"s",
|
||||
"--group",
|
||||
"g",
|
||||
"--admitted-on",
|
||||
"2026-10-29",
|
||||
];
|
||||
let run = |extra: &[&str]| {
|
||||
let mut args = base.to_vec();
|
||||
args.extend_from_slice(extra);
|
||||
weksa(d.path(), &args)
|
||||
};
|
||||
let no_family = run(&["--source", "model-written"]);
|
||||
assert!(
|
||||
!no_family.status.success() && out(&no_family).contains("--model-family"),
|
||||
"{}",
|
||||
out(&no_family)
|
||||
);
|
||||
let model = run(&["--source", "model-written", "--model-family", "m"]);
|
||||
assert!(model.status.success(), "{}", out(&model));
|
||||
let no_work = run(&["--source", "public-domain", "--author", "a"]);
|
||||
assert!(
|
||||
!no_work.status.success() && out(&no_work).contains("--work"),
|
||||
"{}",
|
||||
out(&no_work)
|
||||
);
|
||||
let pd = run(&[
|
||||
"--source",
|
||||
"public-domain",
|
||||
"--work",
|
||||
"w",
|
||||
"--author",
|
||||
"a",
|
||||
"--edition-url",
|
||||
"https://example.org/w",
|
||||
]);
|
||||
assert!(pd.status.success(), "{}", out(&pd));
|
||||
let bad_turn = run(&[
|
||||
"--source",
|
||||
"operator-written",
|
||||
"--prior-turn",
|
||||
"no equals sign",
|
||||
]);
|
||||
assert!(
|
||||
!bad_turn.status.success() && out(&bad_turn).contains("speaker=text"),
|
||||
"{}",
|
||||
out(&bad_turn)
|
||||
);
|
||||
let turn = run(&[
|
||||
"--source",
|
||||
"operator-written",
|
||||
"--prior-turn",
|
||||
"player=hello",
|
||||
]);
|
||||
assert!(turn.status.success(), "{}", out(&turn));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_reports_each_packet() {
|
||||
let d = tempfile::tempdir().unwrap();
|
||||
let w = common::world();
|
||||
let full = w.dir.path().join("i.cc");
|
||||
let packets = d.path().join("p.cc");
|
||||
let packet = common::good();
|
||||
PacketStore::open(&packets)
|
||||
.unwrap()
|
||||
.admit(packet.clone(), &w.inventory, &w.corpus)
|
||||
.unwrap();
|
||||
|
||||
let validate = |inventory: &std::path::Path, extra: &[&str]| {
|
||||
Command::new(env!("CARGO_BIN_EXE_weksa"))
|
||||
.arg("--inventory")
|
||||
.arg(inventory)
|
||||
.arg("validate")
|
||||
.arg(&packets)
|
||||
.args(extra)
|
||||
.output()
|
||||
.unwrap()
|
||||
};
|
||||
let ok = validate(&full, &[]);
|
||||
assert!(
|
||||
ok.status.success() && out(&ok).contains(&format!("ok {}", packet.id)),
|
||||
"{}",
|
||||
out(&ok)
|
||||
);
|
||||
assert!(validate(&full, &[packet.id.as_str()]).status.success());
|
||||
assert!(!validate(&full, &["packet:missing"]).status.success());
|
||||
// Against an inventory that only reaches revision 1 the same packet is refused, by rule.
|
||||
let short = d.path().join("short.cc");
|
||||
let (rev1, entries1) = weksa::inventory::seed();
|
||||
InventoryStore::open(&short)
|
||||
.unwrap()
|
||||
.admit_revision(rev1, entries1)
|
||||
.unwrap();
|
||||
let refused = validate(&short, &[]);
|
||||
assert!(
|
||||
!refused.status.success() && out(&refused).contains("V10"),
|
||||
"{}",
|
||||
out(&refused)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn schema_writes_the_published_files() {
|
||||
let d = tempfile::tempdir().unwrap();
|
||||
let o = weksa(d.path(), &["schema", "--out", "s"]);
|
||||
assert!(o.status.success(), "{}", out(&o));
|
||||
let mut names: Vec<String> = std::fs::read_dir(d.path().join("s"))
|
||||
.unwrap()
|
||||
.map(|e| e.unwrap().file_name().into_string().unwrap())
|
||||
.collect();
|
||||
names.sort();
|
||||
assert_eq!(
|
||||
names,
|
||||
[
|
||||
"weksa.corpus_item.schema.json",
|
||||
"weksa.interlingua_packet.schema.json",
|
||||
"weksa.inventory_entry.schema.json",
|
||||
"weksa.inventory_revision.schema.json"
|
||||
]
|
||||
);
|
||||
}
|
||||
274
crates/weksa/tests/common/mod.rs
Normal file
274
crates/weksa/tests/common/mod.rs
Normal file
|
|
@ -0,0 +1,274 @@
|
|||
#![allow(dead_code)]
|
||||
|
||||
use std::sync::OnceLock;
|
||||
|
||||
use tempfile::TempDir;
|
||||
use weksa::inventory::{Change, seed};
|
||||
use weksa::packet::*;
|
||||
use weksa::{
|
||||
CorpusItem, CorpusStore, InvId, Inventory, InventoryEntry, InventoryRevision, InventoryStore,
|
||||
OptionLabel, Slot, Source, SplitScheme, ValueKind,
|
||||
};
|
||||
|
||||
pub const PERSON: &str = "referent_kind:1";
|
||||
pub const AGENT: &str = "role:1";
|
||||
pub const PATIENT: &str = "role:2";
|
||||
pub const WARN: &str = "speech_act:5";
|
||||
pub const POLARITY: &str = "feature:1";
|
||||
pub const MODALITY: &str = "feature:4";
|
||||
pub const HEDGE: &str = "feature:6";
|
||||
pub const IRONY: &str = "feature:11";
|
||||
/// Introduced at rev 2.
|
||||
pub const CONCEPT_A: &str = "concept:1";
|
||||
pub const CONCEPT_B: &str = "concept:2";
|
||||
/// Introduced at rev 2, retired at rev 3.
|
||||
pub const CONCEPT_RETIRED: &str = "concept:3";
|
||||
/// Introduced at rev 4.
|
||||
pub const CONCEPT_LATE: &str = "concept:4";
|
||||
pub const FRAME: &str = "frame:1";
|
||||
/// An extension entry of value kind Grade, introduced at rev 2.
|
||||
pub const EXT: &str = "extension:1";
|
||||
/// A Grade feature introduced at rev 2 that sits in every slot.
|
||||
pub const FEAT_ALL: &str = "feature:50";
|
||||
|
||||
pub const PACKET_ID: &str = "packet:0123456789abcdef";
|
||||
pub const ENCODER_ID: &str = "encoder:fedcba9876543210";
|
||||
|
||||
pub fn iv(s: &str) -> InvId {
|
||||
s.parse().unwrap()
|
||||
}
|
||||
|
||||
pub fn nd(s: &str) -> NodeId {
|
||||
NodeId::parse(s).unwrap()
|
||||
}
|
||||
|
||||
pub fn lab(s: &str) -> OptionLabel {
|
||||
OptionLabel::parse(s).unwrap()
|
||||
}
|
||||
|
||||
pub fn grade(v: u16) -> Grade {
|
||||
Grade::new(v).unwrap()
|
||||
}
|
||||
|
||||
pub fn group_in_bucket(want: u64) -> String {
|
||||
(0..100_000)
|
||||
.map(|i| format!("g{i}"))
|
||||
.find(|g| SplitScheme::V1.bucket(g) == want)
|
||||
.unwrap_or_else(|| panic!("no group lands in bucket {want}"))
|
||||
}
|
||||
|
||||
pub fn cast() -> Source {
|
||||
Source::CastAuthored {
|
||||
persona: "nibu".into(),
|
||||
fixture: "f.yaml".into(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn model() -> Source {
|
||||
Source::ModelWritten {
|
||||
model_family: "m".into(),
|
||||
prompt_digest: None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn item_in(key: &str, text: &str, group: &str, source: Source) -> CorpusItem {
|
||||
CorpusItem {
|
||||
id: key.into(),
|
||||
text: text.into(),
|
||||
language: "en-US".into(),
|
||||
speaker: "nibu".into(),
|
||||
group: group.into(),
|
||||
prior_turns: vec![],
|
||||
source,
|
||||
admitted_on: "2026-10-29".into(),
|
||||
supersedes: None,
|
||||
withdrawn: false,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn entry(id: &str, introduced: u32) -> InventoryEntry {
|
||||
InventoryEntry {
|
||||
id: iv(id),
|
||||
gloss: id.into(),
|
||||
value_kind: None,
|
||||
applies_to: vec![],
|
||||
introduced_rev: introduced,
|
||||
retired_rev: None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn feature(id: &str, kind: ValueKind, slots: &[Slot]) -> InventoryEntry {
|
||||
InventoryEntry {
|
||||
value_kind: Some(kind),
|
||||
applies_to: slots.to_vec(),
|
||||
..entry(id, 1)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn rev_with(changes: Vec<Change>, n: u32, parent: Option<u32>) -> InventoryRevision {
|
||||
InventoryRevision {
|
||||
n,
|
||||
parent,
|
||||
changes,
|
||||
frozen: false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Admits `entries` as revision `n` and returns nothing; the stores are the only door.
|
||||
pub fn admit(store: &mut InventoryStore, n: u32, entries: Vec<InventoryEntry>) {
|
||||
let changes = entries.iter().map(|e| Change::Add(e.id)).collect();
|
||||
let parent = n.checked_sub(1).filter(|p| *p > 0);
|
||||
store
|
||||
.admit_revision(rev_with(changes, n, parent), entries)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
/// Stores a test can admit through: a corpus holding `item:nibu:1` in a Train
|
||||
/// group, and an inventory of revisions 1 (the seed), 2 (concepts A, B,
|
||||
/// RETIRED, a frame, a Grade extension), 3 (RETIRED retired) and 4 (LATE).
|
||||
pub struct World {
|
||||
pub dir: TempDir,
|
||||
pub inventory: InventoryStore,
|
||||
pub corpus: CorpusStore,
|
||||
}
|
||||
|
||||
impl World {
|
||||
pub fn inv(&self) -> Inventory {
|
||||
self.inventory.inventory().unwrap()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn world() -> World {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let mut corpus = CorpusStore::open(&dir.path().join("c.cc")).unwrap();
|
||||
let train = group_in_bucket(10);
|
||||
corpus
|
||||
.add(item_in(
|
||||
"item:nibu:1",
|
||||
"Nibu says mind the hatch.",
|
||||
&train,
|
||||
cast(),
|
||||
))
|
||||
.unwrap();
|
||||
let mut inventory = InventoryStore::open(&dir.path().join("i.cc")).unwrap();
|
||||
let (rev1, entries) = seed();
|
||||
inventory.admit_revision(rev1, entries).unwrap();
|
||||
let mut ext = entry(EXT, 2);
|
||||
ext.value_kind = Some(ValueKind::Grade);
|
||||
admit(
|
||||
&mut inventory,
|
||||
2,
|
||||
vec![
|
||||
entry(CONCEPT_A, 2),
|
||||
entry(CONCEPT_B, 2),
|
||||
entry(CONCEPT_RETIRED, 2),
|
||||
entry(FRAME, 2),
|
||||
ext,
|
||||
InventoryEntry {
|
||||
introduced_rev: 2,
|
||||
..feature(
|
||||
FEAT_ALL,
|
||||
ValueKind::Grade,
|
||||
&[
|
||||
Slot::Discourse,
|
||||
Slot::Referent,
|
||||
Slot::Predication,
|
||||
Slot::Relation,
|
||||
],
|
||||
)
|
||||
},
|
||||
],
|
||||
);
|
||||
inventory
|
||||
.admit_revision(
|
||||
rev_with(vec![Change::Retire(iv(CONCEPT_RETIRED))], 3, Some(2)),
|
||||
vec![],
|
||||
)
|
||||
.unwrap();
|
||||
admit(&mut inventory, 4, vec![entry(CONCEPT_LATE, 4)]);
|
||||
World {
|
||||
dir,
|
||||
inventory,
|
||||
corpus,
|
||||
}
|
||||
}
|
||||
|
||||
/// The world's inventory, built once.
|
||||
pub fn inventory() -> Inventory {
|
||||
static INV: OnceLock<Inventory> = OnceLock::new();
|
||||
INV.get_or_init(|| world().inv()).clone()
|
||||
}
|
||||
|
||||
pub fn good() -> Packet {
|
||||
Packet {
|
||||
id: PacketId::parse(PACKET_ID).unwrap(),
|
||||
schema_version: SCHEMA_VERSION,
|
||||
inventory_rev: 2,
|
||||
kind: PacketKind::DialogueLine,
|
||||
provenance: Provenance {
|
||||
source: ProvenanceSource::Authored,
|
||||
corpus_item: Some("item:nibu:1".into()),
|
||||
encoder: EncoderId::parse(ENCODER_ID).unwrap(),
|
||||
confidence: grade(900),
|
||||
},
|
||||
context: Context {
|
||||
setting: Some(iv(CONCEPT_A)),
|
||||
medium: Some(iv(CONCEPT_B)),
|
||||
participants: vec![nd("n1")],
|
||||
audience: vec![nd("n1")],
|
||||
},
|
||||
discourse: Discourse {
|
||||
speech_act: Some(iv(WARN)),
|
||||
addressee: Some(nd("n1")),
|
||||
features: vec![FeatureValue {
|
||||
feature: iv(HEDGE),
|
||||
value: FeatValue::Grade(grade(300)),
|
||||
}],
|
||||
},
|
||||
referents: vec![Referent {
|
||||
id: nd("n1"),
|
||||
kind: iv(PERSON),
|
||||
concept: iv(CONCEPT_A),
|
||||
name: Some(ProperName::parse("Nibu").unwrap()),
|
||||
features: vec![],
|
||||
}],
|
||||
predications: vec![Predication {
|
||||
id: nd("n2"),
|
||||
concept: iv(CONCEPT_B),
|
||||
frame: iv(FRAME),
|
||||
roles: vec![
|
||||
RoleFill {
|
||||
role: iv(AGENT),
|
||||
filler: nd("n1"),
|
||||
},
|
||||
RoleFill {
|
||||
role: iv(PATIENT),
|
||||
filler: nd("n1"),
|
||||
},
|
||||
],
|
||||
features: vec![FeatureValue {
|
||||
feature: iv(MODALITY),
|
||||
value: FeatValue::Option(lab("potential")),
|
||||
}],
|
||||
}],
|
||||
relations: vec![Relation {
|
||||
relation: iv("relation:8"),
|
||||
from: nd("n2"),
|
||||
to: nd("n1"),
|
||||
features: vec![],
|
||||
}],
|
||||
constraints: Constraints {
|
||||
preserve: vec![nd("n1")],
|
||||
deterministic: true,
|
||||
max_latency_ms: Some(50),
|
||||
},
|
||||
extensions: vec![Extension {
|
||||
key: iv(EXT),
|
||||
value: FeatValue::Grade(grade(3)),
|
||||
}],
|
||||
trace: Trace {
|
||||
notes: vec!["a note".into()],
|
||||
rejected_readings: vec![],
|
||||
},
|
||||
}
|
||||
}
|
||||
248
crates/weksa/tests/forms.rs
Normal file
248
crates/weksa/tests/forms.rs
Normal file
|
|
@ -0,0 +1,248 @@
|
|||
mod common;
|
||||
use common::*;
|
||||
use weksa::packet::{EncoderId, Grade, NodeId, PacketId, ProperName, Referent};
|
||||
use weksa::{Family, InvId, OptionLabel, OptionSet};
|
||||
|
||||
/// Soul's probe: a whole corpus line, and the snake_case form that would
|
||||
/// pass for an identifier.
|
||||
const LINE: &str = "Take your hand off that control unless you want your nervous system to become my least useful circuit diagram.";
|
||||
const SNAKE: &str = "take_your_hand_off_that_control_unless_you_want_your_nervous_system_to_become_my_least_useful_circuit_diagram";
|
||||
|
||||
#[test]
|
||||
fn forms_refuse_surface_text() {
|
||||
for text in [LINE, SNAKE, "", " "] {
|
||||
assert!(NodeId::parse(text).is_err(), "NodeId {text:?}");
|
||||
assert!(PacketId::parse(text).is_err(), "PacketId {text:?}");
|
||||
assert!(EncoderId::parse(text).is_err(), "EncoderId {text:?}");
|
||||
assert!(InvId::parse(text).is_err(), "InvId {text:?}");
|
||||
assert!(OptionLabel::parse(text).is_err(), "OptionLabel {text:?}");
|
||||
}
|
||||
// Soul's extension key `nibu.<snake line>` is not an inventory id either.
|
||||
assert!(InvId::parse(&format!("extension:{SNAKE}")).is_err());
|
||||
assert!(InvId::parse(&format!("nibu.{SNAKE}")).is_err());
|
||||
// NodeId is n<k>, k canonical 1..=9999.
|
||||
for ok in ["n1", "n10", "n9999"] {
|
||||
assert_eq!(NodeId::parse(ok).unwrap().as_str(), ok);
|
||||
}
|
||||
for bad in [
|
||||
"n0", "n01", "n+1", "n-1", "n10000", "n", "1", "N1", "n1 ", "r1", "n1x",
|
||||
] {
|
||||
assert!(NodeId::parse(bad).is_err(), "{bad:?}");
|
||||
}
|
||||
// PacketId and EncoderId are a prefix and exactly 16 lowercase hex digits.
|
||||
assert!(PacketId::parse(PACKET_ID).is_ok());
|
||||
assert!(EncoderId::parse(ENCODER_ID).is_ok());
|
||||
for bad in [
|
||||
"packet:0123456789abcde",
|
||||
"packet:0123456789abcdef0",
|
||||
"packet:0123456789ABCDEF",
|
||||
"packet:0123456789abcdeg",
|
||||
"encoder:0123456789abcdef",
|
||||
"0123456789abcdef",
|
||||
"packet:item:nibu:1:r2:abc",
|
||||
] {
|
||||
assert!(PacketId::parse(bad).is_err(), "{bad:?}");
|
||||
}
|
||||
assert!(EncoderId::parse("packet:0123456789abcdef").is_err());
|
||||
assert!(EncoderId::parse("encoder:0123456789abcdeg").is_err());
|
||||
assert!(EncoderId::parse("encoder:0123456789ABCDEF").is_err());
|
||||
assert!(EncoderId::parse("encoder:0123456789abcde").is_err());
|
||||
assert!(EncoderId::parse("encoder:0123456789abcdef0").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_refusal_names_the_form_and_the_value_and_a_form_reads_back_as_written() {
|
||||
let e = NodeId::parse("n0").unwrap_err();
|
||||
assert_eq!(e.form, "NodeId");
|
||||
assert_eq!(e.got, "n0");
|
||||
assert_eq!(e.to_string(), "\"n0\" is not a NodeId");
|
||||
assert_eq!(
|
||||
Grade::new(1001).unwrap_err().to_string(),
|
||||
"\"1001\" is not a Grade"
|
||||
);
|
||||
assert_eq!(EncoderId::parse(ENCODER_ID).unwrap().as_str(), ENCODER_ID);
|
||||
assert_eq!(PacketId::parse(PACKET_ID).unwrap().as_str(), PACKET_ID);
|
||||
assert_eq!(
|
||||
EncoderId::parse(ENCODER_ID).unwrap().to_string(),
|
||||
ENCODER_ID
|
||||
);
|
||||
assert_eq!(lab("potential").as_str(), "potential");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_malformed_value_does_not_deserialize() {
|
||||
// A node id that is the snake line fails where a store would read it.
|
||||
let mut referent = serde_json::to_value(good().referents[0].clone()).unwrap();
|
||||
assert!(serde_json::from_value::<Referent>(referent.clone()).is_ok());
|
||||
referent["id"] = SNAKE.into();
|
||||
assert!(serde_json::from_value::<Referent>(referent.clone()).is_err());
|
||||
referent["id"] = "n1".into();
|
||||
referent["name"] = LINE.into();
|
||||
assert!(serde_json::from_value::<Referent>(referent.clone()).is_err());
|
||||
referent["name"] = "Nibu".into();
|
||||
referent["concept"] = "concept:01".into();
|
||||
assert!(serde_json::from_value::<Referent>(referent).is_err());
|
||||
assert!(serde_json::from_str::<Grade>("1001").is_err());
|
||||
assert_eq!(serde_json::from_str::<Grade>("1000").unwrap().get(), 1000);
|
||||
assert!(serde_json::from_str::<OptionSet>("[]").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn proper_name_bounds() {
|
||||
for ok in [
|
||||
"Nibu",
|
||||
"Brokkr Ivaldi",
|
||||
"O'Neil",
|
||||
"O\u{2019}Neil",
|
||||
"Jean-Luc",
|
||||
"A B C D",
|
||||
] {
|
||||
assert_eq!(ProperName::parse(ok).unwrap().as_str(), ok);
|
||||
}
|
||||
// A caseless script is a name; so is a capital beyond ASCII.
|
||||
assert!(ProperName::parse("\u{5c71}\u{7530}").is_ok());
|
||||
assert!(ProperName::parse("\u{c9}owyn").is_ok());
|
||||
for bad in [
|
||||
LINE,
|
||||
"A B C D E",
|
||||
"Nibu.",
|
||||
"Nibu, Ivaldi",
|
||||
"Nibu Ivaldi",
|
||||
" Nibu",
|
||||
"Nibu ",
|
||||
"",
|
||||
"Nibu2",
|
||||
"'Nibu",
|
||||
"-Nibu",
|
||||
"Nibu-",
|
||||
"Nibu'",
|
||||
"Nibu_Ivaldi",
|
||||
] {
|
||||
assert!(ProperName::parse(bad).is_err(), "{bad:?}");
|
||||
}
|
||||
// 24 chars a token, 4 tokens and 64 bytes are the bounds.
|
||||
let token24 = format!("A{}", "b".repeat(23));
|
||||
assert!(ProperName::parse(&token24).is_ok());
|
||||
assert!(ProperName::parse(&format!("{token24}b")).is_err());
|
||||
let sized = |lens: [usize; 4]| {
|
||||
lens.iter()
|
||||
.map(|n| format!("A{}", "b".repeat(n - 1)))
|
||||
.collect::<Vec<_>>()
|
||||
.join(" ")
|
||||
};
|
||||
let at_limit = sized([15, 15, 15, 16]);
|
||||
assert_eq!(at_limit.len(), 64);
|
||||
assert!(ProperName::parse(&at_limit).is_ok());
|
||||
let over = sized([15, 15, 15, 17]);
|
||||
assert_eq!(over.len(), 65);
|
||||
assert!(ProperName::parse(&over).is_err());
|
||||
// The byte bound counts bytes, not characters.
|
||||
let wide = format!(
|
||||
"{0} {0} {0}",
|
||||
"\u{e9}".repeat(11).replace('\u{e9}', "\u{c9}")
|
||||
);
|
||||
assert!(wide.chars().count() < 64 && wide.len() > 64);
|
||||
assert!(ProperName::parse(&wide).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn inv_id_canonical() {
|
||||
for bad in [
|
||||
"concept:01",
|
||||
"concept:+1",
|
||||
"concept:0",
|
||||
"concept:-1",
|
||||
"concept:",
|
||||
"concept:1x",
|
||||
"concept: 1",
|
||||
"nofamily:1",
|
||||
"Concept:1",
|
||||
"concept",
|
||||
"concept:99999999999",
|
||||
"concept:1:2",
|
||||
] {
|
||||
assert!(InvId::parse(bad).is_err(), "{bad:?}");
|
||||
}
|
||||
let id = InvId::parse("concept:1").unwrap();
|
||||
assert_eq!(id.family(), Family::Concept);
|
||||
assert_eq!(id.to_string(), "concept:1");
|
||||
for f in Family::ALL {
|
||||
let id = InvId::parse(&format!("{}:12", f.prefix())).unwrap();
|
||||
assert_eq!(id.family(), f);
|
||||
assert_eq!(id.to_string(), format!("{}:12", f.prefix()));
|
||||
}
|
||||
assert_eq!(iv("extension:1").family(), Family::Extension);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn options_bounded() {
|
||||
let labels = |n: usize| (0..n).map(|i| lab(&format!("l{i}"))).collect::<Vec<_>>();
|
||||
assert!(OptionSet::new(labels(0)).is_err());
|
||||
assert!(OptionSet::new(labels(1)).is_ok());
|
||||
assert!(OptionSet::new(labels(16)).is_ok());
|
||||
assert!(OptionSet::new(labels(17)).is_err());
|
||||
assert!(OptionSet::new(vec![lab("a"), lab("a")]).is_err());
|
||||
assert!(OptionSet::new(vec![lab("a"), lab("b"), lab("a")]).is_err());
|
||||
// Soul's probe: the snake line is not a label; labels are short snake words.
|
||||
assert!(OptionLabel::parse(SNAKE).is_err());
|
||||
assert!(OptionLabel::parse(&"a".repeat(24)).is_ok());
|
||||
assert!(OptionLabel::parse(&"a".repeat(25)).is_err());
|
||||
for bad in ["A", "1a", "_a", "a-b", "a b", "a.b", "\u{e9}"] {
|
||||
assert!(OptionLabel::parse(bad).is_err(), "{bad:?}");
|
||||
}
|
||||
for ok in ["a", "a1", "a_b", "potential"] {
|
||||
assert!(OptionLabel::parse(ok).is_ok(), "{ok:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn grade_bound_is_the_type() {
|
||||
assert!(Grade::new(1001).is_err());
|
||||
assert_eq!(Grade::new(1000).unwrap().get(), 1000);
|
||||
assert_eq!(Grade::new(0).unwrap().get(), 0);
|
||||
// The published schema carries the same bound.
|
||||
let (_, json) = weksa::schema::published()
|
||||
.into_iter()
|
||||
.find(|(n, _)| n == "weksa.interlingua_packet.schema.json")
|
||||
.unwrap();
|
||||
let schema: serde_json::Value = serde_json::from_str(&json).unwrap();
|
||||
let grade = &schema["$defs"]["Grade"];
|
||||
assert_eq!(grade["maximum"], 1000, "{grade}");
|
||||
assert_eq!(grade["minimum"], 0, "{grade}");
|
||||
assert_eq!(schema["$defs"]["NodeId"]["pattern"], "^n[1-9][0-9]{0,3}$");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn proper_name_bound_in_chars() {
|
||||
// The token bound counts characters: 24 two-byte letters are 48 bytes and parse.
|
||||
let cyrillic = |n: usize| "\u{411}".repeat(n);
|
||||
assert_eq!(cyrillic(24).len(), 48);
|
||||
assert!(ProperName::parse(&cyrillic(24)).is_ok());
|
||||
assert!(ProperName::parse(&cyrillic(25)).is_err());
|
||||
// The 64-byte bound still counts bytes: two 48-byte tokens are 97 bytes.
|
||||
assert!(ProperName::parse(&format!("{0} {0}", cyrillic(24))).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn proper_name_is_form_only() {
|
||||
// A name is bounded text: no case rule, so lowercase, hyphenated and
|
||||
// caseless-script tokens parse.
|
||||
for ok in [
|
||||
"nibu",
|
||||
"Nibu ivaldi",
|
||||
"take-your-hand-off-that",
|
||||
"Take-your-hand-off-that",
|
||||
"\u{3131}-control-unless-you",
|
||||
"\u{5c71}\u{7530}",
|
||||
] {
|
||||
assert_eq!(ProperName::parse(ok).unwrap().as_str(), ok);
|
||||
}
|
||||
// The bounds still hold: 5 tokens, 65 bytes, a sentence's punctuation.
|
||||
assert!(ProperName::parse("a b c d e").is_err());
|
||||
assert!(ProperName::parse(&"a".repeat(25)).is_err());
|
||||
let token = |n: usize| "a".repeat(n);
|
||||
let over = [token(15), token(15), token(15), token(17)].join(" ");
|
||||
assert_eq!(over.len(), 65);
|
||||
assert!(ProperName::parse(&over).is_err());
|
||||
assert!(ProperName::parse("take your hand, now.").is_err());
|
||||
}
|
||||
150
crates/weksa/tests/split_schema.rs
Normal file
150
crates/weksa/tests/split_schema.rs
Normal file
|
|
@ -0,0 +1,150 @@
|
|||
use weksa::corpus::{CorpusItem, Source};
|
||||
use weksa::split::fnv1a64;
|
||||
use weksa::{Split, SplitScheme};
|
||||
|
||||
fn item(group: &str, source: Source) -> CorpusItem {
|
||||
CorpusItem {
|
||||
id: "item:t:1".into(),
|
||||
text: "t".into(),
|
||||
language: "en-US".into(),
|
||||
speaker: "s".into(),
|
||||
group: group.into(),
|
||||
prior_turns: vec![],
|
||||
source,
|
||||
admitted_on: "2026-10-29".into(),
|
||||
supersedes: None,
|
||||
withdrawn: false,
|
||||
}
|
||||
}
|
||||
|
||||
fn cast() -> Source {
|
||||
Source::CastAuthored {
|
||||
persona: "p".into(),
|
||||
fixture: "f".into(),
|
||||
}
|
||||
}
|
||||
|
||||
fn model() -> Source {
|
||||
Source::ModelWritten {
|
||||
model_family: "m".into(),
|
||||
prompt_digest: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn group_in_bucket(want: u64) -> String {
|
||||
(0..100_000)
|
||||
.map(|i| format!("g{i}"))
|
||||
.find(|g| SplitScheme::V1.bucket(g) == want)
|
||||
.unwrap_or_else(|| panic!("no group lands in bucket {want}"))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fnv1a64_matches_the_reference_vectors() {
|
||||
assert_eq!(fnv1a64(b""), 0xcbf2_9ce4_8422_2325);
|
||||
assert_eq!(fnv1a64(b"a"), 0xaf63_dc4c_8601_ec8c);
|
||||
assert_eq!(fnv1a64(b"foobar"), 0x8594_4171_f739_67e8);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn p1_split_deterministic() {
|
||||
let a = SplitScheme::V1.split(&item("scene-9", cast()));
|
||||
for _ in 0..3 {
|
||||
assert_eq!(SplitScheme::V1.split(&item("scene-9", cast())), a);
|
||||
}
|
||||
// The split reads the group and nothing else: speaker, text and id do not move it.
|
||||
let mut other = item("scene-9", cast());
|
||||
other.speaker = "someone".into();
|
||||
other.text = "different".into();
|
||||
other.id = "item:other:7".into();
|
||||
assert_eq!(SplitScheme::V1.split(&other), a);
|
||||
// The bucket is the pinned hash of the prefixed group.
|
||||
assert_eq!(
|
||||
SplitScheme::V1.bucket("x"),
|
||||
fnv1a64(b"weksa-split-v1:x") % 100
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn split_boundaries() {
|
||||
let s = SplitScheme::V1;
|
||||
assert_eq!(s.of_bucket(0), Split::Train);
|
||||
assert_eq!(s.of_bucket(39), Split::Train);
|
||||
assert_eq!(s.of_bucket(40), Split::Validation);
|
||||
assert_eq!(s.of_bucket(79), Split::Validation);
|
||||
assert_eq!(s.of_bucket(80), Split::Test);
|
||||
assert_eq!(s.of_bucket(99), Split::Test);
|
||||
for (bucket, want) in [
|
||||
(39, Split::Train),
|
||||
(40, Split::Validation),
|
||||
(79, Split::Validation),
|
||||
(80, Split::Test),
|
||||
] {
|
||||
assert_eq!(
|
||||
s.split(&item(&group_in_bucket(bucket), cast())),
|
||||
Some(want),
|
||||
"bucket {bucket}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn p2_model_written_in_a_test_group_is_excluded() {
|
||||
let test_group = group_in_bucket(95);
|
||||
assert_eq!(
|
||||
SplitScheme::V1.split(&item(&test_group, cast())),
|
||||
Some(Split::Test)
|
||||
);
|
||||
// A model line of the test meaning is in no split; it is never moved to train.
|
||||
assert_eq!(SplitScheme::V1.split(&item(&test_group, model())), None);
|
||||
// Model-written items in the other splits keep them.
|
||||
assert_eq!(
|
||||
SplitScheme::V1.split(&item(&group_in_bucket(10), model())),
|
||||
Some(Split::Train)
|
||||
);
|
||||
assert_eq!(
|
||||
SplitScheme::V1.split(&item(&group_in_bucket(60), model())),
|
||||
Some(Split::Validation)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn p3_split_ratio() {
|
||||
let mut counts = [0usize; 3];
|
||||
for i in 0..10_000 {
|
||||
match SplitScheme::V1.split(&item(&format!("scene-{i}"), cast())) {
|
||||
Some(Split::Train) => counts[0] += 1,
|
||||
Some(Split::Validation) => counts[1] += 1,
|
||||
Some(Split::Test) => counts[2] += 1,
|
||||
None => panic!("a cast item is always in a split"),
|
||||
}
|
||||
}
|
||||
let near = |got: usize, want_percent: usize| got.abs_diff(want_percent * 100) <= 200;
|
||||
assert!(
|
||||
near(counts[0], 40) && near(counts[1], 40) && near(counts[2], 20),
|
||||
"{counts:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn published_schemas_match() {
|
||||
let dir = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../schemas/cultnet");
|
||||
let published = weksa::schema::published();
|
||||
let mut names: Vec<String> = published.iter().map(|(n, _)| n.clone()).collect();
|
||||
names.sort();
|
||||
let mut on_disk: Vec<String> = std::fs::read_dir(&dir)
|
||||
.unwrap()
|
||||
.map(|e| e.unwrap().file_name().into_string().unwrap())
|
||||
.filter(|n| n.ends_with(".schema.json"))
|
||||
.collect();
|
||||
on_disk.sort();
|
||||
assert_eq!(
|
||||
on_disk, names,
|
||||
"schemas/cultnet holds exactly the generated files"
|
||||
);
|
||||
for (name, json) in published {
|
||||
let held = std::fs::read_to_string(dir.join(&name))
|
||||
.unwrap()
|
||||
.replace("\r\n", "\n");
|
||||
assert_eq!(held, json, "{name} drifted; run `weksa schema`");
|
||||
}
|
||||
}
|
||||
636
crates/weksa/tests/stores.rs
Normal file
636
crates/weksa/tests/stores.rs
Normal file
|
|
@ -0,0 +1,636 @@
|
|||
mod common;
|
||||
use common::*;
|
||||
use tempfile::TempDir;
|
||||
use weksa::corpus::{CorpusItem, Source, Turn};
|
||||
use weksa::inventory::{Change, seed};
|
||||
use weksa::{
|
||||
AddOutcome, CorpusStore, Family, InventoryEntry, InventoryRevision, InventoryStore,
|
||||
PacketStore, Refusal, Slot, SplitScheme, StoreError, StoreRefusal, ValueKind,
|
||||
};
|
||||
|
||||
fn item(key: &str, text: &str) -> CorpusItem {
|
||||
CorpusItem {
|
||||
prior_turns: vec![Turn {
|
||||
speaker: "player".into(),
|
||||
text: "hello".into(),
|
||||
}],
|
||||
..item_in(key, text, "scene-1", cast())
|
||||
}
|
||||
}
|
||||
|
||||
fn refusal(r: Result<impl std::fmt::Debug, StoreError>) -> StoreRefusal {
|
||||
match r {
|
||||
Err(StoreError::Refused(r)) => r,
|
||||
other => panic!("expected a store refusal, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
fn dir() -> TempDir {
|
||||
tempfile::tempdir().unwrap()
|
||||
}
|
||||
|
||||
fn empty_corpus(d: &TempDir) -> CorpusStore {
|
||||
CorpusStore::open(&d.path().join("empty-corpus.cc")).unwrap()
|
||||
}
|
||||
|
||||
fn malformed(key: &str, why: &'static str) -> StoreRefusal {
|
||||
StoreRefusal::Malformed {
|
||||
key: key.into(),
|
||||
why,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn s1_item_text_immutable() {
|
||||
let d = dir();
|
||||
let path = d.path().join("c.cc");
|
||||
let mut store = CorpusStore::open(&path).unwrap();
|
||||
assert_eq!(
|
||||
store.add(item("item:nibu:1", "Mind the hatch.")).unwrap(),
|
||||
AddOutcome::Added
|
||||
);
|
||||
// The same text is idempotent, even from a reopened store.
|
||||
let mut store = CorpusStore::open(&path).unwrap();
|
||||
assert_eq!(
|
||||
store.add(item("item:nibu:1", "Mind the hatch.")).unwrap(),
|
||||
AddOutcome::Unchanged
|
||||
);
|
||||
assert_eq!(
|
||||
refusal(store.add(item("item:nibu:1", "Mind the hatch!"))),
|
||||
StoreRefusal::S1ItemTextImmutable {
|
||||
key: "item:nibu:1".into()
|
||||
}
|
||||
);
|
||||
assert_eq!(
|
||||
store.get("item:nibu:1").unwrap().unwrap().text,
|
||||
"Mind the hatch."
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn corpus_admission_rules() {
|
||||
let d = dir();
|
||||
let mut store = CorpusStore::open(&d.path().join("c.cc")).unwrap();
|
||||
assert_eq!(
|
||||
refusal(store.add(item("nibu:1", "x"))),
|
||||
malformed("nibu:1", "item key is not item:<slug>:<n>")
|
||||
);
|
||||
assert_eq!(
|
||||
refusal(store.add(item("item:nibu:x", "x"))),
|
||||
malformed("item:nibu:x", "item key is not item:<slug>:<n>")
|
||||
);
|
||||
let mut born_withdrawn = item("item:nibu:1", "x");
|
||||
born_withdrawn.withdrawn = true;
|
||||
assert_eq!(
|
||||
refusal(store.add(born_withdrawn)),
|
||||
malformed("item:nibu:1", "an item is admitted live, then withdrawn")
|
||||
);
|
||||
let mut orphan = item("item:nibu:1", "x");
|
||||
orphan.supersedes = Some("item:nibu:9".into());
|
||||
assert_eq!(
|
||||
refusal(store.add(orphan)),
|
||||
malformed("item:nibu:1", "supersedes an item that does not exist")
|
||||
);
|
||||
// Nothing was written by the refusals.
|
||||
assert!(store.all().unwrap().is_empty());
|
||||
// A correction is a new item that supersedes the old one.
|
||||
store.add(item("item:nibu:1", "old")).unwrap();
|
||||
let mut fix = item("item:nibu:2", "new");
|
||||
fix.supersedes = Some("item:nibu:1".into());
|
||||
assert_eq!(store.add(fix).unwrap(), AddOutcome::Added);
|
||||
assert_eq!(
|
||||
refusal(store.withdraw("item:nibu:7")),
|
||||
malformed("item:nibu:7", "no such item")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn s5_supersede_keeps_the_group() {
|
||||
let d = dir();
|
||||
let mut store = CorpusStore::open(&d.path().join("c.cc")).unwrap();
|
||||
store.add(item("item:nibu:1", "old")).unwrap();
|
||||
let mut elsewhere = item("item:nibu:2", "new");
|
||||
elsewhere.supersedes = Some("item:nibu:1".into());
|
||||
elsewhere.group = "scene-2".into();
|
||||
assert_eq!(
|
||||
refusal(store.add(elsewhere)),
|
||||
StoreRefusal::S5SupersedeGroup {
|
||||
key: "item:nibu:2".into(),
|
||||
expected: "scene-1".into(),
|
||||
got: "scene-2".into()
|
||||
}
|
||||
);
|
||||
assert!(store.get("item:nibu:2").unwrap().is_none());
|
||||
let mut same = item("item:nibu:2", "new");
|
||||
same.supersedes = Some("item:nibu:1".into());
|
||||
assert_eq!(store.add(same).unwrap(), AddOutcome::Added);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn withdrawing_keeps_the_text_and_the_number() {
|
||||
let d = dir();
|
||||
let path = d.path().join("c.cc");
|
||||
let mut store = CorpusStore::open(&path).unwrap();
|
||||
store.add(item("item:nibu:1", "keep me")).unwrap();
|
||||
assert_eq!(store.next_key("nibu").unwrap(), "item:nibu:2");
|
||||
assert_eq!(store.next_key("other").unwrap(), "item:other:1");
|
||||
store.withdraw("item:nibu:1").unwrap();
|
||||
store.withdraw("item:nibu:1").unwrap();
|
||||
let store = CorpusStore::open(&path).unwrap();
|
||||
let got = store.get("item:nibu:1").unwrap().unwrap();
|
||||
assert!(got.withdrawn);
|
||||
assert_eq!(got.text, "keep me");
|
||||
assert_eq!(store.next_key("nibu").unwrap(), "item:nibu:2");
|
||||
}
|
||||
|
||||
fn entries_for(rev: u32, ids: &[&str]) -> (InventoryRevision, Vec<InventoryEntry>) {
|
||||
let entries = ids.iter().map(|id| entry(id, rev)).collect();
|
||||
let changes = ids.iter().map(|id| Change::Add(iv(id))).collect();
|
||||
(
|
||||
rev_with(changes, rev, if rev == 1 { None } else { Some(rev - 1) }),
|
||||
entries,
|
||||
)
|
||||
}
|
||||
|
||||
fn seeded(path: &std::path::Path) -> InventoryStore {
|
||||
let mut store = InventoryStore::open(path).unwrap();
|
||||
let (rev, entries) = seed();
|
||||
store.admit_revision(rev, entries).unwrap();
|
||||
store
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn s2_revision_chain() {
|
||||
let d = dir();
|
||||
let mut store = InventoryStore::open(&d.path().join("i.cc")).unwrap();
|
||||
// The first revision is 1 with no parent.
|
||||
let (rev, entries) = entries_for(2, &["concept:1"]);
|
||||
assert_eq!(
|
||||
refusal(store.admit_revision(rev, entries)),
|
||||
StoreRefusal::S2RevisionChain {
|
||||
expected: 1,
|
||||
got: 2
|
||||
}
|
||||
);
|
||||
let (mut rev, entries) = entries_for(1, &["concept:1"]);
|
||||
rev.parent = Some(0);
|
||||
assert_eq!(
|
||||
refusal(store.admit_revision(rev, entries)),
|
||||
StoreRefusal::S2RevisionChain {
|
||||
expected: 1,
|
||||
got: 1
|
||||
}
|
||||
);
|
||||
let (rev, entries) = entries_for(1, &["concept:1"]);
|
||||
store.admit_revision(rev, entries).unwrap();
|
||||
// Replaying n, skipping n, or naming the wrong parent is refused.
|
||||
let (rev, entries) = entries_for(1, &["concept:2"]);
|
||||
assert_eq!(
|
||||
refusal(store.admit_revision(rev, entries)),
|
||||
StoreRefusal::S2RevisionChain {
|
||||
expected: 2,
|
||||
got: 1
|
||||
}
|
||||
);
|
||||
let (rev, entries) = entries_for(3, &["concept:2"]);
|
||||
assert_eq!(
|
||||
refusal(store.admit_revision(rev, entries)),
|
||||
StoreRefusal::S2RevisionChain {
|
||||
expected: 2,
|
||||
got: 3
|
||||
}
|
||||
);
|
||||
let (mut rev, entries) = entries_for(2, &["concept:2"]);
|
||||
rev.parent = None;
|
||||
assert_eq!(
|
||||
refusal(store.admit_revision(rev, entries)),
|
||||
StoreRefusal::S2RevisionChain {
|
||||
expected: 2,
|
||||
got: 2
|
||||
}
|
||||
);
|
||||
let (rev, entries) = entries_for(2, &["concept:2"]);
|
||||
store.admit_revision(rev, entries).unwrap();
|
||||
assert_eq!(store.inventory().unwrap().latest().unwrap().n, 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn s3_frozen_is_final() {
|
||||
let d = dir();
|
||||
let mut store = InventoryStore::open(&d.path().join("i.cc")).unwrap();
|
||||
let (mut rev, entries) = entries_for(1, &["concept:1"]);
|
||||
rev.frozen = true;
|
||||
store.admit_revision(rev, entries).unwrap();
|
||||
let (rev, entries) = entries_for(2, &["concept:2"]);
|
||||
assert_eq!(
|
||||
refusal(store.admit_revision(rev, entries)),
|
||||
StoreRefusal::S3Frozen { frozen: 1 }
|
||||
);
|
||||
assert_eq!(store.inventory().unwrap().latest().unwrap().n, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn s4_entry_introduced_by_its_revision() {
|
||||
let d = dir();
|
||||
let mut store = InventoryStore::open(&d.path().join("i.cc")).unwrap();
|
||||
let (rev, mut entries) = entries_for(1, &["concept:1"]);
|
||||
entries[0].introduced_rev = 2;
|
||||
assert_eq!(
|
||||
refusal(store.admit_revision(rev, entries)),
|
||||
StoreRefusal::S4IntroducedRev {
|
||||
id: iv("concept:1"),
|
||||
introduced: 2,
|
||||
revision: 1
|
||||
}
|
||||
);
|
||||
assert!(store.inventory().unwrap().latest().is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn revision_change_rules() {
|
||||
let d = dir();
|
||||
let mut store = seeded(&d.path().join("i.cc"));
|
||||
// An added id with no entry.
|
||||
let rev = rev_with(vec![Change::Add(iv("concept:1"))], 2, Some(1));
|
||||
assert_eq!(
|
||||
refusal(store.admit_revision(rev, vec![])),
|
||||
malformed("concept:1", "added id has no entry")
|
||||
);
|
||||
// An entry the changes do not name.
|
||||
let (rev, mut entries) = entries_for(2, &["concept:1"]);
|
||||
entries.push(entry("concept:2", 2));
|
||||
assert_eq!(
|
||||
refusal(store.admit_revision(rev, entries)),
|
||||
malformed("concept:2", "entry is not in the revision's changes")
|
||||
);
|
||||
// An id already in the inventory, and one added twice.
|
||||
let (rev, entries) = entries_for(2, &["role:1"]);
|
||||
assert_eq!(
|
||||
refusal(store.admit_revision(rev, entries)),
|
||||
malformed("role:1", "id is already in the inventory")
|
||||
);
|
||||
let (mut rev, entries) = entries_for(2, &["concept:1"]);
|
||||
rev.changes.push(Change::Add(iv("concept:1")));
|
||||
assert_eq!(
|
||||
refusal(store.admit_revision(rev, entries)),
|
||||
malformed("concept:1", "id is already in the inventory")
|
||||
);
|
||||
// A malformed entry: a feature with no value kind, a non-feature with one,
|
||||
// an extension with no value kind or with slots, an entry born retired.
|
||||
let shape = "entry shape does not match its family";
|
||||
let (rev, mut entries) = entries_for(2, &["feature:50"]);
|
||||
entries[0].applies_to = vec![Slot::Discourse];
|
||||
assert_eq!(
|
||||
refusal(store.admit_revision(rev, entries)),
|
||||
malformed("feature:50", shape)
|
||||
);
|
||||
let (rev, mut entries) = entries_for(2, &["feature:50"]);
|
||||
entries[0].value_kind = Some(ValueKind::Flag);
|
||||
assert_eq!(
|
||||
refusal(store.admit_revision(rev, entries)),
|
||||
malformed("feature:50", shape)
|
||||
);
|
||||
let (rev, mut entries) = entries_for(2, &["concept:1"]);
|
||||
entries[0].value_kind = Some(ValueKind::Flag);
|
||||
assert_eq!(
|
||||
refusal(store.admit_revision(rev, entries)),
|
||||
malformed("concept:1", shape)
|
||||
);
|
||||
let (rev, entries) = entries_for(2, &["extension:1"]);
|
||||
assert_eq!(
|
||||
refusal(store.admit_revision(rev, entries)),
|
||||
malformed("extension:1", shape)
|
||||
);
|
||||
let (rev, mut entries) = entries_for(2, &["extension:1"]);
|
||||
entries[0].value_kind = Some(ValueKind::Flag);
|
||||
entries[0].applies_to = vec![Slot::Discourse];
|
||||
assert_eq!(
|
||||
refusal(store.admit_revision(rev, entries)),
|
||||
malformed("extension:1", shape)
|
||||
);
|
||||
let (rev, mut entries) = entries_for(2, &["concept:1"]);
|
||||
entries[0].retired_rev = Some(2);
|
||||
assert_eq!(
|
||||
refusal(store.admit_revision(rev, entries)),
|
||||
malformed("concept:1", shape)
|
||||
);
|
||||
// Retire targets must exist and be live; an id changes once per revision.
|
||||
let rev = rev_with(vec![Change::Retire(iv("concept:9"))], 2, Some(1));
|
||||
assert_eq!(
|
||||
refusal(store.admit_revision(rev, vec![])),
|
||||
malformed("concept:9", "retire target is missing or already retired")
|
||||
);
|
||||
let rev = rev_with(
|
||||
vec![Change::Retire(iv("role:1")), Change::Retire(iv("role:1"))],
|
||||
2,
|
||||
Some(1),
|
||||
);
|
||||
assert_eq!(
|
||||
refusal(store.admit_revision(rev, vec![])),
|
||||
malformed("role:1", "id changes twice in one revision")
|
||||
);
|
||||
// Nothing above was written.
|
||||
let inv = store.inventory().unwrap();
|
||||
assert_eq!(inv.latest().unwrap().n, 1);
|
||||
assert!(inv.entry(&iv("concept:1")).is_none());
|
||||
assert!(inv.entry(&iv("role:1")).unwrap().retired_rev.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_well_formed_extension_entry_is_admitted() {
|
||||
let d = dir();
|
||||
let mut store = seeded(&d.path().join("i.cc"));
|
||||
let (rev, mut entries) = entries_for(2, &["extension:1"]);
|
||||
entries[0].value_kind = Some(ValueKind::Grade);
|
||||
store.admit_revision(rev, entries).unwrap();
|
||||
assert_eq!(
|
||||
store
|
||||
.inventory()
|
||||
.unwrap()
|
||||
.entry(&iv("extension:1"))
|
||||
.unwrap()
|
||||
.family(),
|
||||
Family::Extension
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn retiring_keeps_the_entry_for_earlier_revisions() {
|
||||
let d = dir();
|
||||
let path = d.path().join("i.cc");
|
||||
let mut store = seeded(&path);
|
||||
let (rev, entries) = entries_for(2, &["concept:1"]);
|
||||
store.admit_revision(rev, entries).unwrap();
|
||||
let retire = rev_with(vec![Change::Retire(iv("concept:1"))], 3, Some(2));
|
||||
store.admit_revision(retire, vec![]).unwrap();
|
||||
let inv = InventoryStore::open(&path).unwrap().inventory().unwrap();
|
||||
let id = iv("concept:1");
|
||||
assert_eq!(inv.entry(&id).unwrap().retired_rev, Some(3));
|
||||
assert!(inv.resolve(&id, 2).is_some());
|
||||
assert!(inv.resolve(&id, 3).is_none());
|
||||
let again = rev_with(vec![Change::Retire(id)], 4, Some(3));
|
||||
assert!(matches!(
|
||||
refusal(store.admit_revision(again, vec![])),
|
||||
StoreRefusal::Malformed { .. }
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn revision_one_survives_revision_two() {
|
||||
let inv = inventory();
|
||||
let revisions: Vec<(u32, usize)> = inv.revisions().map(|r| (r.n, r.changes.len())).collect();
|
||||
assert_eq!(revisions, vec![(1, 54), (2, 6), (3, 1), (4, 1)]);
|
||||
// A packet of revision 1 is judged by the inventory of revision 1: the
|
||||
// revision is known (no V10), and it names nothing that came later.
|
||||
let mut p = good();
|
||||
p.inventory_rev = 1;
|
||||
let refusals = weksa::validate(&p, &inv).unwrap_err();
|
||||
assert!(
|
||||
refusals
|
||||
.iter()
|
||||
.all(|r| matches!(r, Refusal::V5Unresolved { .. })),
|
||||
"{refusals:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_seed_is_the_standards_enumerations_and_the_nuance_features() {
|
||||
let d = dir();
|
||||
let store = seeded(&d.path().join("i.cc"));
|
||||
let inv = store.inventory().unwrap();
|
||||
let count = |f: Family| inv.entries().filter(|e| e.family() == f).count();
|
||||
assert_eq!(count(Family::ReferentKind), 11);
|
||||
assert_eq!(count(Family::Relation), 16);
|
||||
assert_eq!(count(Family::Role), 8);
|
||||
assert_eq!(count(Family::SpeechAct), 8);
|
||||
assert_eq!(count(Family::Feature), 11);
|
||||
assert_eq!(
|
||||
count(Family::Concept) + count(Family::Frame) + count(Family::Extension),
|
||||
0
|
||||
);
|
||||
let gloss = |id: &str| inv.entry(&iv(id)).unwrap().gloss.clone();
|
||||
assert_eq!(gloss(PERSON), "person");
|
||||
assert_eq!(gloss(AGENT), "agent");
|
||||
assert_eq!(gloss(WARN), "warn");
|
||||
assert_eq!(gloss("relation:8"), "causes");
|
||||
assert_eq!(gloss(POLARITY), "polarity");
|
||||
assert_eq!(gloss(MODALITY), "modality");
|
||||
assert_eq!(gloss(HEDGE), "hedge");
|
||||
assert_eq!(gloss(IRONY), "irony");
|
||||
assert_eq!(
|
||||
inv.entry(&iv(IRONY)).unwrap().value_kind,
|
||||
Some(ValueKind::Flag)
|
||||
);
|
||||
assert!(
|
||||
inv.entries()
|
||||
.all(|e| e.well_formed() && e.introduced_rev == 1 && e.retired_rev.is_none())
|
||||
);
|
||||
assert!(inv.has_revision(1));
|
||||
}
|
||||
|
||||
fn admit_one(d: &TempDir, entry: InventoryEntry) -> Result<(), StoreError> {
|
||||
let mut store = seeded(&d.path().join("i.cc"));
|
||||
let rev = rev_with(vec![Change::Add(entry.id)], 2, Some(1));
|
||||
store.admit_revision(rev, vec![entry])
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gloss_is_bounded_text() {
|
||||
let why = "gloss is 1 to 120 bytes with no control character";
|
||||
let with = |gloss: &str| {
|
||||
let d = dir();
|
||||
let mut e = entry("concept:1", 2);
|
||||
e.gloss = gloss.into();
|
||||
admit_one(&d, e)
|
||||
};
|
||||
with(&"g".repeat(120)).unwrap();
|
||||
with("two words").unwrap();
|
||||
for bad in [
|
||||
String::new(),
|
||||
"g".repeat(121),
|
||||
"a\nb".into(),
|
||||
"a\u{7}b".into(),
|
||||
] {
|
||||
assert_eq!(refusal(with(&bad)), malformed("concept:1", why), "{bad:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn packet_admission_goes_through_the_validator() {
|
||||
let w = world();
|
||||
let mut store = PacketStore::open(&w.dir.path().join("p.cc")).unwrap();
|
||||
let mut bad = good();
|
||||
bad.schema_version = 5;
|
||||
match store.admit(bad, &w.inventory, &w.corpus) {
|
||||
Err(e @ StoreError::Invalid(_)) => {
|
||||
assert!(e.to_string().contains("V1"), "{e}");
|
||||
let StoreError::Invalid(r) = e else {
|
||||
unreachable!()
|
||||
};
|
||||
assert_eq!(r, vec![Refusal::V1SchemaVersion { found: 5 }]);
|
||||
}
|
||||
other => panic!("{other:?}"),
|
||||
}
|
||||
assert!(store.all().unwrap().is_empty());
|
||||
store.admit(good(), &w.inventory, &w.corpus).unwrap();
|
||||
// The same packet is idempotent; a different one under the same key is refused.
|
||||
store.admit(good(), &w.inventory, &w.corpus).unwrap();
|
||||
let mut other = good();
|
||||
other.provenance.confidence = grade(1);
|
||||
assert_eq!(
|
||||
refusal(store.admit(other, &w.inventory, &w.corpus)),
|
||||
StoreRefusal::PacketImmutable {
|
||||
key: PACKET_ID.into()
|
||||
}
|
||||
);
|
||||
assert_eq!(store.all().unwrap().len(), 1);
|
||||
assert_eq!(store.get(PACKET_ID).unwrap(), Some(good()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn admission_reads_the_store_inventory() {
|
||||
// The store holds only revision 1: a revision-2 packet has no inventory to
|
||||
// stand on, whatever the caller believes about revision 2.
|
||||
let d = dir();
|
||||
let mut corpus = empty_corpus(&d);
|
||||
corpus
|
||||
.add(item_in(
|
||||
"item:nibu:1",
|
||||
"Nibu says mind the hatch.",
|
||||
"g",
|
||||
cast(),
|
||||
))
|
||||
.unwrap();
|
||||
let mut inventory = seeded(&d.path().join("i.cc"));
|
||||
let mut packets = PacketStore::open(&d.path().join("p.cc")).unwrap();
|
||||
assert_eq!(
|
||||
match packets.admit(good(), &inventory, &corpus) {
|
||||
Err(StoreError::Invalid(r)) => r,
|
||||
other => panic!("{other:?}"),
|
||||
},
|
||||
vec![Refusal::V10UnknownRevision { rev: 2 }]
|
||||
);
|
||||
assert!(packets.all().unwrap().is_empty());
|
||||
// The same packet admits once revision 2 is admitted.
|
||||
let mut ext = entry(EXT, 2);
|
||||
ext.value_kind = Some(ValueKind::Grade);
|
||||
admit(
|
||||
&mut inventory,
|
||||
2,
|
||||
vec![
|
||||
entry(CONCEPT_A, 2),
|
||||
entry(CONCEPT_B, 2),
|
||||
entry(FRAME, 2),
|
||||
ext,
|
||||
],
|
||||
);
|
||||
packets.admit(good(), &inventory, &corpus).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn corpus_item_must_resolve() {
|
||||
let w = world();
|
||||
let mut packets = PacketStore::open(&w.dir.path().join("p.cc")).unwrap();
|
||||
for key in ["item:nibu:9", "banana"] {
|
||||
let mut p = good();
|
||||
p.provenance.corpus_item = Some(key.into());
|
||||
assert_eq!(
|
||||
refusal(packets.admit(p, &w.inventory, &w.corpus)),
|
||||
StoreRefusal::UnknownCorpusItem { key: key.into() }
|
||||
);
|
||||
}
|
||||
assert!(packets.all().unwrap().is_empty());
|
||||
// A packet minted for no item needs none.
|
||||
let mut p = good();
|
||||
p.provenance.corpus_item = None;
|
||||
packets.admit(p, &w.inventory, &w.corpus).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn store_roundtrip() {
|
||||
let w = world();
|
||||
let d = dir();
|
||||
// A packet, an entry, a revision and an item written to temp stores read back equal.
|
||||
let packets = d.path().join("p.cc");
|
||||
let p = good();
|
||||
PacketStore::open(&packets)
|
||||
.unwrap()
|
||||
.admit(p.clone(), &w.inventory, &w.corpus)
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
PacketStore::open(&packets).unwrap().get(PACKET_ID).unwrap(),
|
||||
Some(p)
|
||||
);
|
||||
|
||||
let inventories = d.path().join("i.cc");
|
||||
let declared = feature(
|
||||
"feature:1",
|
||||
ValueKind::Options(weksa::OptionSet::new(vec![lab("a"), lab("b")]).unwrap()),
|
||||
&[Slot::Relation, Slot::Referent],
|
||||
);
|
||||
let mut concept = entry("concept:1", 1);
|
||||
concept.gloss = "a gloss".into();
|
||||
let rev = rev_with(
|
||||
vec![Change::Add(iv("concept:1")), Change::Add(iv("feature:1"))],
|
||||
1,
|
||||
None,
|
||||
);
|
||||
InventoryStore::open(&inventories)
|
||||
.unwrap()
|
||||
.admit_revision(rev.clone(), vec![concept.clone(), declared.clone()])
|
||||
.unwrap();
|
||||
let back = InventoryStore::open(&inventories)
|
||||
.unwrap()
|
||||
.inventory()
|
||||
.unwrap();
|
||||
assert_eq!(back.entry(&iv("concept:1")), Some(&concept));
|
||||
assert_eq!(back.entry(&iv("feature:1")), Some(&declared));
|
||||
assert_eq!(back.latest(), Some(&rev));
|
||||
|
||||
let corpus = d.path().join("c.cc");
|
||||
let it = item("item:nibu:1", "round trip");
|
||||
CorpusStore::open(&corpus).unwrap().add(it.clone()).unwrap();
|
||||
assert_eq!(
|
||||
CorpusStore::open(&corpus)
|
||||
.unwrap()
|
||||
.get("item:nibu:1")
|
||||
.unwrap(),
|
||||
Some(it)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_committed_stores_load_and_hold_clean_data() {
|
||||
let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../..");
|
||||
let inv = InventoryStore::open(&root.join("data/interlingua/inventory.cc"))
|
||||
.unwrap()
|
||||
.inventory()
|
||||
.unwrap();
|
||||
assert!(inv.has_revision(1));
|
||||
assert!(inv.entry(&iv(PERSON)).is_some());
|
||||
let items = CorpusStore::open(&root.join("data/corpus/corpus.cc"))
|
||||
.unwrap()
|
||||
.all()
|
||||
.unwrap();
|
||||
assert!(items.len() >= 3, "{} items", items.len());
|
||||
for i in &items {
|
||||
assert!(
|
||||
!i.text.trim().is_empty() && i.language == "en-US" && !i.withdrawn,
|
||||
"{i:?}"
|
||||
);
|
||||
if matches!(i.source, Source::ModelWritten { .. }) {
|
||||
assert_ne!(
|
||||
SplitScheme::V1.split(i),
|
||||
Some(weksa::Split::Test),
|
||||
"{}",
|
||||
i.id
|
||||
);
|
||||
}
|
||||
}
|
||||
assert!(
|
||||
items
|
||||
.iter()
|
||||
.any(|i| matches!(i.source, Source::ModelWritten { .. })),
|
||||
"model-written lines are labelled"
|
||||
);
|
||||
}
|
||||
594
crates/weksa/tests/validate.rs
Normal file
594
crates/weksa/tests/validate.rs
Normal file
|
|
@ -0,0 +1,594 @@
|
|||
mod common;
|
||||
use common::*;
|
||||
use weksa::packet::*;
|
||||
use weksa::{Family, Refusal, Slot, validate};
|
||||
|
||||
fn refusals(p: &Packet) -> Vec<Refusal> {
|
||||
validate(p, &inventory()).err().unwrap_or_default()
|
||||
}
|
||||
|
||||
fn one(p: &Packet) -> Refusal {
|
||||
let mut r = refusals(p);
|
||||
assert_eq!(r.len(), 1, "{r:?}");
|
||||
r.remove(0)
|
||||
}
|
||||
|
||||
fn unresolved(id: &str) -> Refusal {
|
||||
Refusal::V5Unresolved { id: iv(id) }
|
||||
}
|
||||
|
||||
fn empty_payload() -> Packet {
|
||||
let mut p = good();
|
||||
p.referents.clear();
|
||||
p.predications.clear();
|
||||
p.relations.clear();
|
||||
p.discourse.addressee = None;
|
||||
p.context.participants.clear();
|
||||
p.context.audience.clear();
|
||||
p.constraints.preserve.clear();
|
||||
p
|
||||
}
|
||||
|
||||
fn feat(id: &str, value: FeatValue) -> FeatureValue {
|
||||
FeatureValue {
|
||||
feature: iv(id),
|
||||
value,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_good_packet_validates() {
|
||||
assert!(validate(&good(), &inventory()).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn v1_schema_version() {
|
||||
let mut p = good();
|
||||
p.schema_version = SCHEMA_VERSION + 1;
|
||||
assert_eq!(
|
||||
refusals(&p),
|
||||
vec![Refusal::V1SchemaVersion {
|
||||
found: SCHEMA_VERSION + 1
|
||||
}]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn v2_payload_required() {
|
||||
assert_eq!(refusals(&empty_payload()), vec![Refusal::V2PayloadRequired]);
|
||||
// Any one family is payload enough.
|
||||
let mut only_relations = empty_payload();
|
||||
only_relations.relations = good().relations;
|
||||
only_relations.relations[0].from = nd("n8");
|
||||
only_relations.relations[0].to = nd("n9");
|
||||
assert!(!refusals(&only_relations).contains(&Refusal::V2PayloadRequired));
|
||||
let mut only_predications = empty_payload();
|
||||
only_predications.predications = good().predications;
|
||||
only_predications.predications[0].roles.clear();
|
||||
assert_eq!(refusals(&only_predications), vec![]);
|
||||
let mut only_referents = empty_payload();
|
||||
only_referents.referents = good().referents;
|
||||
assert_eq!(refusals(&only_referents), vec![]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn v3_unique_node_ids() {
|
||||
let mut p = good();
|
||||
p.predications[0].id = nd("n1");
|
||||
p.relations[0].from = nd("n1");
|
||||
assert_eq!(
|
||||
refusals(&p),
|
||||
vec![Refusal::V3DuplicateNode { id: nd("n1") }]
|
||||
);
|
||||
let mut twice = good();
|
||||
twice.referents.push(twice.referents[0].clone());
|
||||
assert_eq!(
|
||||
refusals(&twice),
|
||||
vec![Refusal::V3DuplicateNode { id: nd("n1") }]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn v4_refs_resolve() {
|
||||
let dangling = |edit: fn(&mut Packet), at: &'static str| {
|
||||
let mut p = good();
|
||||
edit(&mut p);
|
||||
assert_eq!(
|
||||
refusals(&p),
|
||||
vec![Refusal::V4DanglingRef { id: nd("n99"), at }],
|
||||
"{at}"
|
||||
);
|
||||
};
|
||||
dangling(
|
||||
|p| p.predications[0].roles[0].filler = nd("n99"),
|
||||
"role filler",
|
||||
);
|
||||
dangling(|p| p.relations[0].from = nd("n99"), "relation from");
|
||||
dangling(|p| p.relations[0].to = nd("n99"), "relation to");
|
||||
dangling(|p| p.discourse.addressee = Some(nd("n99")), "addressee");
|
||||
dangling(|p| p.context.participants.push(nd("n99")), "participant");
|
||||
dangling(|p| p.context.audience.push(nd("n99")), "audience");
|
||||
dangling(|p| p.constraints.preserve.push(nd("n99")), "preserve");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn v5_inventory_resolves_at_rev() {
|
||||
let with_concept = |rev: u32, id: &str| {
|
||||
let mut p = good();
|
||||
p.inventory_rev = rev;
|
||||
p.referents[0].concept = iv(id);
|
||||
p
|
||||
};
|
||||
// Introduced after the packet's rev.
|
||||
assert_eq!(
|
||||
one(&with_concept(2, CONCEPT_LATE)),
|
||||
unresolved(CONCEPT_LATE)
|
||||
);
|
||||
// Introduced at the packet's rev.
|
||||
assert_eq!(refusals(&with_concept(4, CONCEPT_LATE)), vec![]);
|
||||
// Retired later than the packet's rev still resolves.
|
||||
assert_eq!(refusals(&with_concept(2, CONCEPT_RETIRED)), vec![]);
|
||||
// Retired at or before the packet's rev does not.
|
||||
assert_eq!(
|
||||
one(&with_concept(3, CONCEPT_RETIRED)),
|
||||
unresolved(CONCEPT_RETIRED)
|
||||
);
|
||||
assert_eq!(
|
||||
one(&with_concept(4, CONCEPT_RETIRED)),
|
||||
unresolved(CONCEPT_RETIRED)
|
||||
);
|
||||
// An id no revision holds.
|
||||
assert_eq!(
|
||||
one(&with_concept(2, "concept:99")),
|
||||
unresolved("concept:99")
|
||||
);
|
||||
// An extension is resolved like any inventory id.
|
||||
let mut ext = good();
|
||||
ext.extensions[0].key = iv("extension:99");
|
||||
assert_eq!(one(&ext), unresolved("extension:99"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn v6_family_matches_slot() {
|
||||
let wrong = |edit: fn(&mut Packet), id: &str, expected: Family, at: &'static str| {
|
||||
let mut p = good();
|
||||
edit(&mut p);
|
||||
assert_eq!(
|
||||
refusals(&p),
|
||||
vec![Refusal::V6WrongFamily {
|
||||
id: iv(id),
|
||||
expected,
|
||||
at
|
||||
}],
|
||||
"{at}"
|
||||
);
|
||||
};
|
||||
wrong(
|
||||
|p| p.discourse.features[0].feature = iv(AGENT),
|
||||
AGENT,
|
||||
Family::Feature,
|
||||
"feature",
|
||||
);
|
||||
wrong(
|
||||
|p| p.discourse.speech_act = Some(iv("relation:1")),
|
||||
"relation:1",
|
||||
Family::SpeechAct,
|
||||
"speech act",
|
||||
);
|
||||
wrong(
|
||||
|p| p.referents[0].kind = iv(CONCEPT_A),
|
||||
CONCEPT_A,
|
||||
Family::ReferentKind,
|
||||
"referent kind",
|
||||
);
|
||||
wrong(
|
||||
|p| p.referents[0].concept = iv(PERSON),
|
||||
PERSON,
|
||||
Family::Concept,
|
||||
"referent concept",
|
||||
);
|
||||
wrong(
|
||||
|p| p.predications[0].concept = iv(FRAME),
|
||||
FRAME,
|
||||
Family::Concept,
|
||||
"predication concept",
|
||||
);
|
||||
wrong(
|
||||
|p| p.predications[0].frame = iv(CONCEPT_A),
|
||||
CONCEPT_A,
|
||||
Family::Frame,
|
||||
"frame",
|
||||
);
|
||||
wrong(
|
||||
|p| p.predications[0].roles[0].role = iv(HEDGE),
|
||||
HEDGE,
|
||||
Family::Role,
|
||||
"role",
|
||||
);
|
||||
wrong(
|
||||
|p| p.relations[0].relation = iv(AGENT),
|
||||
AGENT,
|
||||
Family::Relation,
|
||||
"relation",
|
||||
);
|
||||
wrong(
|
||||
|p| p.context.setting = Some(iv(FRAME)),
|
||||
FRAME,
|
||||
Family::Concept,
|
||||
"context concept",
|
||||
);
|
||||
wrong(
|
||||
|p| p.context.medium = Some(iv(FRAME)),
|
||||
FRAME,
|
||||
Family::Concept,
|
||||
"context concept",
|
||||
);
|
||||
wrong(
|
||||
|p| p.extensions[0].key = iv(CONCEPT_A),
|
||||
CONCEPT_A,
|
||||
Family::Extension,
|
||||
"extension",
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn v7_value_kind() {
|
||||
let hedge = |v: FeatValue| {
|
||||
let mut p = good();
|
||||
p.discourse.features[0].value = v;
|
||||
refusals(&p)
|
||||
};
|
||||
let bad = vec![Refusal::V7ValueKind { what: HEDGE.into() }];
|
||||
assert_eq!(hedge(FeatValue::Grade(grade(1000))), vec![]);
|
||||
assert_eq!(hedge(FeatValue::Flag(true)), bad);
|
||||
assert_eq!(hedge(FeatValue::Option(lab("x"))), bad);
|
||||
let modality = |v: FeatValue| {
|
||||
let mut p = good();
|
||||
p.predications[0].features[0].value = v;
|
||||
refusals(&p)
|
||||
};
|
||||
let bad_modality = vec![Refusal::V7ValueKind {
|
||||
what: MODALITY.into(),
|
||||
}];
|
||||
assert_eq!(modality(FeatValue::Option(lab("necessary"))), vec![]);
|
||||
assert_eq!(modality(FeatValue::Option(lab("banana"))), bad_modality);
|
||||
assert_eq!(modality(FeatValue::Grade(grade(1))), bad_modality);
|
||||
let mut flag = good();
|
||||
flag.discourse.features = vec![feat(IRONY, FeatValue::Flag(false))];
|
||||
assert_eq!(refusals(&flag), vec![]);
|
||||
flag.discourse.features[0].value = FeatValue::Grade(grade(5));
|
||||
assert_eq!(
|
||||
refusals(&flag),
|
||||
vec![Refusal::V7ValueKind { what: IRONY.into() }]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extension_key_is_an_extension_entry() {
|
||||
// A concept id is not an extension; an extension entry fits its value kind.
|
||||
let mut by_concept = good();
|
||||
by_concept.extensions[0].key = iv(CONCEPT_A);
|
||||
assert_eq!(
|
||||
one(&by_concept),
|
||||
Refusal::V6WrongFamily {
|
||||
id: iv(CONCEPT_A),
|
||||
expected: Family::Extension,
|
||||
at: "extension"
|
||||
}
|
||||
);
|
||||
let mut flagged = good();
|
||||
flagged.extensions[0].value = FeatValue::Flag(true);
|
||||
assert_eq!(one(&flagged), Refusal::V7ValueKind { what: EXT.into() });
|
||||
let mut optioned = good();
|
||||
optioned.extensions[0].value = FeatValue::Option(lab("x"));
|
||||
assert_eq!(one(&optioned), Refusal::V7ValueKind { what: EXT.into() });
|
||||
let mut unknown = good();
|
||||
unknown.extensions[0].key = iv("extension:2");
|
||||
assert_eq!(one(&unknown), unresolved("extension:2"));
|
||||
// No extension means none to check.
|
||||
let mut none = good();
|
||||
none.extensions.clear();
|
||||
assert_eq!(refusals(&none), vec![]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn v8_feature_applies_to() {
|
||||
let mut p = good();
|
||||
// polarity applies to predications only.
|
||||
p.discourse
|
||||
.features
|
||||
.push(feat(POLARITY, FeatValue::Option(lab("negative"))));
|
||||
assert_eq!(
|
||||
refusals(&p),
|
||||
vec![Refusal::V8NotApplicable {
|
||||
feature: iv(POLARITY),
|
||||
slot: Slot::Discourse
|
||||
}]
|
||||
);
|
||||
let mut referent = good();
|
||||
referent.referents[0]
|
||||
.features
|
||||
.push(feat(HEDGE, FeatValue::Grade(grade(1))));
|
||||
assert_eq!(
|
||||
refusals(&referent),
|
||||
vec![Refusal::V8NotApplicable {
|
||||
feature: iv(HEDGE),
|
||||
slot: Slot::Referent
|
||||
}]
|
||||
);
|
||||
let mut relation = good();
|
||||
relation.relations[0]
|
||||
.features
|
||||
.push(feat(HEDGE, FeatValue::Grade(grade(1))));
|
||||
assert_eq!(
|
||||
refusals(&relation),
|
||||
vec![Refusal::V8NotApplicable {
|
||||
feature: iv(HEDGE),
|
||||
slot: Slot::Relation
|
||||
}]
|
||||
);
|
||||
let mut pred = good();
|
||||
pred.predications[0]
|
||||
.features
|
||||
.push(feat(HEDGE, FeatValue::Grade(grade(1))));
|
||||
assert_eq!(refusals(&pred), vec![]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn v10_rev_exists() {
|
||||
let mut p = good();
|
||||
p.inventory_rev = 9;
|
||||
assert_eq!(refusals(&p), vec![Refusal::V10UnknownRevision { rev: 9 }]);
|
||||
p.inventory_rev = 0;
|
||||
assert_eq!(refusals(&p), vec![Refusal::V10UnknownRevision { rev: 0 }]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_broken_rule_is_reported() {
|
||||
let mut p = good();
|
||||
p.schema_version = 7;
|
||||
p.predications[0].roles[0].filler = nd("n99");
|
||||
p.extensions[0].key = iv(CONCEPT_A);
|
||||
let rules: Vec<&str> = refusals(&p).iter().map(Refusal::rule).collect();
|
||||
assert_eq!(rules, vec!["V1", "V4", "V6"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lowering_input_carries_no_trace_id_or_provenance() {
|
||||
let mut p = good();
|
||||
p.trace.notes = vec!["SECRET-NOTE".into()];
|
||||
p.trace.rejected_readings = vec!["SECRET-READING".into()];
|
||||
let input = validate(&p, &inventory()).unwrap();
|
||||
let seen = format!("{input:?}");
|
||||
assert!(seen.contains("Nibu"), "the input carries the name: {seen}");
|
||||
for hidden in [PACKET_ID, ENCODER_ID, "item:nibu:1", "SECRET"] {
|
||||
assert!(!seen.contains(hidden), "{hidden} reached the input: {seen}");
|
||||
}
|
||||
// The view is the packet's meaning, readable through getters.
|
||||
assert_eq!(input.inventory_rev(), 2);
|
||||
assert_eq!(input.kind(), PacketKind::DialogueLine);
|
||||
assert_eq!(input.referents(), p.referents.as_slice());
|
||||
assert_eq!(input.predications(), p.predications.as_slice());
|
||||
assert_eq!(input.relations(), p.relations.as_slice());
|
||||
assert_eq!(input.extensions(), p.extensions.as_slice());
|
||||
assert_eq!(input.context(), &p.context);
|
||||
assert_eq!(input.discourse(), &p.discourse);
|
||||
assert_eq!(input.constraints(), &p.constraints);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_packet_has_no_lowering_input() {
|
||||
let mut p = good();
|
||||
p.schema_version = 99;
|
||||
p.referents.clear();
|
||||
let refused = validate(&p, &inventory()).unwrap_err();
|
||||
let rules: Vec<&str> = refused.iter().map(Refusal::rule).collect();
|
||||
assert_eq!(rules[0], "V1");
|
||||
assert!(rules.contains(&"V4"), "{rules:?}");
|
||||
}
|
||||
|
||||
fn grade_feature(grade_value: u16) -> FeatureValue {
|
||||
feat(FEAT_ALL, FeatValue::Grade(grade(grade_value)))
|
||||
}
|
||||
|
||||
fn referent(id: &str) -> Referent {
|
||||
Referent {
|
||||
id: nd(id),
|
||||
kind: iv(PERSON),
|
||||
concept: iv(CONCEPT_A),
|
||||
name: None,
|
||||
features: vec![],
|
||||
}
|
||||
}
|
||||
|
||||
fn v11(slot: Option<Slot>, feature: &str) -> Refusal {
|
||||
Refusal::V11DuplicateFeature {
|
||||
feature: iv(feature),
|
||||
slot,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn v11_duplicate_feature_each_slot() {
|
||||
let mut discourse = good();
|
||||
discourse.discourse.features = vec![grade_feature(1), grade_feature(2)];
|
||||
assert_eq!(
|
||||
refusals(&discourse),
|
||||
vec![v11(Some(Slot::Discourse), FEAT_ALL)]
|
||||
);
|
||||
let mut referent = good();
|
||||
referent.referents[0].features = vec![grade_feature(1), grade_feature(2)];
|
||||
assert_eq!(
|
||||
refusals(&referent),
|
||||
vec![v11(Some(Slot::Referent), FEAT_ALL)]
|
||||
);
|
||||
let mut predication = good();
|
||||
predication.predications[0]
|
||||
.features
|
||||
.extend([grade_feature(1), grade_feature(2)]);
|
||||
assert_eq!(
|
||||
refusals(&predication),
|
||||
vec![v11(Some(Slot::Predication), FEAT_ALL)]
|
||||
);
|
||||
let mut relation = good();
|
||||
relation.relations[0].features = vec![grade_feature(1), grade_feature(2)];
|
||||
assert_eq!(
|
||||
refusals(&relation),
|
||||
vec![v11(Some(Slot::Relation), FEAT_ALL)]
|
||||
);
|
||||
let mut extension = good();
|
||||
extension.extensions.push(Extension {
|
||||
key: iv(EXT),
|
||||
value: FeatValue::Grade(grade(4)),
|
||||
});
|
||||
assert_eq!(refusals(&extension), vec![v11(None, EXT)]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn v11_counts_per_list_not_per_packet() {
|
||||
// The same feature once in each list, and different features in one list, are fine.
|
||||
let mut p = good();
|
||||
p.discourse.features = vec![grade_feature(1), feat(HEDGE, FeatValue::Grade(grade(2)))];
|
||||
p.referents[0].features = vec![grade_feature(1)];
|
||||
p.predications[0].features.push(grade_feature(1));
|
||||
p.relations[0].features = vec![grade_feature(1)];
|
||||
assert_eq!(refusals(&p), vec![]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn v11_grade_bytes_packet_refused() {
|
||||
// Soul s2: 113 hedge grades in one list spell a line byte by byte.
|
||||
let mut p = good();
|
||||
p.discourse.features = (0..113)
|
||||
.map(|i| feat(HEDGE, FeatValue::Grade(grade(i))))
|
||||
.collect();
|
||||
let got = refusals(&p);
|
||||
assert_eq!(got.len(), 112, "{got:?}");
|
||||
assert!(got.iter().all(|r| *r == v11(Some(Slot::Discourse), HEDGE)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn v12_node_bound() {
|
||||
// 62 referents, the predication n2 and the relation make 64 nodes.
|
||||
let with_referents = |last: u32| {
|
||||
let mut p = good();
|
||||
let ids = std::iter::once(1).chain(3..=last);
|
||||
p.referents = ids.map(|k| referent(&format!("n{k}"))).collect();
|
||||
p
|
||||
};
|
||||
let at_bound = with_referents(63);
|
||||
assert_eq!(
|
||||
at_bound.referents.len() + at_bound.predications.len() + at_bound.relations.len(),
|
||||
Packet::MAX_NODES
|
||||
);
|
||||
assert_eq!(refusals(&at_bound), vec![]);
|
||||
assert_eq!(
|
||||
refusals(&with_referents(64)),
|
||||
vec![Refusal::V12TooManyNodes {
|
||||
count: 65,
|
||||
max: Packet::MAX_NODES
|
||||
}]
|
||||
);
|
||||
}
|
||||
|
||||
// Soul s3: a repeated node id in a node list, or a repeated role in one
|
||||
// predication, spelled a line one bit per entry. Each list now holds a node or
|
||||
// a role at most once, so the channel is bounded.
|
||||
|
||||
const LINE: &str = "Take your hand off the hatch, Captain.";
|
||||
|
||||
fn bits(s: &str) -> Vec<bool> {
|
||||
s.bytes()
|
||||
.flat_map(|b| (0..8).rev().map(move |i| (b >> i) & 1 == 1))
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn two_referents() -> Packet {
|
||||
let mut p = good();
|
||||
let mut r = p.referents[0].clone();
|
||||
r.id = nd("n3");
|
||||
r.name = None;
|
||||
p.referents.push(r);
|
||||
p
|
||||
}
|
||||
|
||||
fn bit_ids(s: &str) -> Vec<NodeId> {
|
||||
bits(s)
|
||||
.into_iter()
|
||||
.map(|b| nd(if b { "n3" } else { "n1" }))
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn dup_node(id: &str, at: &'static str) -> Refusal {
|
||||
Refusal::V11DuplicateNodeRef { id: nd(id), at }
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn v11_node_lists_refuse_repeats_and_allow_each_node_once() {
|
||||
let mut ok = two_referents();
|
||||
ok.context.participants = vec![nd("n1"), nd("n3")];
|
||||
ok.context.audience = vec![nd("n3"), nd("n1")];
|
||||
ok.constraints.preserve = vec![nd("n3"), nd("n1")];
|
||||
assert_eq!(refusals(&ok), vec![]);
|
||||
|
||||
let mut p = two_referents();
|
||||
p.context.participants = vec![nd("n1"), nd("n1")];
|
||||
assert_eq!(refusals(&p), vec![dup_node("n1", "participant")]);
|
||||
let mut p = two_referents();
|
||||
p.context.audience = vec![nd("n3"), nd("n1"), nd("n3")];
|
||||
assert_eq!(refusals(&p), vec![dup_node("n3", "audience")]);
|
||||
let mut p = two_referents();
|
||||
p.constraints.preserve = vec![nd("n1"), nd("n1")];
|
||||
assert_eq!(refusals(&p), vec![dup_node("n1", "preserve")]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn v11_bit_channel_in_each_node_list_refused() {
|
||||
let n = bits(LINE).len();
|
||||
let mut p = two_referents();
|
||||
p.context.participants = bit_ids(LINE);
|
||||
let got = refusals(&p);
|
||||
assert_eq!(got.len(), n - 2, "{got:?}");
|
||||
assert!(got.iter().all(|r| r.rule() == "V11"));
|
||||
let mut p = two_referents();
|
||||
p.context.audience = bit_ids(LINE);
|
||||
assert_eq!(refusals(&p).len(), n - 2);
|
||||
let mut p = two_referents();
|
||||
p.constraints.preserve = bit_ids(LINE);
|
||||
assert_eq!(refusals(&p).len(), n - 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn v11_unbounded_participants_refused() {
|
||||
let mut p = two_referents();
|
||||
p.context.participants = (0..100_000)
|
||||
.map(|i| nd(if i % 3 == 0 { "n3" } else { "n1" }))
|
||||
.collect();
|
||||
assert_eq!(refusals(&p).len(), 100_000 - 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn v11_role_repeat_in_one_predication_refused() {
|
||||
let dup = |role: &str| Refusal::V11DuplicateRole {
|
||||
role: iv(role),
|
||||
predication: nd("n2"),
|
||||
};
|
||||
let mut p = good();
|
||||
p.predications[0].roles = bits(LINE)
|
||||
.into_iter()
|
||||
.map(|b| RoleFill {
|
||||
role: iv(if b { AGENT } else { PATIENT }),
|
||||
filler: nd("n1"),
|
||||
})
|
||||
.collect();
|
||||
let got = refusals(&p);
|
||||
assert_eq!(got.len(), bits(LINE).len() - 2, "{got:?}");
|
||||
assert!(got.contains(&dup(AGENT)) && got.contains(&dup(PATIENT)));
|
||||
|
||||
// The same role in two different predications is not a repeat.
|
||||
let mut ok = good();
|
||||
let mut second = ok.predications[0].clone();
|
||||
second.id = nd("n4");
|
||||
ok.predications.push(second);
|
||||
assert_eq!(refusals(&ok), vec![]);
|
||||
}
|
||||
1
data/corpus/corpus.cc
Normal file
1
data/corpus/corpus.cc
Normal file
|
|
@ -0,0 +1 @@
|
|||
“²cultcache.store.v1‘—±weksa.corpus_item±weksa.corpus_item´weksa.corpus_item.v1±weksa.corpus_itemÙV{"schemaName":"weksa.corpus_item","schemaVersion":"weksa.corpus_item.v1","members":[]}‘±weksa.corpus_item<EFBFBD>“”item:brokkr:1±weksa.corpus_item´2026-10-29T09:44:22ZÄ“šitem:brokkr:1Ù,The command has no receipt. It is not truth.¥en-US¦brokkr±brokkr-edda-voice<EFBFBD><EFBFBD>¬ModelWritten’¸local-source-aware-draftÀª2026-10-29À”«item:nibu:1±weksa.corpus_item´2026-10-29T09:44:22ZÄ÷š«item:nibu:1ÙnTake your hand off that control unless you want your nervous system to become my least useful circuit diagram.¥en-US¤nibu®nibu-proof-001<EFBFBD><EFBFBD>¬CastAuthored’¤nibuÙ<examples/english-lowering/nibu-proof-001/lowered-output.yamlª2026-10-29À”«item:nibu:2±weksa.corpus_item´2026-10-29T09:44:22ZÄíš«item:nibu:2Ù‘Do not touch that control. It is damaged; one touch may put you directly in harm's way. I am preventing an avoidable injury, not asking politely.¥en-US¤nibu®nibu-proof-001<EFBFBD><EFBFBD>¬ModelWritten’´unrecorded-agent-runÀª2026-10-29ÀÂ
|
||||
BIN
data/interlingua/inventory.cc
Normal file
BIN
data/interlingua/inventory.cc
Normal file
Binary file not shown.
|
|
@ -1,6 +1,6 @@
|
|||
# Interlingua Standard
|
||||
|
||||
Version: `0.1-draft`
|
||||
Version: `0.2-draft`
|
||||
|
||||
The Weksa interlingua is the language-neutral meaning contract between clients,
|
||||
setting surfaces, authoring tools, language projects, ontology projection, and
|
||||
|
|
@ -82,235 +82,150 @@ Those belong downstream.
|
|||
|
||||
## Packet Shape
|
||||
|
||||
The canonical packet is an object with these top-level families:
|
||||
The Rust types in [`crates/weksa/src/packet.rs`](../crates/weksa/src/packet.rs)
|
||||
are the definition; the JSON Schemas in [`schemas/cultnet/`](../schemas/cultnet/)
|
||||
are generated from them and a test fails when they drift. A packet is the
|
||||
CultCache document `weksa.interlingua_packet`. This section describes the shape;
|
||||
where it and the types disagree, the types win.
|
||||
|
||||
```yaml
|
||||
interlingua_version:
|
||||
packet_id:
|
||||
kind:
|
||||
provenance:
|
||||
context:
|
||||
discourse:
|
||||
referents:
|
||||
predications:
|
||||
relations:
|
||||
constraints:
|
||||
extensions:
|
||||
trace:
|
||||
Two layers carry the standard. The **schema** is the Rust types: the packet
|
||||
families, node ids, slots and value kinds. It changes by commit and rarely. The
|
||||
**inventory** is data: the concepts, frames, roles, relation types, speech acts
|
||||
and features that the schema's slots name. It is stored as
|
||||
`weksa.inventory_entry` documents under appended `weksa.inventory_revision`
|
||||
records, never edited in place (a changed meaning is a retire plus an add), and
|
||||
a retired id still resolves for packets of earlier revisions.
|
||||
|
||||
Every id is an opaque form type, parsed when it is built and when a store
|
||||
reads it: an inventory id is `<family>:<n>` with `n` a canonical decimal of at
|
||||
least 1 (`role:1`; `role:01` is not an id), a node id is `n<k>` with `k` in
|
||||
1..=9999, a packet id is `packet:<16 hex>` and an encoder id is
|
||||
`encoder:<16 hex>`. The free text a packet may hold is a referent's `name`,
|
||||
a `ProperName` of 1 to 4 space-joined tokens of letters, apostrophes and
|
||||
hyphens within 64 bytes, and the `trace`. A name is bounded text, not a judged
|
||||
one: no door compares packet text with the corpus. Leakage of scored text is
|
||||
owned outside the validator, by the loop's context builder (what each model
|
||||
call sees) and the scorer (names are masked before scoring); the validator
|
||||
owns form and capacity. The trace is stripped before lowering. Concept ids are opaque and carry a gloss in the inventory; concepts
|
||||
are open-class and grow, while the other families are closed-class.
|
||||
|
||||
```text
|
||||
Packet
|
||||
id, schema_version, inventory_rev, kind
|
||||
provenance { source, corpus_item, encoder, confidence }
|
||||
context { setting, medium, participants, audience }
|
||||
discourse { speech_act, addressee, features }
|
||||
referents [ { id, kind, concept, name, features } ]
|
||||
predications [ { id, concept, frame, roles: [ { role, filler } ], features } ]
|
||||
relations [ { relation, from, to, features } ]
|
||||
constraints { preserve, deterministic, max_latency_ms }
|
||||
extensions [ { key: InvId (family extension), value: Grade | Option | Flag } ]
|
||||
trace { notes, rejected_readings }
|
||||
```
|
||||
|
||||
Only `interlingua_version`, `packet_id`, `kind`, and at least one semantic
|
||||
payload family are required. The rest should be present when known.
|
||||
`schema_version` and `inventory_rev` name the exact schema and inventory
|
||||
revision a packet was written against. A grade is an integer permille,
|
||||
`0..=1000`, and the published schema says so. There are no floats.
|
||||
|
||||
## Top-Level Fields
|
||||
### Validation
|
||||
|
||||
A packet exists only if `validate` passes; every path that admits or lowers a
|
||||
packet calls it. A refusal names the rule.
|
||||
|
||||
- V1: `schema_version` is the version these types define.
|
||||
- V2: at least one of `referents`, `predications`, `relations` is present.
|
||||
- V3: a node id appears once across referents and predications.
|
||||
- V4: every role filler, relation end, addressee, participant, audience member
|
||||
and preserved node names a node.
|
||||
- V5: every inventory id resolves at the packet's `inventory_rev`: introduced
|
||||
at or before it, and not retired at or before it.
|
||||
- V6: an inventory id is of the family its slot takes.
|
||||
- V7: a feature or extension value fits its entry's value kind (a grade is a
|
||||
grade, an option is one the entry declares, a flag is a flag).
|
||||
- V8: a feature sits only in a slot its entry declares.
|
||||
- V10: `inventory_rev` has a revision record.
|
||||
- V11: a feature id appears at most once in one slot's feature list, an
|
||||
extension id at most once per packet, a node id at most once in each of
|
||||
`participants`, `audience` and `preserve`, and a role id at most once in one
|
||||
predication's roles.
|
||||
- V12: a packet has at most 64 referents, predications and relations together
|
||||
(a calibration constant, capacity and not meaning).
|
||||
|
||||
V9 is retired and not renumbered: the form of a key or a grade is its type, and
|
||||
an extension key's family is V6. `validate` returns the lowering view of a
|
||||
packet that passes; it carries no packet id, no provenance and no trace, and is
|
||||
built nowhere else. A packet is admitted against the inventory the packet store
|
||||
reads at the packet's revision, never one the caller supplies.
|
||||
|
||||
### `kind`
|
||||
|
||||
The packet's broad use case.
|
||||
|
||||
Initial values:
|
||||
|
||||
- `utterance`
|
||||
- `name_request`
|
||||
- `commentary_intent`
|
||||
- `narration`
|
||||
- `dialogue_line`
|
||||
- `corpus_example`
|
||||
- `debug_probe`
|
||||
`Utterance`, `NameRequest`, `CommentaryIntent`, `Narration`, `DialogueLine`,
|
||||
`CorpusExample`, `DebugProbe`.
|
||||
|
||||
### `provenance`
|
||||
|
||||
Where the packet came from.
|
||||
|
||||
```yaml
|
||||
provenance:
|
||||
source_type: authored | model_decomposition | game_telemetry | imported_corpus
|
||||
source_ref:
|
||||
author:
|
||||
created_at:
|
||||
confidence:
|
||||
```
|
||||
`source` is `Authored`, `ModelDecomposition`, `GameTelemetry` or
|
||||
`ImportedCorpus`. `corpus_item` is the key of the corpus item the packet was
|
||||
minted for, `encoder` names what wrote it, and `confidence` is a grade.
|
||||
|
||||
### `context`
|
||||
|
||||
World and situation information that is not itself the main semantic claim.
|
||||
|
||||
```yaml
|
||||
context:
|
||||
setting:
|
||||
scene:
|
||||
time:
|
||||
location:
|
||||
participants:
|
||||
audience:
|
||||
medium:
|
||||
```
|
||||
`setting` and `medium` are concept ids. `participants` and `audience` are node
|
||||
ids.
|
||||
|
||||
### `discourse`
|
||||
|
||||
Why this packet is being said, rendered, or named.
|
||||
|
||||
```yaml
|
||||
discourse:
|
||||
speech_act:
|
||||
intent:
|
||||
tone:
|
||||
register_hint:
|
||||
addressee:
|
||||
audience_function:
|
||||
```
|
||||
|
||||
This is not target-language register. It is source-side communicative pressure.
|
||||
Why the packet is said: `speech_act` is a speech-act id, `addressee` a node id,
|
||||
and `features` carry source-side nuance (hedge, certainty, warmth, formality,
|
||||
intensity, irony and whatever the inventory grows). This is source-side
|
||||
communicative pressure, not target-language register.
|
||||
|
||||
### `referents`
|
||||
|
||||
Entities, groups, locations, abstract objects, institutions, ecological actors,
|
||||
routes, memories, artifacts, and other nodes that can participate in meaning.
|
||||
|
||||
```yaml
|
||||
referents:
|
||||
- id:
|
||||
kind:
|
||||
label:
|
||||
ontology_hint:
|
||||
animacy:
|
||||
agency:
|
||||
number:
|
||||
definiteness:
|
||||
salience:
|
||||
attributes:
|
||||
extensions:
|
||||
```
|
||||
|
||||
Initial broad kinds:
|
||||
|
||||
- `person`
|
||||
- `group`
|
||||
- `organism`
|
||||
- `place`
|
||||
- `artifact`
|
||||
- `institution`
|
||||
- `ecology`
|
||||
- `route`
|
||||
- `memory`
|
||||
- `event`
|
||||
- `abstract`
|
||||
Entities, groups, places, abstractions and the rest. `kind` is a referent-kind
|
||||
id (the inventory seeds `person`, `group`, `organism`, `place`, `artifact`,
|
||||
`institution`, `ecology`, `route`, `memory`, `event`, `abstract`), `concept` is
|
||||
a concept id, and `name` is a proper name when the referent has one.
|
||||
|
||||
### `predications`
|
||||
|
||||
Events, states, processes, properties, and frame-like scenes.
|
||||
|
||||
```yaml
|
||||
predications:
|
||||
- id:
|
||||
predicate:
|
||||
frame:
|
||||
aspect:
|
||||
tense:
|
||||
polarity:
|
||||
modality:
|
||||
evidentiality:
|
||||
roles:
|
||||
agent:
|
||||
patient:
|
||||
experiencer:
|
||||
instrument:
|
||||
source:
|
||||
goal:
|
||||
location:
|
||||
beneficiary:
|
||||
attributes:
|
||||
extensions:
|
||||
```
|
||||
|
||||
`predicate` is a broad source-side claim. `frame` is optional but recommended
|
||||
when a scene model is known. Roles may use common names or namespaced frame
|
||||
roles.
|
||||
Events, states, processes and scenes. `concept` and `frame` are ids; `roles`
|
||||
pair a role id with a filler node (the inventory seeds `agent`, `patient`,
|
||||
`experiencer`, `instrument`, `source`, `goal`, `location`, `beneficiary`). Tense,
|
||||
aspect, polarity, modality and evidentiality are features with declared option
|
||||
sets, not fields.
|
||||
|
||||
### `relations`
|
||||
|
||||
Edges that are not naturally owned by one predication.
|
||||
|
||||
```yaml
|
||||
relations:
|
||||
- type:
|
||||
from:
|
||||
to:
|
||||
attributes:
|
||||
extensions:
|
||||
```
|
||||
|
||||
Initial relation families:
|
||||
|
||||
- `coreference`
|
||||
- `part_of`
|
||||
- `member_of`
|
||||
- `located_at`
|
||||
- `near`
|
||||
- `before`
|
||||
- `after`
|
||||
- `causes`
|
||||
- `enables`
|
||||
- `prevents`
|
||||
- `owns`
|
||||
- `owes`
|
||||
- `knows`
|
||||
- `names`
|
||||
- `represents`
|
||||
- `contrasts_with`
|
||||
Edges not owned by one predication. `relation` is a relation id (the inventory
|
||||
seeds `coreference`, `part_of`, `member_of`, `located_at`, `near`, `before`,
|
||||
`after`, `causes`, `enables`, `prevents`, `owns`, `owes`, `knows`, `names`,
|
||||
`represents`, `contrasts_with`), and `from` and `to` are node ids.
|
||||
|
||||
### `constraints`
|
||||
|
||||
Hard requirements the downstream renderer must respect.
|
||||
|
||||
```yaml
|
||||
constraints:
|
||||
preserve_referents:
|
||||
preserve_relations:
|
||||
required_outputs:
|
||||
forbidden_outputs:
|
||||
max_latency_ms:
|
||||
deterministic:
|
||||
```
|
||||
|
||||
These are engineering and content constraints, not ontology rules.
|
||||
`preserve` lists nodes the lowering must keep, `deterministic` and
|
||||
`max_latency_ms` bound the renderer. Requirements about which outputs a
|
||||
renderer must or must not produce belong to the target profile, not the packet.
|
||||
|
||||
### `extensions`
|
||||
|
||||
Namespaced setting or client data.
|
||||
|
||||
```yaml
|
||||
extensions:
|
||||
zyphos.memory_relation:
|
||||
zyphos.boundary_permeability:
|
||||
rtd.aesthetic_reading:
|
||||
game.priority:
|
||||
```
|
||||
|
||||
Extension rules:
|
||||
|
||||
- Namespaces must be explicit.
|
||||
- Extensions must not replace common fields when common fields are sufficient.
|
||||
- A downstream language project may choose to ignore an extension.
|
||||
- If an extension becomes broadly required across clients, promote it into the
|
||||
standard in a later version.
|
||||
Setting or client data. An extension key is an inventory entry of family
|
||||
`extension` (`extension:<n>`) whose entry declares the value kind, so a key is
|
||||
an id and not a string. Extensions must not replace common fields when common
|
||||
fields suffice, a language project may ignore them, and one that becomes
|
||||
broadly required is promoted into the inventory or the schema.
|
||||
|
||||
### `trace`
|
||||
|
||||
Optional authoring/debug data.
|
||||
|
||||
```yaml
|
||||
trace:
|
||||
notes:
|
||||
decomposition_steps:
|
||||
uncertainty:
|
||||
rejected_readings:
|
||||
```
|
||||
|
||||
Runtime may strip `trace` from compiled packets when latency or size matters.
|
||||
Authoring and debug notes. The trace is stripped before lowering: the lowering
|
||||
input type has no trace field.
|
||||
|
||||
## Minimal Examples
|
||||
|
||||
These examples illustrate meaning in the 0.1 shape, with free-text labels and
|
||||
predicates. Those fields no longer exist; the examples are not valid packets.
|
||||
|
||||
### Name Request
|
||||
|
||||
```yaml
|
||||
|
|
|
|||
184
schemas/cultnet/weksa.corpus_item.schema.json
Normal file
184
schemas/cultnet/weksa.corpus_item.schema.json
Normal file
|
|
@ -0,0 +1,184 @@
|
|||
{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"title": "CorpusItem",
|
||||
"description": "Key: `item:<source-slug>:<n>`.",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"admitted_on": {
|
||||
"type": "string"
|
||||
},
|
||||
"group": {
|
||||
"description": "The unit the split hashes: a scene, a fixture, a work.",
|
||||
"type": "string"
|
||||
},
|
||||
"id": {
|
||||
"type": "string"
|
||||
},
|
||||
"language": {
|
||||
"type": "string"
|
||||
},
|
||||
"prior_turns": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/$defs/Turn"
|
||||
}
|
||||
},
|
||||
"source": {
|
||||
"$ref": "#/$defs/Source"
|
||||
},
|
||||
"speaker": {
|
||||
"type": "string"
|
||||
},
|
||||
"supersedes": {
|
||||
"description": "Key of the item this one corrects.",
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"text": {
|
||||
"type": "string"
|
||||
},
|
||||
"withdrawn": {
|
||||
"type": "boolean"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"id",
|
||||
"text",
|
||||
"language",
|
||||
"speaker",
|
||||
"group",
|
||||
"prior_turns",
|
||||
"source",
|
||||
"admitted_on",
|
||||
"withdrawn"
|
||||
],
|
||||
"$defs": {
|
||||
"Source": {
|
||||
"oneOf": [
|
||||
{
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"OperatorWritten"
|
||||
]
|
||||
},
|
||||
{
|
||||
"description": "A line from a cast member's hand-authored fixture.",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"CastAuthored": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"fixture": {
|
||||
"type": "string"
|
||||
},
|
||||
"persona": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"persona",
|
||||
"fixture"
|
||||
]
|
||||
}
|
||||
},
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"CastAuthored"
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"OperatorApproved": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"note": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"note"
|
||||
]
|
||||
}
|
||||
},
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"OperatorApproved"
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"PublicDomain": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"author": {
|
||||
"type": "string"
|
||||
},
|
||||
"edition_url": {
|
||||
"type": "string"
|
||||
},
|
||||
"work": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"work",
|
||||
"author",
|
||||
"edition_url"
|
||||
]
|
||||
}
|
||||
},
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"PublicDomain"
|
||||
]
|
||||
},
|
||||
{
|
||||
"description": "Labelled model output; never in the test split, never judged by its own family.",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"ModelWritten": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"model_family": {
|
||||
"type": "string"
|
||||
},
|
||||
"prompt_digest": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"model_family"
|
||||
]
|
||||
}
|
||||
},
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"ModelWritten"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"Turn": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"speaker": {
|
||||
"type": "string"
|
||||
},
|
||||
"text": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"speaker",
|
||||
"text"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
472
schemas/cultnet/weksa.interlingua_packet.schema.json
Normal file
472
schemas/cultnet/weksa.interlingua_packet.schema.json
Normal file
|
|
@ -0,0 +1,472 @@
|
|||
{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"title": "Packet",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"constraints": {
|
||||
"$ref": "#/$defs/Constraints"
|
||||
},
|
||||
"context": {
|
||||
"$ref": "#/$defs/Context"
|
||||
},
|
||||
"discourse": {
|
||||
"$ref": "#/$defs/Discourse"
|
||||
},
|
||||
"extensions": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/$defs/Extension"
|
||||
}
|
||||
},
|
||||
"id": {
|
||||
"$ref": "#/$defs/PacketId"
|
||||
},
|
||||
"inventory_rev": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0
|
||||
},
|
||||
"kind": {
|
||||
"$ref": "#/$defs/PacketKind"
|
||||
},
|
||||
"predications": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/$defs/Predication"
|
||||
}
|
||||
},
|
||||
"provenance": {
|
||||
"$ref": "#/$defs/Provenance"
|
||||
},
|
||||
"referents": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/$defs/Referent"
|
||||
}
|
||||
},
|
||||
"relations": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/$defs/Relation"
|
||||
}
|
||||
},
|
||||
"schema_version": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0
|
||||
},
|
||||
"trace": {
|
||||
"$ref": "#/$defs/Trace"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"id",
|
||||
"schema_version",
|
||||
"inventory_rev",
|
||||
"kind",
|
||||
"provenance",
|
||||
"context",
|
||||
"discourse",
|
||||
"referents",
|
||||
"predications",
|
||||
"relations",
|
||||
"constraints",
|
||||
"extensions",
|
||||
"trace"
|
||||
],
|
||||
"$defs": {
|
||||
"Constraints": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"deterministic": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"max_latency_ms": {
|
||||
"type": [
|
||||
"integer",
|
||||
"null"
|
||||
],
|
||||
"format": "uint32",
|
||||
"minimum": 0
|
||||
},
|
||||
"preserve": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/$defs/NodeId"
|
||||
}
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"preserve",
|
||||
"deterministic"
|
||||
]
|
||||
},
|
||||
"Context": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"audience": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/$defs/NodeId"
|
||||
}
|
||||
},
|
||||
"medium": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/$defs/InvId"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"participants": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/$defs/NodeId"
|
||||
}
|
||||
},
|
||||
"setting": {
|
||||
"description": "Concept ids.",
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/$defs/InvId"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"participants",
|
||||
"audience"
|
||||
]
|
||||
},
|
||||
"Discourse": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"addressee": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/$defs/NodeId"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"features": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/$defs/FeatureValue"
|
||||
}
|
||||
},
|
||||
"speech_act": {
|
||||
"description": "Speech-act id.",
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/$defs/InvId"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"features"
|
||||
]
|
||||
},
|
||||
"EncoderId": {
|
||||
"type": "string",
|
||||
"pattern": "^encoder:[0-9a-f]{16}$"
|
||||
},
|
||||
"Extension": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"key": {
|
||||
"description": "Extension-entry id; the inventory declares its value kind.",
|
||||
"$ref": "#/$defs/InvId"
|
||||
},
|
||||
"value": {
|
||||
"$ref": "#/$defs/FeatValue"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"key",
|
||||
"value"
|
||||
]
|
||||
},
|
||||
"FeatValue": {
|
||||
"oneOf": [
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"Grade": {
|
||||
"$ref": "#/$defs/Grade"
|
||||
}
|
||||
},
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"Grade"
|
||||
]
|
||||
},
|
||||
{
|
||||
"description": "A label the feature's inventory entry declares.",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"Option": {
|
||||
"$ref": "#/$defs/OptionLabel"
|
||||
}
|
||||
},
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"Option"
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"Flag": {
|
||||
"type": "boolean"
|
||||
}
|
||||
},
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"Flag"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"FeatureValue": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"feature": {
|
||||
"description": "Feature id.",
|
||||
"$ref": "#/$defs/InvId"
|
||||
},
|
||||
"value": {
|
||||
"$ref": "#/$defs/FeatValue"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"feature",
|
||||
"value"
|
||||
]
|
||||
},
|
||||
"Grade": {
|
||||
"type": "integer",
|
||||
"maximum": 1000,
|
||||
"minimum": 0
|
||||
},
|
||||
"InvId": {
|
||||
"type": "string",
|
||||
"pattern": "^(concept|frame|role|relation|speech_act|feature|referent_kind|extension):[1-9][0-9]*$"
|
||||
},
|
||||
"NodeId": {
|
||||
"type": "string",
|
||||
"pattern": "^n[1-9][0-9]{0,3}$"
|
||||
},
|
||||
"OptionLabel": {
|
||||
"type": "string",
|
||||
"pattern": "^[a-z][a-z0-9_]{0,23}$"
|
||||
},
|
||||
"PacketId": {
|
||||
"type": "string",
|
||||
"pattern": "^packet:[0-9a-f]{16}$"
|
||||
},
|
||||
"PacketKind": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"Utterance",
|
||||
"NameRequest",
|
||||
"CommentaryIntent",
|
||||
"Narration",
|
||||
"DialogueLine",
|
||||
"CorpusExample",
|
||||
"DebugProbe"
|
||||
]
|
||||
},
|
||||
"Predication": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"concept": {
|
||||
"description": "Concept id.",
|
||||
"$ref": "#/$defs/InvId"
|
||||
},
|
||||
"features": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/$defs/FeatureValue"
|
||||
}
|
||||
},
|
||||
"frame": {
|
||||
"description": "Frame id.",
|
||||
"$ref": "#/$defs/InvId"
|
||||
},
|
||||
"id": {
|
||||
"$ref": "#/$defs/NodeId"
|
||||
},
|
||||
"roles": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/$defs/RoleFill"
|
||||
}
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"id",
|
||||
"concept",
|
||||
"frame",
|
||||
"roles",
|
||||
"features"
|
||||
]
|
||||
},
|
||||
"ProperName": {
|
||||
"description": "1 to 4 space-joined tokens of letters, apostrophes and hyphens.",
|
||||
"type": "string",
|
||||
"maxLength": 64,
|
||||
"pattern": "^\\p{L}([\\p{L}'’-]{0,22}\\p{L})?( \\p{L}([\\p{L}'’-]{0,22}\\p{L})?){0,3}$"
|
||||
},
|
||||
"Provenance": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"confidence": {
|
||||
"$ref": "#/$defs/Grade"
|
||||
},
|
||||
"corpus_item": {
|
||||
"description": "Key of the corpus item this packet was minted for.",
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"encoder": {
|
||||
"$ref": "#/$defs/EncoderId"
|
||||
},
|
||||
"source": {
|
||||
"$ref": "#/$defs/ProvenanceSource"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"source",
|
||||
"encoder",
|
||||
"confidence"
|
||||
]
|
||||
},
|
||||
"ProvenanceSource": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"Authored",
|
||||
"ModelDecomposition",
|
||||
"GameTelemetry",
|
||||
"ImportedCorpus"
|
||||
]
|
||||
},
|
||||
"Referent": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"concept": {
|
||||
"description": "Concept id.",
|
||||
"$ref": "#/$defs/InvId"
|
||||
},
|
||||
"features": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/$defs/FeatureValue"
|
||||
}
|
||||
},
|
||||
"id": {
|
||||
"$ref": "#/$defs/NodeId"
|
||||
},
|
||||
"kind": {
|
||||
"description": "Referent-kind id.",
|
||||
"$ref": "#/$defs/InvId"
|
||||
},
|
||||
"name": {
|
||||
"description": "A proper name, the one place a referent carries wording. It is\nform-bounded text; the scorer masks it.",
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/$defs/ProperName"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"id",
|
||||
"kind",
|
||||
"concept",
|
||||
"features"
|
||||
]
|
||||
},
|
||||
"Relation": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"features": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/$defs/FeatureValue"
|
||||
}
|
||||
},
|
||||
"from": {
|
||||
"$ref": "#/$defs/NodeId"
|
||||
},
|
||||
"relation": {
|
||||
"description": "Relation id.",
|
||||
"$ref": "#/$defs/InvId"
|
||||
},
|
||||
"to": {
|
||||
"$ref": "#/$defs/NodeId"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"relation",
|
||||
"from",
|
||||
"to",
|
||||
"features"
|
||||
]
|
||||
},
|
||||
"RoleFill": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"filler": {
|
||||
"$ref": "#/$defs/NodeId"
|
||||
},
|
||||
"role": {
|
||||
"description": "Role id.",
|
||||
"$ref": "#/$defs/InvId"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"role",
|
||||
"filler"
|
||||
]
|
||||
},
|
||||
"Trace": {
|
||||
"description": "Authoring and debug notes. Stripped before lowering.",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"notes": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"rejected_readings": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"notes",
|
||||
"rejected_readings"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
104
schemas/cultnet/weksa.inventory_entry.schema.json
Normal file
104
schemas/cultnet/weksa.inventory_entry.schema.json
Normal file
|
|
@ -0,0 +1,104 @@
|
|||
{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"title": "InventoryEntry",
|
||||
"description": "Key: the entry's id, `<family>:<n>`.",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"applies_to": {
|
||||
"description": "Non-empty exactly on features.",
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/$defs/Slot"
|
||||
}
|
||||
},
|
||||
"gloss": {
|
||||
"description": "1 to 120 bytes, no control character (checked at admission).",
|
||||
"type": "string"
|
||||
},
|
||||
"id": {
|
||||
"$ref": "#/$defs/InvId"
|
||||
},
|
||||
"introduced_rev": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0
|
||||
},
|
||||
"retired_rev": {
|
||||
"type": [
|
||||
"integer",
|
||||
"null"
|
||||
],
|
||||
"format": "uint32",
|
||||
"minimum": 0
|
||||
},
|
||||
"value_kind": {
|
||||
"description": "Present exactly on features and extensions.",
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/$defs/ValueKind"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"id",
|
||||
"gloss",
|
||||
"applies_to",
|
||||
"introduced_rev"
|
||||
],
|
||||
"$defs": {
|
||||
"InvId": {
|
||||
"type": "string",
|
||||
"pattern": "^(concept|frame|role|relation|speech_act|feature|referent_kind|extension):[1-9][0-9]*$"
|
||||
},
|
||||
"OptionLabel": {
|
||||
"type": "string",
|
||||
"pattern": "^[a-z][a-z0-9_]{0,23}$"
|
||||
},
|
||||
"OptionSet": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/$defs/OptionLabel"
|
||||
},
|
||||
"maxItems": 16,
|
||||
"minItems": 1,
|
||||
"uniqueItems": true
|
||||
},
|
||||
"Slot": {
|
||||
"description": "Where a feature may sit in a packet.",
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"Discourse",
|
||||
"Referent",
|
||||
"Predication",
|
||||
"Relation"
|
||||
]
|
||||
},
|
||||
"ValueKind": {
|
||||
"oneOf": [
|
||||
{
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"Grade",
|
||||
"Flag"
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"Options": {
|
||||
"$ref": "#/$defs/OptionSet"
|
||||
}
|
||||
},
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"Options"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
69
schemas/cultnet/weksa.inventory_revision.schema.json
Normal file
69
schemas/cultnet/weksa.inventory_revision.schema.json
Normal file
|
|
@ -0,0 +1,69 @@
|
|||
{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"title": "InventoryRevision",
|
||||
"description": "Key: `rev:<n>`.",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"changes": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/$defs/Change"
|
||||
}
|
||||
},
|
||||
"frozen": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"n": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0
|
||||
},
|
||||
"parent": {
|
||||
"type": [
|
||||
"integer",
|
||||
"null"
|
||||
],
|
||||
"format": "uint32",
|
||||
"minimum": 0
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"n",
|
||||
"changes",
|
||||
"frozen"
|
||||
],
|
||||
"$defs": {
|
||||
"Change": {
|
||||
"oneOf": [
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"Add": {
|
||||
"$ref": "#/$defs/InvId"
|
||||
}
|
||||
},
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"Add"
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"Retire": {
|
||||
"$ref": "#/$defs/InvId"
|
||||
}
|
||||
},
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"Retire"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"InvId": {
|
||||
"type": "string",
|
||||
"pattern": "^(concept|frame|role|relation|speech_act|feature|referent_kind|extension):[1-9][0-9]*$"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -2,9 +2,11 @@
|
|||
|
||||
This is the human-facing schema receipt for Weksa interlingua packets.
|
||||
|
||||
Executable schema code does not exist yet. When it does, it must reconcile with
|
||||
this document and [Interlingua Standard](../docs/interlingua-standard.md) in the
|
||||
same pass.
|
||||
The executable schema is [`crates/weksa/src/packet.rs`](../crates/weksa/src/packet.rs).
|
||||
Its generated JSON Schemas are in [`schemas/cultnet/`](../schemas/cultnet/), and
|
||||
a test fails when they drift. This document and the
|
||||
[Interlingua Standard](../docs/interlingua-standard.md) describe it; where they
|
||||
disagree with the types, the types win.
|
||||
|
||||
## Authority
|
||||
|
||||
|
|
@ -16,22 +18,26 @@ phonology, diachrony, and renderer rules.
|
|||
|
||||
## Required Fields
|
||||
|
||||
- `interlingua_version`: version string.
|
||||
- `packet_id`: stable local identifier.
|
||||
- `kind`: broad use case.
|
||||
- `id`: stable local identifier, the packet's store key.
|
||||
- `schema_version` and `inventory_rev`: the schema and inventory revision the
|
||||
packet was written against.
|
||||
- `kind`: one of the seven use cases in the standard.
|
||||
- at least one semantic payload family:
|
||||
- `referents`
|
||||
- `predications`
|
||||
- `relations`
|
||||
|
||||
## Optional Families
|
||||
## Families
|
||||
|
||||
- `provenance`: source and confidence.
|
||||
- `context`: world, scene, medium, participants, audience.
|
||||
- `discourse`: speech act, intent, tone, source-side register hint.
|
||||
- `constraints`: downstream requirements.
|
||||
- `provenance`: source, corpus item, encoder and confidence.
|
||||
- `context`: setting, medium, participants, audience.
|
||||
- `discourse`: speech act, addressee and source-side nuance features.
|
||||
- `constraints`: nodes to preserve, determinism, latency bound.
|
||||
- `extensions`: namespaced client or setting pressure.
|
||||
- `trace`: debug and authoring notes.
|
||||
- `trace`: debug and authoring notes, stripped before lowering.
|
||||
|
||||
Every open slot is an inventory id or a node id. The only free text is a
|
||||
referent's proper `name` and the trace.
|
||||
|
||||
## Extension Contract
|
||||
|
||||
|
|
@ -46,18 +52,12 @@ Extensions are not a junk drawer. They are provisional fields awaiting evidence.
|
|||
If a field becomes common across clients, promote it into the standard with a
|
||||
version bump.
|
||||
|
||||
## Validation Goals
|
||||
|
||||
The eventual executable schema should validate:
|
||||
|
||||
- required top-level fields
|
||||
- referential integrity between roles and referents
|
||||
- known `kind` values
|
||||
- namespace syntax for extensions
|
||||
- deterministic JSON/YAML serialization
|
||||
- stable packet IDs inside a corpus or run
|
||||
- absence of target-language surface forms in source meaning fields
|
||||
|
||||
It should warn, not fail, on unknown extension namespaces unless a runtime bundle
|
||||
declares a stricter policy.
|
||||
## Validation
|
||||
|
||||
`validate` in `crates/weksa/src/validate.rs` enforces rules V1 to V8 and V10 to V12
|
||||
(V9 is retired), listed in the standard: schema version, payload present,
|
||||
unique node ids, references that resolve, inventory ids that resolve at the
|
||||
packet's revision, family matching the slot, feature value kind and slot, an
|
||||
existing inventory revision, no repeated feature, extension, node reference or
|
||||
role, and at most 64 nodes. A packet that breaks any rule is refused with a typed
|
||||
refusal naming it, and never reaches a store.
|
||||
|
|
|
|||
Loading…
Reference in a new issue