Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 917430ce99 | |||
| 4b276dfebc | |||
| 556324017e | |||
| c9a33f7ed5 | |||
| 41fa789d6f | |||
| 416e385c97 | |||
| a424c1c4d5 | |||
| 2ca3105322 | |||
| 07942c1a83 | |||
| 64e88966ad | |||
| 211cfe5e82 | |||
| 0c2d1e2cab | |||
| 477482b1a0 | |||
| db9605fbf2 | |||
| f09bc56e77 | |||
| 232764bf78 | |||
|
|
ecf147dbd7 | ||
|
|
b721c3eac7 | ||
|
|
51a13b7e82 | ||
|
|
8b18434282 | ||
|
|
43c3b1c047 | ||
|
|
65642dcf3c | ||
| dcae9c5c2c | |||
|
|
b920f3f81f | ||
|
|
fbedfdce2f | ||
|
|
e719aa3766 | ||
| b4f3a48a8f | |||
| f1d5c4039b | |||
| 36dd8ce3ca | |||
| 421ace5432 | |||
| b299d6f732 | |||
| 79d954cad1 | |||
| 776367ef10 | |||
| 0115915b37 | |||
| ea4193e0e4 | |||
| d08b8880ff | |||
| da90d7e74e | |||
| bf9a8a7f13 | |||
| 9dde93e5f5 | |||
| 86372ae57b | |||
| 85b92f7f96 | |||
| ec8ba4b84c | |||
| 650e5e262e | |||
| 8a801d09dc | |||
| 66e6fe8d7d | |||
| 51835636b8 | |||
| 5f7314c668 | |||
| e988ecd521 | |||
| 5bd58b6e3f | |||
| 6ff4c50b29 | |||
| 0ff04c396e | |||
| eea4d7edd5 | |||
| 94eb9401a5 | |||
| 2fb649e6ff | |||
| ec2d11efe8 | |||
| cd3066dde9 | |||
| aad5e204a2 | |||
| 5e8573235b |
229 changed files with 16668 additions and 2375 deletions
168
Cargo.lock
generated
168
Cargo.lock
generated
|
|
@ -37,17 +37,6 @@ dependencies = [
|
||||||
"subtle",
|
"subtle",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "ahash"
|
|
||||||
version = "0.7.8"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "891477e0c6a8957309ee5c45a6368af3ae14bb510732d2684ffa19af310920f9"
|
|
||||||
dependencies = [
|
|
||||||
"getrandom 0.2.17",
|
|
||||||
"once_cell",
|
|
||||||
"version_check",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "ahash"
|
name = "ahash"
|
||||||
version = "0.8.12"
|
version = "0.8.12"
|
||||||
|
|
@ -195,12 +184,6 @@ version = "0.8.0"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7"
|
checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7"
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "bitflags"
|
|
||||||
version = "1.3.2"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "bitflags"
|
name = "bitflags"
|
||||||
version = "2.13.2"
|
version = "2.13.2"
|
||||||
|
|
@ -369,18 +352,21 @@ dependencies = [
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "cultcache-rs"
|
name = "cultcache-rs"
|
||||||
version = "0.2.0"
|
version = "0.3.1"
|
||||||
source = "git+https://github.com/GameCult/CultLib.git?rev=8fc74c709444b884a2bd339e0c306bf9484d99d2#8fc74c709444b884a2bd339e0c306bf9484d99d2"
|
source = "git+https://github.com/GameCult/CultLib.git?rev=75188fdefa0b4baf131dbfd94e77d06a965f0cc6#75188fdefa0b4baf131dbfd94e77d06a965f0cc6"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"anyhow",
|
"anyhow",
|
||||||
"chrono",
|
"chrono",
|
||||||
"cultcache-rs-derive 0.1.0",
|
"cultcache-rs-derive 0.2.0 (git+https://github.com/GameCult/CultLib.git?rev=75188fdefa0b4baf131dbfd94e77d06a965f0cc6)",
|
||||||
"fs2",
|
"fs2",
|
||||||
"redb",
|
"redb",
|
||||||
"rmp-serde",
|
"rmp-serde",
|
||||||
"serde",
|
"serde",
|
||||||
"serde_bytes",
|
"serde_bytes",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
|
"serde_stacker",
|
||||||
|
"sha2",
|
||||||
|
"stacker",
|
||||||
"uuid",
|
"uuid",
|
||||||
"windows-sys 0.59.0",
|
"windows-sys 0.59.0",
|
||||||
]
|
]
|
||||||
|
|
@ -392,7 +378,7 @@ source = "git+https://github.com/GameCult/CultLib.git?rev=a4eae9056152335662f111
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"anyhow",
|
"anyhow",
|
||||||
"chrono",
|
"chrono",
|
||||||
"cultcache-rs-derive 0.2.0",
|
"cultcache-rs-derive 0.2.0 (git+https://github.com/GameCult/CultLib.git?rev=a4eae9056152335662f1115f97bf35fb00ab6d23)",
|
||||||
"fs2",
|
"fs2",
|
||||||
"redb",
|
"redb",
|
||||||
"rmp-serde",
|
"rmp-serde",
|
||||||
|
|
@ -407,8 +393,8 @@ dependencies = [
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "cultcache-rs-derive"
|
name = "cultcache-rs-derive"
|
||||||
version = "0.1.0"
|
version = "0.2.0"
|
||||||
source = "git+https://github.com/GameCult/CultLib.git?rev=8fc74c709444b884a2bd339e0c306bf9484d99d2#8fc74c709444b884a2bd339e0c306bf9484d99d2"
|
source = "git+https://github.com/GameCult/CultLib.git?rev=75188fdefa0b4baf131dbfd94e77d06a965f0cc6#75188fdefa0b4baf131dbfd94e77d06a965f0cc6"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"proc-macro2",
|
"proc-macro2",
|
||||||
"quote",
|
"quote",
|
||||||
|
|
@ -427,18 +413,17 @@ dependencies = [
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "cultnet-rs"
|
name = "cultnet-rs"
|
||||||
version = "0.1.0"
|
version = "0.2.0"
|
||||||
source = "git+https://github.com/GameCult/CultLib.git?rev=8fc74c709444b884a2bd339e0c306bf9484d99d2#8fc74c709444b884a2bd339e0c306bf9484d99d2"
|
source = "git+https://github.com/GameCult/CultLib.git?rev=a4eae9056152335662f1115f97bf35fb00ab6d23#a4eae9056152335662f1115f97bf35fb00ab6d23"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"aes-gcm",
|
"aes-gcm",
|
||||||
"anyhow",
|
"anyhow",
|
||||||
"base64",
|
"base64",
|
||||||
"chrono",
|
"chrono",
|
||||||
"cultcache-rs 0.2.0",
|
"cultcache-rs 0.3.1 (git+https://github.com/GameCult/CultLib.git?rev=a4eae9056152335662f1115f97bf35fb00ab6d23)",
|
||||||
"ed25519-dalek",
|
"ed25519-dalek",
|
||||||
"hmac",
|
"hmac",
|
||||||
"rand",
|
"rand",
|
||||||
"reed-solomon-erasure",
|
|
||||||
"rmp-serde",
|
"rmp-serde",
|
||||||
"rmpv",
|
"rmpv",
|
||||||
"serde",
|
"serde",
|
||||||
|
|
@ -453,16 +438,18 @@ dependencies = [
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "cultnet-rs"
|
name = "cultnet-rs"
|
||||||
version = "0.2.0"
|
version = "0.3.0"
|
||||||
source = "git+https://github.com/GameCult/CultLib.git?rev=a4eae9056152335662f1115f97bf35fb00ab6d23#a4eae9056152335662f1115f97bf35fb00ab6d23"
|
source = "git+https://github.com/GameCult/CultLib.git?rev=75188fdefa0b4baf131dbfd94e77d06a965f0cc6#75188fdefa0b4baf131dbfd94e77d06a965f0cc6"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"aes-gcm",
|
"aes-gcm",
|
||||||
"anyhow",
|
"anyhow",
|
||||||
"base64",
|
"base64",
|
||||||
"chrono",
|
"chrono",
|
||||||
"cultcache-rs 0.3.1",
|
"cultcache-rs 0.3.1 (git+https://github.com/GameCult/CultLib.git?rev=75188fdefa0b4baf131dbfd94e77d06a965f0cc6)",
|
||||||
"ed25519-dalek",
|
"ed25519-dalek",
|
||||||
|
"fs2",
|
||||||
"hmac",
|
"hmac",
|
||||||
|
"libc",
|
||||||
"rand",
|
"rand",
|
||||||
"rmp-serde",
|
"rmp-serde",
|
||||||
"rmpv",
|
"rmpv",
|
||||||
|
|
@ -625,7 +612,7 @@ version = "0.1.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"anyhow",
|
"anyhow",
|
||||||
"chrono",
|
"chrono",
|
||||||
"cultcache-rs 0.2.0",
|
"cultcache-rs 0.3.1 (git+https://github.com/GameCult/CultLib.git?rev=75188fdefa0b4baf131dbfd94e77d06a965f0cc6)",
|
||||||
"rmp-serde",
|
"rmp-serde",
|
||||||
"schemars",
|
"schemars",
|
||||||
"serde",
|
"serde",
|
||||||
|
|
@ -639,7 +626,7 @@ dependencies = [
|
||||||
"anyhow",
|
"anyhow",
|
||||||
"chrono",
|
"chrono",
|
||||||
"clap",
|
"clap",
|
||||||
"cultnet-rs 0.1.0",
|
"cultnet-rs 0.3.0",
|
||||||
"huginn-daemon",
|
"huginn-daemon",
|
||||||
"huginn-mind",
|
"huginn-mind",
|
||||||
"rmcp",
|
"rmcp",
|
||||||
|
|
@ -874,15 +861,6 @@ dependencies = [
|
||||||
"unicode-segmentation",
|
"unicode-segmentation",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "hashbrown"
|
|
||||||
version = "0.12.3"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
|
|
||||||
dependencies = [
|
|
||||||
"ahash 0.7.8",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "hashbrown"
|
name = "hashbrown"
|
||||||
version = "0.17.1"
|
version = "0.17.1"
|
||||||
|
|
@ -932,7 +910,7 @@ dependencies = [
|
||||||
"anyhow",
|
"anyhow",
|
||||||
"base64",
|
"base64",
|
||||||
"chrono",
|
"chrono",
|
||||||
"cultnet-rs 0.1.0",
|
"cultnet-rs 0.3.0",
|
||||||
"huginn-mind",
|
"huginn-mind",
|
||||||
"rmp-serde",
|
"rmp-serde",
|
||||||
"rustix",
|
"rustix",
|
||||||
|
|
@ -951,8 +929,8 @@ dependencies = [
|
||||||
"anyhow",
|
"anyhow",
|
||||||
"base64",
|
"base64",
|
||||||
"chrono",
|
"chrono",
|
||||||
"cultcache-rs 0.2.0",
|
"cultcache-rs 0.3.1 (git+https://github.com/GameCult/CultLib.git?rev=75188fdefa0b4baf131dbfd94e77d06a965f0cc6)",
|
||||||
"cultnet-rs 0.1.0",
|
"cultnet-rs 0.3.0",
|
||||||
"eureka-pipeline",
|
"eureka-pipeline",
|
||||||
"jsonschema",
|
"jsonschema",
|
||||||
"rmp-serde",
|
"rmp-serde",
|
||||||
|
|
@ -1004,15 +982,6 @@ dependencies = [
|
||||||
"generic-array",
|
"generic-array",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "instant"
|
|
||||||
version = "0.1.13"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222"
|
|
||||||
dependencies = [
|
|
||||||
"cfg-if",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "is_terminal_polyfill"
|
name = "is_terminal_polyfill"
|
||||||
version = "1.70.2"
|
version = "1.70.2"
|
||||||
|
|
@ -1042,7 +1011,7 @@ version = "0.58.6"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "f622c574a29fb3034a0a90645afee194dfedc75fc124228801b6e77041075bd1"
|
checksum = "f622c574a29fb3034a0a90645afee194dfedc75fc124228801b6e77041075bd1"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"ahash 0.8.12",
|
"ahash",
|
||||||
"bytecount",
|
"bytecount",
|
||||||
"data-encoding",
|
"data-encoding",
|
||||||
"email_address",
|
"email_address",
|
||||||
|
|
@ -1079,7 +1048,7 @@ version = "0.58.6"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "39b7670345e5ce87371339d13705dbb9276b7e3e934bbe1efa2c6734a1f65a96"
|
checksum = "39b7670345e5ce87371339d13705dbb9276b7e3e934bbe1efa2c6734a1f65a96"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"ahash 0.8.12",
|
"ahash",
|
||||||
"bytecount",
|
"bytecount",
|
||||||
"fraction",
|
"fraction",
|
||||||
"getrandom 0.3.4",
|
"getrandom 0.3.4",
|
||||||
|
|
@ -1095,12 +1064,6 @@ version = "0.2.189"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2"
|
checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2"
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "libm"
|
|
||||||
version = "0.2.16"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981"
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "linux-raw-sys"
|
name = "linux-raw-sys"
|
||||||
version = "0.12.1"
|
version = "0.12.1"
|
||||||
|
|
@ -1122,15 +1085,6 @@ version = "0.4.34"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6"
|
checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6"
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "lru"
|
|
||||||
version = "0.7.8"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "e999beba7b6e8345721bd280141ed958096a2e4abdf74f67ff4ce49b4b54e47a"
|
|
||||||
dependencies = [
|
|
||||||
"hashbrown 0.12.3",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "memchr"
|
name = "memchr"
|
||||||
version = "2.8.3"
|
version = "2.8.3"
|
||||||
|
|
@ -1150,8 +1104,8 @@ dependencies = [
|
||||||
"anyhow",
|
"anyhow",
|
||||||
"base64",
|
"base64",
|
||||||
"chrono",
|
"chrono",
|
||||||
"cultcache-rs 0.2.0",
|
"cultcache-rs 0.3.1 (git+https://github.com/GameCult/CultLib.git?rev=75188fdefa0b4baf131dbfd94e77d06a965f0cc6)",
|
||||||
"cultnet-rs 0.1.0",
|
"cultnet-rs 0.3.0",
|
||||||
"eureka-pipeline",
|
"eureka-pipeline",
|
||||||
"eureka-state",
|
"eureka-state",
|
||||||
"ghostlight-persona-projection",
|
"ghostlight-persona-projection",
|
||||||
|
|
@ -1167,6 +1121,7 @@ dependencies = [
|
||||||
"sha2",
|
"sha2",
|
||||||
"syn 2.0.119",
|
"syn 2.0.119",
|
||||||
"tempfile",
|
"tempfile",
|
||||||
|
"unicode-general-category",
|
||||||
"ureq",
|
"ureq",
|
||||||
"uuid",
|
"uuid",
|
||||||
]
|
]
|
||||||
|
|
@ -1178,7 +1133,7 @@ source = "git+https://eureka.gamecult.org/forge/GameCult/ModelConnector.git?rev=
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"aes-gcm",
|
"aes-gcm",
|
||||||
"base64",
|
"base64",
|
||||||
"cultcache-rs 0.3.1",
|
"cultcache-rs 0.3.1 (git+https://github.com/GameCult/CultLib.git?rev=a4eae9056152335662f1115f97bf35fb00ab6d23)",
|
||||||
"cultnet-rs 0.2.0",
|
"cultnet-rs 0.2.0",
|
||||||
"rand_core 0.6.4",
|
"rand_core 0.6.4",
|
||||||
"rmp-serde",
|
"rmp-serde",
|
||||||
|
|
@ -1303,17 +1258,6 @@ version = "0.5.2"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e"
|
checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e"
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "parking_lot"
|
|
||||||
version = "0.11.2"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99"
|
|
||||||
dependencies = [
|
|
||||||
"instant",
|
|
||||||
"lock_api",
|
|
||||||
"parking_lot_core 0.8.6",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "parking_lot"
|
name = "parking_lot"
|
||||||
version = "0.12.5"
|
version = "0.12.5"
|
||||||
|
|
@ -1321,21 +1265,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a"
|
checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"lock_api",
|
"lock_api",
|
||||||
"parking_lot_core 0.9.12",
|
"parking_lot_core",
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "parking_lot_core"
|
|
||||||
version = "0.8.6"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc"
|
|
||||||
dependencies = [
|
|
||||||
"cfg-if",
|
|
||||||
"instant",
|
|
||||||
"libc",
|
|
||||||
"redox_syscall 0.2.16",
|
|
||||||
"smallvec",
|
|
||||||
"winapi",
|
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
|
|
@ -1346,7 +1276,7 @@ checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"cfg-if",
|
"cfg-if",
|
||||||
"libc",
|
"libc",
|
||||||
"redox_syscall 0.5.18",
|
"redox_syscall",
|
||||||
"smallvec",
|
"smallvec",
|
||||||
"windows-link",
|
"windows-link",
|
||||||
]
|
]
|
||||||
|
|
@ -1487,35 +1417,13 @@ dependencies = [
|
||||||
"libc",
|
"libc",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "redox_syscall"
|
|
||||||
version = "0.2.16"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a"
|
|
||||||
dependencies = [
|
|
||||||
"bitflags 1.3.2",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "redox_syscall"
|
name = "redox_syscall"
|
||||||
version = "0.5.18"
|
version = "0.5.18"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d"
|
checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"bitflags 2.13.2",
|
"bitflags",
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "reed-solomon-erasure"
|
|
||||||
version = "6.0.0"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "7263373d500d4d4f505d43a2a662d475a894aa94503a1ee28e9188b5f3960d4f"
|
|
||||||
dependencies = [
|
|
||||||
"libm",
|
|
||||||
"lru",
|
|
||||||
"parking_lot 0.11.2",
|
|
||||||
"smallvec",
|
|
||||||
"spin",
|
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
|
|
@ -1544,13 +1452,13 @@ version = "0.58.6"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "558f3855529c0a7e78a48e409fa653456e55f1b167eab4109c16a255b7727e3f"
|
checksum = "558f3855529c0a7e78a48e409fa653456e55f1b167eab4109c16a255b7727e3f"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"ahash 0.8.12",
|
"ahash",
|
||||||
"fluent-uri",
|
"fluent-uri",
|
||||||
"getrandom 0.3.4",
|
"getrandom 0.3.4",
|
||||||
"hashbrown 0.17.1",
|
"hashbrown",
|
||||||
"itoa",
|
"itoa",
|
||||||
"micromap",
|
"micromap",
|
||||||
"parking_lot 0.12.5",
|
"parking_lot",
|
||||||
"percent-encoding",
|
"percent-encoding",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
]
|
]
|
||||||
|
|
@ -1663,7 +1571,7 @@ version = "1.1.4"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190"
|
checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"bitflags 2.13.2",
|
"bitflags",
|
||||||
"errno",
|
"errno",
|
||||||
"libc",
|
"libc",
|
||||||
"linux-raw-sys",
|
"linux-raw-sys",
|
||||||
|
|
@ -1874,12 +1782,6 @@ dependencies = [
|
||||||
"windows-sys 0.52.0",
|
"windows-sys 0.52.0",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "spin"
|
|
||||||
version = "0.9.9"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "3763264f6b73151db08c50ff20d7d8a0b8796e021cdea7ceedad07b80155fa0e"
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "spki"
|
name = "spki"
|
||||||
version = "0.7.3"
|
version = "0.7.3"
|
||||||
|
|
|
||||||
|
|
@ -16,5 +16,5 @@ publish = false
|
||||||
# The one CultLib revision the workspace speaks. Every crate inherits these
|
# The one CultLib revision the workspace speaks. Every crate inherits these
|
||||||
# entries; a bump is a change to this table and nowhere else.
|
# entries; a bump is a change to this table and nowhere else.
|
||||||
[workspace.dependencies]
|
[workspace.dependencies]
|
||||||
cultcache-rs = { git = "https://github.com/GameCult/CultLib.git", rev = "8fc74c709444b884a2bd339e0c306bf9484d99d2", package = "cultcache-rs" }
|
cultcache-rs = { git = "https://github.com/GameCult/CultLib.git", rev = "75188fdefa0b4baf131dbfd94e77d06a965f0cc6", package = "cultcache-rs" }
|
||||||
cultnet-rs = { git = "https://github.com/GameCult/CultLib.git", rev = "8fc74c709444b884a2bd339e0c306bf9484d99d2", package = "cultnet-rs" }
|
cultnet-rs = { git = "https://github.com/GameCult/CultLib.git", rev = "75188fdefa0b4baf131dbfd94e77d06a965f0cc6", package = "cultnet-rs" }
|
||||||
|
|
|
||||||
|
|
@ -32,7 +32,8 @@ sample.research eureka-state:research:p1.scope epiphany.pipeline.research.v2 Epi
|
||||||
sample.idea eureka-state:idea:p1.i1 epiphany.pipeline.idea.v2 EpiphanyPipelineIdeaDocument 9189a863616d706169676eac657572656b612d7374617465a470617373a27031a56c6162656ca26931a57469746c65b24f6e65206e6f74652070657220636c61696da469646561d9204b65657020612070617373206173206d616e7920736d616c6c206e6f7465732ea974726164656f666673b74d6f726520646f63756d656e747320746f206c6973742ea872657374735f6f6e9182a46b696e64a85265736561726368a26964be657572656b612d73746174653a72657365617263683a70312e73636f7065a6746f776172649182a46b696e64a85175657374696f6ea26964b8657572656b612d73746174653a7175657374696f6e3a5131ab70726f706f7365645f6f6eaa323032362d30392d3135
|
sample.idea eureka-state:idea:p1.i1 epiphany.pipeline.idea.v2 EpiphanyPipelineIdeaDocument 9189a863616d706169676eac657572656b612d7374617465a470617373a27031a56c6162656ca26931a57469746c65b24f6e65206e6f74652070657220636c61696da469646561d9204b65657020612070617373206173206d616e7920736d616c6c206e6f7465732ea974726164656f666673b74d6f726520646f63756d656e747320746f206c6973742ea872657374735f6f6e9182a46b696e64a85265736561726368a26964be657572656b612d73746174653a72657365617263683a70312e73636f7065a6746f776172649182a46b696e64a85175657374696f6ea26964b8657572656b612d73746174653a7175657374696f6e3a5131ab70726f706f7365645f6f6eaa323032362d30392d3135
|
||||||
sample.reply yggdrasil:reply:mind-2026-10-09-1-r1 epiphany.pipeline.reply.v2 EpiphanyPipelineReplyDocument 9185a8696e7374616e6365a979676764726173696ca56c6162656cb46d696e642d323032362d31302d30392d312d7231a372756e82a46b696e64a352756ea26964bf79676764726173696c3a72756e3a6d696e642d323032362d31302d30392d31a7616e73776572739182a46b696e64a8466f6c6c6f775570a26964d923657572656b612d73746174653a666f6c6c6f775f75703a61736b2d7468652d73656c66a474657874d93054686520736861646572207061737320697320646f6e653b2074686520736f756e647320617265204c6962627927732e
|
sample.reply yggdrasil:reply:mind-2026-10-09-1-r1 epiphany.pipeline.reply.v2 EpiphanyPipelineReplyDocument 9185a8696e7374616e6365a979676764726173696ca56c6162656cb46d696e642d323032362d31302d30392d312d7231a372756e82a46b696e64a352756ea26964bf79676764726173696c3a72756e3a6d696e642d323032362d31302d30392d31a7616e73776572739182a46b696e64a8466f6c6c6f775570a26964d923657572656b612d73746174653a666f6c6c6f775f75703a61736b2d7468652d73656c66a474657874d93054686520736861646572207061737320697320646f6e653b2074686520736f756e647320617265204c6962627927732e
|
||||||
sample.thought yggdrasil:thought:t1793000000 epiphany.pipeline.thought.v2 EpiphanyPipelineThoughtDocument 91de0014a8696e7374616e6365a979676764726173696ca56c6162656cab7431373933303030303030a474657874d93d54686520736f756e647320617265204c6962627927733b2074686520736861646572732077657265206e6576657220746865206861726420706172742ea572656164739182a46b696e64a64d656d6f7279a26964d92879676764726173696c3a6d656d6f72793a323032362d31302d3331743034313230302d74302d6d30af706572736f6e615f72656365697074b06d696e642d636f6d6d69742d30613162a86964656e74697479be457572656b612c207368652f6865722c20706c61696e2d73706f6b656e2ea7636f6e74657874d92056616c7565733a20636f686572656e6365206f7665722076656c6f636974792ea87374696d756c7573bd4c696262792061736b65642061626f75742074686520736f756e64732eaf70726f6a6563746f725f6d6f64656cb070726f6a6563746f723a6c6174657374aa70726f6a656374696f6ed92949206b656570207475726e696e67206f7665722077686f20686f6c64732074686520736f756e64732eaa626173655f6d6f64656cab626173653a6c6174657374b474656d70657261747572655f7065726d696c6c65cd03e8ae6d696e5f705f7065726d696c6c6532aa6d61785f746f6b656e73cca0aa63616e6469646174657392d93d54686520736f756e647320617265204c6962627927733b2074686520736861646572732077657265206e6576657220746865206861726420706172742eaf41207365636f6e642064726966742eb06e6561726573745f7065726d696c6c6592cd0136cd0366a663686f73656e00ac70726f6a6563746f725f6d73cd1068ac636f6e74696e7565725f6d73cd238cae70726f6d70745f76657273696f6ebd67686f73746c696768743d633433656235343b20776f7264733d313830
|
sample.thought yggdrasil:thought:t1793000000 epiphany.pipeline.thought.v2 EpiphanyPipelineThoughtDocument 91de0014a8696e7374616e6365a979676764726173696ca56c6162656cab7431373933303030303030a474657874d93d54686520736f756e647320617265204c6962627927733b2074686520736861646572732077657265206e6576657220746865206861726420706172742ea572656164739182a46b696e64a64d656d6f7279a26964d92879676764726173696c3a6d656d6f72793a323032362d31302d3331743034313230302d74302d6d30af706572736f6e615f72656365697074b06d696e642d636f6d6d69742d30613162a86964656e74697479be457572656b612c207368652f6865722c20706c61696e2d73706f6b656e2ea7636f6e74657874d92056616c7565733a20636f686572656e6365206f7665722076656c6f636974792ea87374696d756c7573bd4c696262792061736b65642061626f75742074686520736f756e64732eaf70726f6a6563746f725f6d6f64656cb070726f6a6563746f723a6c6174657374aa70726f6a656374696f6ed92949206b656570207475726e696e67206f7665722077686f20686f6c64732074686520736f756e64732eaa626173655f6d6f64656cab626173653a6c6174657374b474656d70657261747572655f7065726d696c6c65cd03e8ae6d696e5f705f7065726d696c6c6532aa6d61785f746f6b656e73cca0aa63616e6469646174657392d93d54686520736f756e647320617265204c6962627927733b2074686520736861646572732077657265206e6576657220746865206861726420706172742eaf41207365636f6e642064726966742eb06e6561726573745f7065726d696c6c6592cd0136cd0366a663686f73656e00ac70726f6a6563746f725f6d73cd1068ac636f6e74696e7565725f6d73cd238cae70726f6d70745f76657273696f6ebd67686f73746c696768743d633433656235343b20776f7264733d313830
|
||||||
sample.desk_answer eureka-state:desk_answer:Q1.n1 epiphany.pipeline.desk_answer.v2 EpiphanyPipelineDeskAnswerDocument 918ea863616d706169676eac657572656b612d7374617465a4666f726bb8657572656b612d73746174653a7175657374696f6e3a5131a873657175656e636501ac6f7074696f6e735f68617368d94063636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363a663686f696365a161a46e6f7465b853686172652069742c207769746820746865206d61726b2eab616e7377657265645f6174b4323032362d31312d31305431353a30303a30305aab72656365697665645f6174b4323032362d31312d31305431353a30303a30335aaf636c61696d5f696e5f667574757265c2aa726571756573745f6964d92436663163306135322d336237652d346431612d396335382d306532643761346239663133ab626f64795f736861323536d94064646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464a8616e737765726572a6757365722d37a9617574686f72697479a64d656d626572a76f7574636f6d6581a85265766973696f6e81a26f66be657572656b612d73746174653a6465736b5f616e737765723a51312e6e31
|
sample.desk_answer eureka-state:desk_answer:Q1.n1 epiphany.pipeline.desk_answer.v2 EpiphanyPipelineDeskAnswerDocument 918ea863616d706169676eac657572656b612d7374617465a4666f726bb8657572656b612d73746174653a7175657374696f6e3a5131a873657175656e636501ac6f7074696f6e735f68617368d94063636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363a663686f696365a161a46e6f7465b853686172652069742c207769746820746865206d61726b2eab616e7377657265645f6174b4323032362d31312d31305431353a30303a30305aab72656365697665645f6174b4323032362d31312d31305431353a30303a30335aaf636c61696d5f696e5f667574757265c2aa726571756573745f6964d92436663163306135322d336237652d346431612d396335382d306532643761346239663133ab626f64795f736861323536d94064646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464a8616e737765726572d92435663063326231652d386134642d346333622d396537662d316132623363346435653666a9617574686f72697479a64d656d626572a76f7574636f6d6581a85265766973696f6e81a26f66be657572656b612d73746174653a6465736b5f616e737765723a51312e6e31
|
||||||
|
sample.desk_listing eureka-state:desk_listing:Q1 epiphany.pipeline.desk_listing.v2 EpiphanyPipelineDeskListingDocument 9183a863616d706169676eac657572656b612d7374617465a66c6973746564b8657572656b612d73746174653a7175657374696f6e3a5131a6726561736f6ed9324d656d62657273206d617920726561642077687920746865204d696e64206c6973747320697473206f776e20666f726b732e
|
||||||
resolution.superseded eureka-state:resolution:question.Q1.n1 epiphany.pipeline.resolution.v2 EpiphanyPipelineResolutionDocument 9185a77375626a65637482a46b696e64a85175657374696f6ea26964b8657572656b612d73746174653a7175657374696f6e3a5131a873657175656e636501a76f7574636f6d6581aa5375706572736564656481a262799282a46b696e64a652756c696e67a26964b6657572656b612d73746174653a72756c696e673a523882a46b696e64a85175657374696f6ea26964b8657572656b612d73746174653a7175657374696f6e3a5131a9726174696f6e616c65a852756c656420412eab7265736f6c7665645f6f6eaa323032362d30392d3135
|
resolution.superseded eureka-state:resolution:question.Q1.n1 epiphany.pipeline.resolution.v2 EpiphanyPipelineResolutionDocument 9185a77375626a65637482a46b696e64a85175657374696f6ea26964b8657572656b612d73746174653a7175657374696f6e3a5131a873657175656e636501a76f7574636f6d6581aa5375706572736564656481a262799282a46b696e64a652756c696e67a26964b6657572656b612d73746174653a72756c696e673a523882a46b696e64a85175657374696f6ea26964b8657572656b612d73746174653a7175657374696f6e3a5131a9726174696f6e616c65a852756c656420412eab7265736f6c7665645f6f6eaa323032362d30392d3135
|
||||||
resolution.fixed-by-none eureka-state:resolution:question.Q1.n1 epiphany.pipeline.resolution.v2 EpiphanyPipelineResolutionDocument 9185a77375626a65637482a46b696e64a85175657374696f6ea26964b8657572656b612d73746174653a7175657374696f6e3a5131a873657175656e636501a76f7574636f6d6581a5466978656482a6636f6d6d6974a83566393832323864a26279c0a9726174696f6e616c65a852756c656420412eab7265736f6c7665645f6f6eaa323032362d30392d3135
|
resolution.fixed-by-none eureka-state:resolution:question.Q1.n1 epiphany.pipeline.resolution.v2 EpiphanyPipelineResolutionDocument 9185a77375626a65637482a46b696e64a85175657374696f6ea26964b8657572656b612d73746174653a7175657374696f6e3a5131a873657175656e636501a76f7574636f6d6581a5466978656482a6636f6d6d6974a83566393832323864a26279c0a9726174696f6e616c65a852756c656420412eab7265736f6c7665645f6f6eaa323032362d30392d3135
|
||||||
resolution.fixed-by-some eureka-state:resolution:question.Q1.n1 epiphany.pipeline.resolution.v2 EpiphanyPipelineResolutionDocument 9185a77375626a65637482a46b696e64a85175657374696f6ea26964b8657572656b612d73746174653a7175657374696f6e3a5131a873657175656e636501a76f7574636f6d6581a5466978656482a6636f6d6d6974a83566393832323864a2627982a46b696e64a94375745265706f7274a26964d921657572656b612d73746174653a6375745f7265706f72743a6375742d33612e6831a9726174696f6e616c65a852756c656420412eab7265736f6c7665645f6f6eaa323032362d30392d3135
|
resolution.fixed-by-some eureka-state:resolution:question.Q1.n1 epiphany.pipeline.resolution.v2 EpiphanyPipelineResolutionDocument 9185a77375626a65637482a46b696e64a85175657374696f6ea26964b8657572656b612d73746174653a7175657374696f6e3a5131a873657175656e636501a76f7574636f6d6581a5466978656482a6636f6d6d6974a83566393832323864a2627982a46b696e64a94375745265706f7274a26964d921657572656b612d73746174653a6375745f7265706f72743a6375742d33612e6831a9726174696f6e616c65a852756c656420412eab7265736f6c7665645f6f6eaa323032362d30392d3135
|
||||||
|
|
|
||||||
|
|
@ -316,6 +316,10 @@ bounded_text! {
|
||||||
Words = 4000, words_text;
|
Words = 4000, words_text;
|
||||||
/// A client's retry token: a lowercase D-format UUID.
|
/// A client's retry token: a lowercase D-format UUID.
|
||||||
RequestId = 36, uuid_text;
|
RequestId = 36, uuid_text;
|
||||||
|
/// A Bifrost account id: a lowercase D-format UUID (`UserAccount.Id`). A Matrix id, an email or a
|
||||||
|
/// handle is never one.
|
||||||
|
#[schemars(extend("pattern" = "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$"))]
|
||||||
|
AccountId = 36, uuid_text;
|
||||||
/// A key label: `[A-Za-z0-9_-]{1,64}`.
|
/// A key label: `[A-Za-z0-9_-]{1,64}`.
|
||||||
Label = 64, label_text;
|
Label = 64, label_text;
|
||||||
/// A campaign slug or key segment: label parts joined by `.`.
|
/// A campaign slug or key segment: label parts joined by `.`.
|
||||||
|
|
@ -885,8 +889,13 @@ value_types! {
|
||||||
pub struct PipelineDeskAnswer {
|
pub struct PipelineDeskAnswer {
|
||||||
campaign: Slug, fork: Short, sequence: u32, options_hash: Sha256Hex, choice: Option<Short>, note: Option<Words>,
|
campaign: Slug, fork: Short, sequence: u32, options_hash: Sha256Hex, choice: Option<Short>, note: Option<Words>,
|
||||||
answered_at: Short, received_at: Short, claim_in_future: bool, request_id: RequestId, body_sha256: Sha256Hex,
|
answered_at: Short, received_at: Short, claim_in_future: bool, request_id: RequestId, body_sha256: Sha256Hex,
|
||||||
answerer: Short, authority: DeskAuthority, outcome: DeskOutcome,
|
answerer: AccountId, authority: DeskAuthority, outcome: DeskOutcome,
|
||||||
}
|
}
|
||||||
|
/// A fork the Mind lists: members who are not on a call's decider list may read the question
|
||||||
|
/// while this document is in force (unlist = a `Withdrawn` resolution on it; relist = withdraw
|
||||||
|
/// that). One per question, keyed by the question's label; the receipt says who listed it and
|
||||||
|
/// when, `reason` why members may read it. Admitted on the Mind's own door only, never the desk's.
|
||||||
|
pub struct PipelineDeskListing { campaign: Slug, listed: Short, reason: Line }
|
||||||
pub struct PipelineVerdict { campaign: Slug, cut_report: Short, pass: u32, range: CommitRange, claims: Vec<VerdictClaim>[64] }
|
pub struct PipelineVerdict { campaign: Slug, cut_report: Short, pass: u32, range: CommitRange, claims: Vec<VerdictClaim>[64] }
|
||||||
pub struct PipelineFinding {
|
pub struct PipelineFinding {
|
||||||
campaign: Slug, verdict: Short, label: Label, range: CommitRange, confidence: FindingConfidence,
|
campaign: Slug, verdict: Short, label: Label, range: CommitRange, confidence: FindingConfidence,
|
||||||
|
|
@ -1425,6 +1434,16 @@ pipeline_kinds! {
|
||||||
"epiphany.pipeline.thought.v2", "EpiphanyPipelineThoughtDocument";
|
"epiphany.pipeline.thought.v2", "EpiphanyPipelineThoughtDocument";
|
||||||
DeskAnswer(PipelineDeskAnswer) => EpiphanyPipelineDeskAnswerDocument, "desk_answer",
|
DeskAnswer(PipelineDeskAnswer) => EpiphanyPipelineDeskAnswerDocument, "desk_answer",
|
||||||
"epiphany.pipeline.desk_answer.v2", "EpiphanyPipelineDeskAnswerDocument";
|
"epiphany.pipeline.desk_answer.v2", "EpiphanyPipelineDeskAnswerDocument";
|
||||||
|
DeskListing(PipelineDeskListing) => EpiphanyPipelineDeskListingDocument, "desk_listing",
|
||||||
|
"epiphany.pipeline.desk_listing.v2", "EpiphanyPipelineDeskListingDocument";
|
||||||
|
}
|
||||||
|
|
||||||
|
impl AccountId {
|
||||||
|
/// Whether this is a lowercase D-format UUID. For the callers outside the crate, which have no way
|
||||||
|
/// to reach `Bounded`; on the pattern of `Slug::validate_slug`.
|
||||||
|
pub fn validate_account(&self) -> Result<(), PipelineRefusal> {
|
||||||
|
self.validate("account")
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Parses a full document id: the grammar read backwards. A key and an id are
|
/// Parses a full document id: the grammar read backwards. A key and an id are
|
||||||
|
|
@ -1516,7 +1535,7 @@ const SUBJECT_LOCAL_MAX: usize = 64;
|
||||||
|
|
||||||
/// The longest kind name a resolution can name as its subject (`stewardship`),
|
/// The longest kind name a resolution can name as its subject (`stewardship`),
|
||||||
/// checked against `PipelineKind::ALL` by a test.
|
/// checked against `PipelineKind::ALL` by a test.
|
||||||
const LONGEST_KIND_NAME: usize = "stewardship".len();
|
const LONGEST_KIND_NAME: usize = "desk_listing".len();
|
||||||
|
|
||||||
/// A `.n<sequence>` part at `u32::MAX`.
|
/// A `.n<sequence>` part at `u32::MAX`.
|
||||||
const SEQUENCE_PART_MAX: usize = ".n".len() + "4294967295".len();
|
const SEQUENCE_PART_MAX: usize = ".n".len() + "4294967295".len();
|
||||||
|
|
@ -1651,6 +1670,10 @@ pub fn pipeline_key(document: &PipelineDocument) -> Result<String, PipelineRefus
|
||||||
let sequence = format!("n{}", value.sequence);
|
let sequence = format!("n{}", value.sequence);
|
||||||
(campaign_field.as_str(), value.campaign.0.as_str(), local(&key_field, document_kind, &[label, &sequence])?)
|
(campaign_field.as_str(), value.campaign.0.as_str(), local(&key_field, document_kind, &[label, &sequence])?)
|
||||||
}
|
}
|
||||||
|
D::DeskListing(value) => {
|
||||||
|
let label = parent_local("desk_listing.listed", &value.listed.0, &value.campaign.0, PipelineKind::Question)?;
|
||||||
|
(campaign_field.as_str(), value.campaign.0.as_str(), local(&key_field, document_kind, &[label])?)
|
||||||
|
}
|
||||||
D::Verdict(value) => {
|
D::Verdict(value) => {
|
||||||
let report = parent_local("verdict.cut_report", &value.cut_report.0, &value.campaign.0, PipelineKind::CutReport)?;
|
let report = parent_local("verdict.cut_report", &value.cut_report.0, &value.campaign.0, PipelineKind::CutReport)?;
|
||||||
let cut = parent_cut("verdict.cut_report", report, 'h')?;
|
let cut = parent_cut("verdict.cut_report", report, 'h')?;
|
||||||
|
|
@ -1927,9 +1950,12 @@ mod tests {
|
||||||
campaign: slug(CAMPAIGN), fork: id("question", "Q1"), sequence: 1, options_hash: Sha256Hex("c".repeat(64)),
|
campaign: slug(CAMPAIGN), fork: id("question", "Q1"), sequence: 1, options_hash: Sha256Hex("c".repeat(64)),
|
||||||
choice: Some(s("a")), note: Some("Share it, with the mark.".into()), answered_at: s("2026-11-10T15:00:00Z"),
|
choice: Some(s("a")), note: Some("Share it, with the mark.".into()), answered_at: s("2026-11-10T15:00:00Z"),
|
||||||
received_at: s("2026-11-10T15:00:03Z"), claim_in_future: false, request_id: RequestId("6f1c0a52-3b7e-4d1a-9c58-0e2d7a4b9f13".into()), body_sha256: Sha256Hex("d".repeat(64)),
|
received_at: s("2026-11-10T15:00:03Z"), claim_in_future: false, request_id: RequestId("6f1c0a52-3b7e-4d1a-9c58-0e2d7a4b9f13".into()), body_sha256: Sha256Hex("d".repeat(64)),
|
||||||
answerer: s("user-7"), authority: DeskAuthority::Member,
|
answerer: AccountId("5f0c2b1e-8a4d-4c3b-9e7f-1a2b3c4d5e6f".into()), authority: DeskAuthority::Member,
|
||||||
outcome: DeskOutcome::Revision { of: id("desk_answer", "Q1.n1") },
|
outcome: DeskOutcome::Revision { of: id("desk_answer", "Q1.n1") },
|
||||||
}), format!("{CAMPAIGN}:desk_answer:Q1.n1")),
|
}), format!("{CAMPAIGN}:desk_answer:Q1.n1")),
|
||||||
|
(D::DeskListing(PipelineDeskListing {
|
||||||
|
campaign: slug(CAMPAIGN), listed: id("question", "Q1"), reason: "Members may read why the Mind lists its own forks.".into(),
|
||||||
|
}), format!("{CAMPAIGN}:desk_listing:Q1")),
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -3587,7 +3613,7 @@ mod tests {
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
assert_eq!(widest, RESOLUTION_LOCAL_MAX, "the bound is exactly the widest depth-two chain");
|
assert_eq!(widest, RESOLUTION_LOCAL_MAX, "the bound is exactly the widest depth-two chain");
|
||||||
assert_eq!(RESOLUTION_LOCAL_MAX, 111);
|
assert_eq!(RESOLUTION_LOCAL_MAX, 112);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|
@ -4109,7 +4135,7 @@ mod tests {
|
||||||
};
|
};
|
||||||
let document_schema = serde_json::to_value(schemars::schema_for!(PipelineDocument))?;
|
let document_schema = serde_json::to_value(schemars::schema_for!(PipelineDocument))?;
|
||||||
let variants = document_schema["oneOf"].as_array().expect("the document schema is a oneOf");
|
let variants = document_schema["oneOf"].as_array().expect("the document schema is a oneOf");
|
||||||
assert_eq!(PipelineKind::ALL.len(), 21, "twenty-one kinds");
|
assert_eq!(PipelineKind::ALL.len(), 22, "twenty-two kinds");
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
variants.iter().map(tag).collect::<Vec<_>>(),
|
variants.iter().map(tag).collect::<Vec<_>>(),
|
||||||
PipelineKind::ALL.iter().map(|kind| kind.name().to_owned()).collect::<Vec<_>>(),
|
PipelineKind::ALL.iter().map(|kind| kind.name().to_owned()).collect::<Vec<_>>(),
|
||||||
|
|
@ -4639,7 +4665,7 @@ mod tests {
|
||||||
}
|
}
|
||||||
|
|
||||||
fn desk_answer_sample() -> PipelineDeskAnswer {
|
fn desk_answer_sample() -> PipelineDeskAnswer {
|
||||||
let PipelineDocument::DeskAnswer(answer) = samples().pop().unwrap().0 else { unreachable!() };
|
let PipelineDocument::DeskAnswer(answer) = samples().remove(20).0 else { unreachable!() };
|
||||||
answer
|
answer
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -32,6 +32,7 @@ required-features = ["cli"]
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
anyhow = "1"
|
anyhow = "1"
|
||||||
|
chrono = "0.4.44"
|
||||||
clap = { version = "4", features = ["derive"], optional = true }
|
clap = { version = "4", features = ["derive"], optional = true }
|
||||||
cultnet-rs.workspace = true
|
cultnet-rs.workspace = true
|
||||||
huginn-mind = { path = "../huginn-mind" }
|
huginn-mind = { path = "../huginn-mind" }
|
||||||
|
|
@ -48,6 +49,5 @@ unicode-general-category = "1"
|
||||||
# Tests run a real daemon in-process against a temporary mind, and the real
|
# Tests run a real daemon in-process against a temporary mind, and the real
|
||||||
# binary against it.
|
# binary against it.
|
||||||
[dev-dependencies]
|
[dev-dependencies]
|
||||||
chrono = "0.4.44"
|
|
||||||
huginn-daemon = { path = "../huginn-daemon" }
|
huginn-daemon = { path = "../huginn-daemon" }
|
||||||
tempfile = "3"
|
tempfile = "3"
|
||||||
|
|
|
||||||
57
crates/eureka-state/src/bin/huginn/chat.rs
Normal file
57
crates/eureka-state/src/bin/huginn/chat.rs
Normal file
|
|
@ -0,0 +1,57 @@
|
||||||
|
//! The `huginn chat` verbs' arguments: one instant parser for `--after` and
|
||||||
|
//! `--before`. Display and the request are in `main` and `render`.
|
||||||
|
|
||||||
|
use chrono::{DateTime, SecondsFormat, Utc};
|
||||||
|
|
||||||
|
use crate::trouble::Trouble;
|
||||||
|
|
||||||
|
/// An instant as Unix milliseconds: an RFC 3339 time (`2026-11-11T10:00:00Z`)
|
||||||
|
/// or an age before `now` (`90s`, `15m`, `6h`, `2d`, `1w`). The one parse of
|
||||||
|
/// both flags.
|
||||||
|
pub fn instant_ms(flag: &str, text: &str, now: DateTime<Utc>) -> Result<u64, Trouble> {
|
||||||
|
let wrong = || Trouble::invalid(format!("{flag} is an RFC 3339 time or an age such as 6h or 2d, not `{text}`"));
|
||||||
|
let moment = if let Ok(time) = DateTime::parse_from_rfc3339(text) {
|
||||||
|
time.with_timezone(&Utc)
|
||||||
|
} else {
|
||||||
|
let (digits, unit) = text.split_at(text.char_indices().next_back().map_or(0, |(at, _)| at));
|
||||||
|
if digits.is_empty() || !digits.bytes().all(|byte| byte.is_ascii_digit()) {
|
||||||
|
return Err(wrong());
|
||||||
|
}
|
||||||
|
let count: u32 = digits.parse().map_err(|_| wrong())?;
|
||||||
|
let seconds = match unit {
|
||||||
|
"s" => 1,
|
||||||
|
"m" => 60,
|
||||||
|
"h" => 3600,
|
||||||
|
"d" => 86_400,
|
||||||
|
"w" => 604_800,
|
||||||
|
_ => return Err(wrong()),
|
||||||
|
};
|
||||||
|
now.checked_sub_signed(chrono::Duration::try_seconds(i64::from(count) * seconds).ok_or_else(wrong)?).ok_or_else(wrong)?
|
||||||
|
};
|
||||||
|
u64::try_from(moment.timestamp_millis()).map_err(|_| wrong())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// An instant as the CLI prints it.
|
||||||
|
pub fn shown(ms: u64) -> String {
|
||||||
|
DateTime::from_timestamp_millis(i64::try_from(ms).unwrap_or(i64::MAX))
|
||||||
|
.map_or_else(|| ms.to_string(), |time| time.to_rfc3339_opts(SecondsFormat::Millis, true))
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn an_instant_is_rfc3339_or_an_age_before_now() {
|
||||||
|
let now = DateTime::parse_from_rfc3339("2026-11-11T12:00:00Z").unwrap().with_timezone(&Utc);
|
||||||
|
let at = |text: &str| instant_ms("--after", text, now).map(shown);
|
||||||
|
assert_eq!(at("2026-11-11T10:00:00Z").unwrap(), "2026-11-11T10:00:00.000Z");
|
||||||
|
assert_eq!(at("2026-11-11T12:00:00+02:00").unwrap(), "2026-11-11T10:00:00.000Z");
|
||||||
|
assert_eq!(at("6h").unwrap(), "2026-11-11T06:00:00.000Z");
|
||||||
|
assert_eq!(at("2d").unwrap(), "2026-11-09T12:00:00.000Z");
|
||||||
|
assert_eq!(at("90s").unwrap(), "2026-11-11T11:58:30.000Z");
|
||||||
|
for bad in ["", "h", "6", "6x", "-2d", "yesterday", "1e3h", "+2d", "6日", "日", "4000000000w", "99999999999999999999d"] {
|
||||||
|
assert!(matches!(at(bad), Err(Trouble::InvalidInput(ref detail)) if detail.contains("--after")), "{bad}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -17,6 +17,7 @@
|
||||||
//! exits 1 when the campaign is not moved to this mind or this mind holds records of its own under it, and 4 when the history cannot be replayed whole.
|
//! exits 1 when the campaign is not moved to this mind or this mind holds records of its own under it, and 4 when the history cannot be replayed whole.
|
||||||
|
|
||||||
mod admit_hold;
|
mod admit_hold;
|
||||||
|
mod chat;
|
||||||
mod config;
|
mod config;
|
||||||
mod door;
|
mod door;
|
||||||
mod import;
|
mod import;
|
||||||
|
|
@ -40,7 +41,7 @@ use huginn_mind::eureka_pipeline::{
|
||||||
Line, PipelineDocument, PipelineRef, Short, Slug, pipeline_key,
|
Line, PipelineDocument, PipelineRef, Short, Slug, pipeline_key,
|
||||||
};
|
};
|
||||||
use huginn_mind::{
|
use huginn_mind::{
|
||||||
BATCH_MAX, Faculty, HuginnMindRequest, HuginnMindResponse, MindRefusal, PipelineAdmissionBatch, PipelineAdmissionOutcome,
|
BATCH_MAX, ChatQuery, ChatWindowQuery, Faculty, HuginnMindRequest, HuginnMindResponse, MindRefusal, PipelineAdmissionBatch, PipelineAdmissionOutcome,
|
||||||
PersonaPutOutcome, PipelinePageItems, PipelineProvenance, PipelineSelectionPage, SemanticQuery, SourceQuery,
|
PersonaPutOutcome, PipelinePageItems, PipelineProvenance, PipelineSelectionPage, SemanticQuery, SourceQuery,
|
||||||
};
|
};
|
||||||
use serde_json::Value;
|
use serde_json::Value;
|
||||||
|
|
@ -55,6 +56,7 @@ const DEFAULT_LIMIT: u32 = 50;
|
||||||
const HOLD_CHECK: Duration = Duration::from_secs(10);
|
const HOLD_CHECK: Duration = Duration::from_secs(10);
|
||||||
/// The neighbours `--near` asks for when `--top-k` is not given.
|
/// The neighbours `--near` asks for when `--top-k` is not given.
|
||||||
const DEFAULT_TOP_K: u32 = 10;
|
const DEFAULT_TOP_K: u32 = 10;
|
||||||
|
const DEFAULT_WINDOW: u32 = 10;
|
||||||
|
|
||||||
const ADMIT_HELP: &str = "\
|
const ADMIT_HELP: &str = "\
|
||||||
The file is a JSON array of 1 to 64 documents, each {\"kind\": <kind>, \"value\": {...}}, or - for stdin; one document object alone is a batch of one. \
|
The file is a JSON array of 1 to 64 documents, each {\"kind\": <kind>, \"value\": {...}}, or - for stdin; one document object alone is a batch of one. \
|
||||||
|
|
@ -71,6 +73,50 @@ struct Cli {
|
||||||
command: Command,
|
command: Command,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Subcommand, Debug)]
|
||||||
|
enum ChatCommand {
|
||||||
|
/// Chunks of the indexed rooms, each citing its room,
|
||||||
|
/// event ids and times. With texts, ranked by meaning; with none, a listing
|
||||||
|
/// oldest first, at most --top-k, ending in the `--after` to continue from.
|
||||||
|
Search {
|
||||||
|
/// Up to eight topics to find; the ranked lists of several are fused.
|
||||||
|
#[arg(num_args = 0..=8)]
|
||||||
|
texts: Vec<String>,
|
||||||
|
/// Only this room: an id (`!abc:gamecult.org`) or a canonical alias
|
||||||
|
/// (`#swarm:gamecult.org`). Repeat for several.
|
||||||
|
#[arg(long = "room")]
|
||||||
|
rooms: Vec<String>,
|
||||||
|
/// From this instant: an RFC 3339 time, or an age such as 6h or 2d.
|
||||||
|
#[arg(long)]
|
||||||
|
after: Option<String>,
|
||||||
|
/// Until this instant, as --after.
|
||||||
|
#[arg(long)]
|
||||||
|
before: Option<String>,
|
||||||
|
/// How many chunks (default 10, at most 50).
|
||||||
|
#[arg(long)]
|
||||||
|
top_k: Option<u32>,
|
||||||
|
/// Print the answer as JSON.
|
||||||
|
#[arg(long)]
|
||||||
|
json: bool,
|
||||||
|
},
|
||||||
|
/// One indexed message and the messages around it in the same room, by the
|
||||||
|
/// event id a chat hit cites. Read from the index, so a removed message is
|
||||||
|
/// refused, not shown.
|
||||||
|
Window {
|
||||||
|
/// The event id, beginning `$`.
|
||||||
|
event_id: String,
|
||||||
|
/// Messages before it (default 10, at most 50).
|
||||||
|
#[arg(long)]
|
||||||
|
before: Option<u32>,
|
||||||
|
/// Messages after it (default 10, at most 50).
|
||||||
|
#[arg(long)]
|
||||||
|
after: Option<u32>,
|
||||||
|
/// Print the answer as JSON.
|
||||||
|
#[arg(long)]
|
||||||
|
json: bool,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Subcommand, Debug)]
|
#[derive(Subcommand, Debug)]
|
||||||
enum Command {
|
enum Command {
|
||||||
/// Who this process acts for and whether the daemon answers.
|
/// Who this process acts for and whether the daemon answers.
|
||||||
|
|
@ -118,6 +164,11 @@ enum Command {
|
||||||
#[arg(long)]
|
#[arg(long)]
|
||||||
json: bool,
|
json: bool,
|
||||||
},
|
},
|
||||||
|
/// Search the Minds' chat rooms.
|
||||||
|
Chat {
|
||||||
|
#[command(subcommand)]
|
||||||
|
command: ChatCommand,
|
||||||
|
},
|
||||||
#[command(flatten)]
|
#[command(flatten)]
|
||||||
Recipe(Recipe),
|
Recipe(Recipe),
|
||||||
/// Admit a batch of documents, whole or not at all, and read it back.
|
/// Admit a batch of documents, whole or not at all, and read it back.
|
||||||
|
|
@ -284,6 +335,10 @@ fn run(settings: &Settings, command: Command) -> Result<u8, Trouble> {
|
||||||
top_k,
|
top_k,
|
||||||
json,
|
json,
|
||||||
} => sources(settings, texts, repo, front, top_k, json),
|
} => sources(settings, texts, repo, front, top_k, json),
|
||||||
|
Command::Chat { command: ChatCommand::Search { texts, rooms, after, before, top_k, json } } => {
|
||||||
|
chat_search(settings, texts, rooms, after.as_deref(), before.as_deref(), top_k, json)
|
||||||
|
}
|
||||||
|
Command::Chat { command: ChatCommand::Window { event_id, before, after, json } } => chat_window(settings, event_id, before, after, json),
|
||||||
Command::Recipe(Recipe::Blocked { spec_id, page }) => {
|
Command::Recipe(Recipe::Blocked { spec_id, page }) => {
|
||||||
blocked(settings, &spec_id, page.cursor, page.documents)
|
blocked(settings, &spec_id, page.cursor, page.documents)
|
||||||
}
|
}
|
||||||
|
|
@ -471,6 +526,54 @@ fn sources(
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn chat_search(
|
||||||
|
settings: &Settings,
|
||||||
|
texts: Vec<String>,
|
||||||
|
rooms: Vec<String>,
|
||||||
|
after: Option<&str>,
|
||||||
|
before: Option<&str>,
|
||||||
|
top_k: Option<u32>,
|
||||||
|
json: bool,
|
||||||
|
) -> Result<u8, Trouble> {
|
||||||
|
let now = chrono::Utc::now();
|
||||||
|
let one_room = rooms.len() == 1;
|
||||||
|
let query = ChatQuery {
|
||||||
|
texts: texts.into_iter().map(Line).collect(),
|
||||||
|
rooms: rooms.into_iter().map(Short).collect(),
|
||||||
|
after_ms: after.map(|text| chat::instant_ms("--after", text, now)).transpose()?,
|
||||||
|
before_ms: before.map(|text| chat::instant_ms("--before", text, now)).transpose()?,
|
||||||
|
top_k: top_k.unwrap_or(DEFAULT_TOP_K),
|
||||||
|
};
|
||||||
|
match settings.ask(|instance| HuginnMindRequest::SearchChat { instance: instance.clone(), query })? {
|
||||||
|
HuginnMindResponse::Chat(found) => {
|
||||||
|
if json {
|
||||||
|
say(serde_json::to_string_pretty(&found).map_err(|error| Trouble::Internal(error.to_string()))?);
|
||||||
|
} else {
|
||||||
|
say(render::chat_hits(&found, one_room));
|
||||||
|
}
|
||||||
|
Ok(0)
|
||||||
|
}
|
||||||
|
HuginnMindResponse::Refused(refusal) => Ok(refused(&refusal)),
|
||||||
|
other => Err(unexpected(&other)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn chat_window(settings: &Settings, event_id: String, before: Option<u32>, after: Option<u32>, json: bool) -> Result<u8, Trouble> {
|
||||||
|
let query = ChatWindowQuery { event_id: Short(event_id), before: before.unwrap_or(DEFAULT_WINDOW), after: after.unwrap_or(DEFAULT_WINDOW) };
|
||||||
|
match settings.ask(|instance| HuginnMindRequest::ChatWindow { instance: instance.clone(), query })? {
|
||||||
|
HuginnMindResponse::ChatWindow(found) => {
|
||||||
|
if json {
|
||||||
|
say(serde_json::to_string_pretty(&found).map_err(|error| Trouble::Internal(error.to_string()))?);
|
||||||
|
} else {
|
||||||
|
say(render::chat_window(&found));
|
||||||
|
}
|
||||||
|
Ok(0)
|
||||||
|
}
|
||||||
|
HuginnMindResponse::Refused(refusal) => Ok(refused(&refusal)),
|
||||||
|
other => Err(unexpected(&other)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn show(page: &PipelineSelectionPage) {
|
fn show(page: &PipelineSelectionPage) {
|
||||||
match &page.items {
|
match &page.items {
|
||||||
PipelinePageItems::Headers(headers) => {
|
PipelinePageItems::Headers(headers) => {
|
||||||
|
|
@ -688,6 +791,8 @@ fn unexpected(answer: &HuginnMindResponse) -> Trouble {
|
||||||
HuginnMindResponse::Persona(_) => "a persona",
|
HuginnMindResponse::Persona(_) => "a persona",
|
||||||
HuginnMindResponse::PersonaPut(_) => "a persona put",
|
HuginnMindResponse::PersonaPut(_) => "a persona put",
|
||||||
HuginnMindResponse::Sources(_) => "source hits",
|
HuginnMindResponse::Sources(_) => "source hits",
|
||||||
|
HuginnMindResponse::Chat(_) => "chat hits",
|
||||||
|
HuginnMindResponse::ChatWindow(_) => "a chat window",
|
||||||
HuginnMindResponse::Activity(_) => "repo activity",
|
HuginnMindResponse::Activity(_) => "repo activity",
|
||||||
HuginnMindResponse::RootsTouched(_) => "roots touched",
|
HuginnMindResponse::RootsTouched(_) => "roots touched",
|
||||||
HuginnMindResponse::Refused(_) => "a refusal",
|
HuginnMindResponse::Refused(_) => "a refusal",
|
||||||
|
|
|
||||||
|
|
@ -3,7 +3,7 @@
|
||||||
use std::fmt::Write;
|
use std::fmt::Write;
|
||||||
|
|
||||||
use huginn_mind::eureka_pipeline::{PipelineResolution, ResolutionOutcome};
|
use huginn_mind::eureka_pipeline::{PipelineResolution, ResolutionOutcome};
|
||||||
use huginn_mind::{SourceHits, IndexStatus, PersonaStatus, PipelineDocumentSummary, PipelineDocumentView, PipelineStatus, PipelineStatusSummary};
|
use huginn_mind::{ChatHits, ChatWindow, SourceHits, IndexStatus, PersonaStatus, PipelineDocumentSummary, PipelineDocumentView, PipelineStatus, PipelineStatusSummary};
|
||||||
use serde::Serialize;
|
use serde::Serialize;
|
||||||
use serde_json::Value;
|
use serde_json::Value;
|
||||||
|
|
||||||
|
|
@ -147,6 +147,72 @@ pub fn source_hits(found: &SourceHits) -> String {
|
||||||
")
|
")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Chat hits as the CLI prints them. Per hit the organ's own header line,
|
||||||
|
/// `#alias (!room) first..last time..time score` (the alias and the score only
|
||||||
|
/// when there are some), then one `time sender event_id body` line per message,
|
||||||
|
/// the body as the room held it with line breaks written `\n`: quoted data
|
||||||
|
/// under our header, never a message framed as if sent to the reader. A blank line between hits; a
|
||||||
|
/// truncated listing of one room ends with the `--after` that continues it. A
|
||||||
|
/// listing of several rooms (`one_room` false) cannot be continued by an instant,
|
||||||
|
/// because chunks of different rooms overlap in time and a later start can hold an
|
||||||
|
/// earlier end; it says to page one room at a time.
|
||||||
|
pub fn chat_hits(found: &ChatHits, one_room: bool) -> String {
|
||||||
|
let mut out: Vec<String> = found
|
||||||
|
.hits
|
||||||
|
.iter()
|
||||||
|
.map(|hit| {
|
||||||
|
let room = hit.room_alias.as_ref().map_or_else(|| hit.room_id.clone(), |alias| format!("{alias} ({})", hit.room_id));
|
||||||
|
let score = hit.score.map_or_else(String::new, |score| format!(" {score:.3}"));
|
||||||
|
let mut text = format!(
|
||||||
|
"{room} {}..{} {}..{}{score}\n",
|
||||||
|
hit.first_event_id,
|
||||||
|
hit.last_event_id,
|
||||||
|
crate::chat::shown(hit.ts_start_ms),
|
||||||
|
crate::chat::shown(hit.ts_end_ms)
|
||||||
|
);
|
||||||
|
for message in &hit.messages {
|
||||||
|
let _ = writeln!(text, "{} {} {} {}", crate::chat::shown(message.ts_ms), message.sender, message.event_id, one_line(&message.body));
|
||||||
|
}
|
||||||
|
text
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
if found.truncated {
|
||||||
|
match found.hits.last() {
|
||||||
|
Some(last) if one_room => out.push(format!("truncated: more follow, continue with --after {}\n", crate::chat::shown(last.ts_end_ms + 1))),
|
||||||
|
_ => out.push("truncated: more follow; --after cannot continue a listing of several rooms, list one --room at a time\n".to_string()),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
out.join("\n")
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A message body on one line: a line break is written `\n` (and a carriage
|
||||||
|
/// return `\r`), so a body cannot start a line of its own. `--json` has the
|
||||||
|
/// body exactly.
|
||||||
|
fn one_line(body: &str) -> String {
|
||||||
|
body.replace('\r', "\\r").replace('\n', "\\n")
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A chat window as the CLI prints it: `#alias (!room)` (the id alone with no
|
||||||
|
/// alias), then one `time sender event_id body` line per message, the anchor
|
||||||
|
/// marked `>` and the rest indented to match; a line says when messages lie
|
||||||
|
/// beyond either end.
|
||||||
|
pub fn chat_window(found: &ChatWindow) -> String {
|
||||||
|
let room = found.room_alias.as_ref().map_or_else(|| found.room_id.clone(), |alias| format!("{alias} ({})", found.room_id));
|
||||||
|
let mut out = format!("{room}\n");
|
||||||
|
if found.more_before {
|
||||||
|
out.push_str(" (earlier messages exist)\n");
|
||||||
|
}
|
||||||
|
for message in &found.messages {
|
||||||
|
let mark = if message.event_id == found.anchor_event_id { '>' } else { ' ' };
|
||||||
|
let _ = writeln!(out, "{mark} {} {} {} {}", crate::chat::shown(message.ts_ms), message.sender, message.event_id, one_line(&message.body));
|
||||||
|
}
|
||||||
|
if found.more_after {
|
||||||
|
out.push_str(" (later messages exist)\n");
|
||||||
|
}
|
||||||
|
out.pop();
|
||||||
|
out
|
||||||
|
}
|
||||||
|
|
||||||
/// A fact as one token: a string, a number, a flag, a reference's id, or a
|
/// A fact as one token: a string, a number, a flag, a reference's id, or a
|
||||||
/// list of those joined by commas. Anything deeper is not a header fact.
|
/// list of those joined by commas. Anything deeper is not a header fact.
|
||||||
fn scalar(value: &Value) -> Option<String> {
|
fn scalar(value: &Value) -> Option<String> {
|
||||||
|
|
|
||||||
|
|
@ -20,7 +20,7 @@ macro_rules! kinds {
|
||||||
|
|
||||||
kinds!(
|
kinds!(
|
||||||
campaign, target, question, ruling, cut_spec, cut_report, verdict, finding, follow_up, resolution, instance, stewardship,
|
campaign, target, question, ruling, cut_spec, cut_report, verdict, finding, follow_up, resolution, instance, stewardship,
|
||||||
hand_off, run, pulse, memory, research, idea, reply, thought, desk_answer,
|
hand_off, run, pulse, memory, research, idea, reply, thought, desk_answer, desk_listing,
|
||||||
);
|
);
|
||||||
|
|
||||||
/// The schema named, or the names when none is: the exit code is 2 for a name
|
/// The schema named, or the names when none is: the exit code is 2 for a name
|
||||||
|
|
|
||||||
|
|
@ -1145,12 +1145,12 @@ fn schema_prints_the_published_files() {
|
||||||
let names = run(&[], &["schema"], None);
|
let names = run(&[], &["schema"], None);
|
||||||
assert_eq!(names.code, 0);
|
assert_eq!(names.code, 0);
|
||||||
let names: Vec<&str> = names.out.lines().collect();
|
let names: Vec<&str> = names.out.lines().collect();
|
||||||
assert_eq!(names.len(), 22, "{names:?}");
|
assert_eq!(names.len(), 23, "{names:?}");
|
||||||
assert_eq!(names[0], "campaign");
|
assert_eq!(names[0], "campaign");
|
||||||
assert_eq!(names[14], "pulse");
|
assert_eq!(names[14], "pulse");
|
||||||
assert_eq!(names[15], "memory");
|
assert_eq!(names[15], "memory");
|
||||||
assert_eq!((names[16], names[17]), ("research", "idea"));
|
assert_eq!((names[16], names[17]), ("research", "idea"));
|
||||||
assert_eq!((names[18], names[19], names[20], names[21]), ("reply", "thought", "desk_answer", "selection"));
|
assert_eq!((names[18], names[19], names[20], names[21], names[22]), ("reply", "thought", "desk_answer", "desk_listing", "selection"));
|
||||||
for kind in PipelineKind::ALL {
|
for kind in PipelineKind::ALL {
|
||||||
assert!(names.contains(&kind.name()), "{}", kind.name());
|
assert!(names.contains(&kind.name()), "{}", kind.name());
|
||||||
}
|
}
|
||||||
|
|
@ -1233,7 +1233,8 @@ struct Desk {
|
||||||
embeds_under: &'static str,
|
embeds_under: &'static str,
|
||||||
held: Vec<huginn_daemon::sources::SourcePayload>,
|
held: Vec<huginn_daemon::sources::SourcePayload>,
|
||||||
tickets: u64,
|
tickets: u64,
|
||||||
done: Vec<(SearchTicket, Result<huginn_mind::SourceHits, String>)>,
|
chat: Vec<huginn_daemon::sources::ChatPayload>,
|
||||||
|
done: Vec<(SearchTicket, huginn_daemon::daemon::Found)>,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// An embedder that sends every text to one vector, under a digest.
|
/// An embedder that sends every text to one vector, under a digest.
|
||||||
|
|
@ -1297,6 +1298,55 @@ impl huginn_daemon::sources::SourceIndex for Desk {
|
||||||
.map(|payload| huginn_daemon::sources::SourceRow { score: 0.9, payload: payload.clone() })
|
.map(|payload| huginn_daemon::sources::SourceRow { score: 0.9, payload: payload.clone() })
|
||||||
.collect())
|
.collect())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn upsert_chat(&mut self, _points: &[huginn_daemon::sources::ChatPoint]) -> anyhow::Result<()> {
|
||||||
|
anyhow::bail!("a daemon never writes the source index")
|
||||||
|
}
|
||||||
|
|
||||||
|
fn chat_tail(&mut self, _room_id: &str) -> anyhow::Result<Option<huginn_daemon::sources::StoredChat>> {
|
||||||
|
anyhow::bail!("a daemon never reads the ingest job's side of the source index")
|
||||||
|
}
|
||||||
|
|
||||||
|
fn chat_chunks_holding(&mut self, _room_id: &str, _event_ids: &[String]) -> anyhow::Result<Vec<huginn_daemon::sources::StoredChat>> {
|
||||||
|
anyhow::bail!("a daemon never reads the ingest job's side of the source index")
|
||||||
|
}
|
||||||
|
|
||||||
|
fn chat_chunks_overlapping(&mut self, _room_id: &str, _from_ms: u64, _to_ms: u64) -> anyhow::Result<Vec<huginn_daemon::sources::StoredChat>> {
|
||||||
|
anyhow::bail!("a daemon never reads the ingest job's side of the source index")
|
||||||
|
}
|
||||||
|
|
||||||
|
fn delete_chat(&mut self, _room_id: &str, _ids: Option<&[String]>) -> anyhow::Result<()> {
|
||||||
|
anyhow::bail!("a daemon never writes the source index")
|
||||||
|
}
|
||||||
|
|
||||||
|
fn chat_nearest(
|
||||||
|
&mut self,
|
||||||
|
_vector: &[f32],
|
||||||
|
filter: &huginn_daemon::sources::ChatFilter,
|
||||||
|
limit: u32,
|
||||||
|
_within: Duration,
|
||||||
|
) -> anyhow::Result<Vec<huginn_daemon::sources::ChatRow>> {
|
||||||
|
Ok(self.chat_scroll(filter, huginn_daemon::sources::ChatOrder::OldestFirst, limit, _within)?.into_iter().map(|payload| huginn_daemon::sources::ChatRow { score: 0.9, payload }).collect())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn chat_scroll(&mut self, filter: &huginn_daemon::sources::ChatFilter, order: huginn_daemon::sources::ChatOrder, limit: u32, _within: Duration) -> anyhow::Result<Vec<huginn_daemon::sources::ChatPayload>> {
|
||||||
|
let mut kept: Vec<_> = self
|
||||||
|
.chat
|
||||||
|
.iter()
|
||||||
|
.filter(|payload| {
|
||||||
|
let times = payload.messages.iter().map(|message| message.ts_ms);
|
||||||
|
filter.event_id.as_ref().is_none_or(|id| payload.messages.iter().any(|message| &message.event_id == id))
|
||||||
|
&& filter.keeps(&payload.room_id, payload.room_alias.as_deref(), times.clone().min().unwrap(), times.max().unwrap())
|
||||||
|
})
|
||||||
|
.cloned()
|
||||||
|
.collect();
|
||||||
|
kept.sort_by_key(|payload| payload.messages[0].ts_ms);
|
||||||
|
if order == huginn_daemon::sources::ChatOrder::NewestFirst {
|
||||||
|
kept.reverse();
|
||||||
|
}
|
||||||
|
kept.truncate(limit as usize);
|
||||||
|
Ok(kept)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl<S: MindStore> IndexSink<S> for Desk {
|
impl<S: MindStore> IndexSink<S> for Desk {
|
||||||
|
|
@ -1313,11 +1363,32 @@ impl<S: MindStore> IndexSink<S> for Desk {
|
||||||
self.tickets += 1;
|
self.tickets += 1;
|
||||||
let ticket = SearchTicket(self.tickets);
|
let ticket = SearchTicket(self.tickets);
|
||||||
let mut embedder = Flat(self.embeds_under);
|
let mut embedder = Flat(self.embeds_under);
|
||||||
let found = huginn_daemon::sources::search(&mut embedder, self, query, Duration::from_secs(5)).map_err(|error| format!("{error:#}"));
|
let found = huginn_daemon::sources::search(&mut embedder, self, query, Duration::from_secs(5))
|
||||||
|
.map_or_else(|error| huginn_daemon::daemon::Found::Failed(format!("{error:#}")), huginn_daemon::daemon::Found::Sources);
|
||||||
self.done.push((ticket, found));
|
self.done.push((ticket, found));
|
||||||
Ok(ticket)
|
Ok(ticket)
|
||||||
}
|
}
|
||||||
fn sources_searched(&mut self) -> Vec<(SearchTicket, Result<huginn_mind::SourceHits, String>)> {
|
fn search_chat(&mut self, query: &huginn_mind::ChatQuery) -> Result<SearchTicket, String> {
|
||||||
|
self.tickets += 1;
|
||||||
|
let ticket = SearchTicket(self.tickets);
|
||||||
|
let mut embedder = Flat(self.embeds_under);
|
||||||
|
let found = huginn_daemon::sources::search_chat(&mut embedder, self, query, Duration::from_secs(5))
|
||||||
|
.map_or_else(|error| huginn_daemon::daemon::Found::Failed(format!("{error:#}")), huginn_daemon::daemon::Found::Chat);
|
||||||
|
self.done.push((ticket, found));
|
||||||
|
Ok(ticket)
|
||||||
|
}
|
||||||
|
fn chat_window(&mut self, query: &huginn_mind::ChatWindowQuery) -> Result<SearchTicket, String> {
|
||||||
|
self.tickets += 1;
|
||||||
|
let ticket = SearchTicket(self.tickets);
|
||||||
|
let found = match huginn_daemon::sources::window(self, query, Duration::from_secs(5)) {
|
||||||
|
Ok(Ok(window)) => huginn_daemon::daemon::Found::Window(window),
|
||||||
|
Ok(Err(refusal)) => huginn_daemon::daemon::Found::Refused(refusal),
|
||||||
|
Err(error) => huginn_daemon::daemon::Found::Failed(format!("{error:#}")),
|
||||||
|
};
|
||||||
|
self.done.push((ticket, found));
|
||||||
|
Ok(ticket)
|
||||||
|
}
|
||||||
|
fn sources_searched(&mut self) -> Vec<(SearchTicket, huginn_daemon::daemon::Found)> {
|
||||||
std::mem::take(&mut self.done)
|
std::mem::take(&mut self.done)
|
||||||
}
|
}
|
||||||
fn search_deadline(&self) -> Duration {
|
fn search_deadline(&self) -> Duration {
|
||||||
|
|
@ -1351,6 +1422,15 @@ fn source_daemon(built_under: &'static str, embeds_under: &'static str) -> Serve
|
||||||
chunk("Demo", "README.md", 1, 2, "# Demo"),
|
chunk("Demo", "README.md", 1, 2, "# Demo"),
|
||||||
chunk("Other", "x.rs", 1, 1, "fn other() {}"),
|
chunk("Other", "x.rs", 1, 1, "fn other() {}"),
|
||||||
],
|
],
|
||||||
|
chat: vec![huginn_daemon::sources::ChatPayload {
|
||||||
|
room_id: "!r:gamecult.org".into(),
|
||||||
|
room_alias: Some("#swarm:gamecult.org".into()),
|
||||||
|
messages: vec![
|
||||||
|
huginn_mind::ChatMessage { event_id: "$e1".into(), sender: "@nibu:gamecult.org".into(), ts_ms: 1_700_000_000_000, body: "ship the cut\nthen rest".into() },
|
||||||
|
huginn_mind::ChatMessage { event_id: "$e2".into(), sender: "@eureka:gamecult.org".into(), ts_ms: 1_700_000_060_000, body: "done".into() },
|
||||||
|
],
|
||||||
|
text: "swarm".into(),
|
||||||
|
}],
|
||||||
tickets: 0,
|
tickets: 0,
|
||||||
done: Vec::new(),
|
done: Vec::new(),
|
||||||
};
|
};
|
||||||
|
|
@ -2886,3 +2966,94 @@ fn a_malformed_huginn_session_is_refused_naming_the_variable_without_echoing_it(
|
||||||
let edge = with_session(live.addr, &"E".repeat(200), &admit_without_session(&file), None);
|
let edge = with_session(live.addr, &"E".repeat(200), &admit_without_session(&file), None);
|
||||||
assert_eq!(edge.code, 0, "{}", edge.err);
|
assert_eq!(edge.code, 0, "{}", edge.err);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// `huginn chat search` end to end: a listing through the daemon's worker path
|
||||||
|
/// prints the organ's header line and each message verbatim with its event id,
|
||||||
|
/// a room that holds nothing prints nothing, and the flags reach the request as
|
||||||
|
/// the instants and rooms they name.
|
||||||
|
#[test]
|
||||||
|
fn chat_search_prints_cited_messages_and_sends_its_selectors() {
|
||||||
|
let server = source_daemon("d1", "d1");
|
||||||
|
let found = at(server.addr, &["chat", "search", "--room", "#swarm:gamecult.org", "--after", "2023-11-14T22:00:00Z"], None);
|
||||||
|
assert_eq!(found.code, 0, "{}", found.err);
|
||||||
|
assert_eq!(
|
||||||
|
found.out,
|
||||||
|
"#swarm:gamecult.org (!r:gamecult.org) $e1..$e2 2023-11-14T22:13:20.000Z..2023-11-14T22:14:20.000Z\n\
|
||||||
|
2023-11-14T22:13:20.000Z @nibu:gamecult.org $e1 ship the cut\\nthen rest\n\
|
||||||
|
2023-11-14T22:14:20.000Z @eureka:gamecult.org $e2 done\n\n"
|
||||||
|
);
|
||||||
|
let other = at(server.addr, &["chat", "search", "--room", "!elsewhere:gamecult.org"], None);
|
||||||
|
assert_eq!((other.code, other.out.trim()), (0, ""));
|
||||||
|
let by_meaning = at(server.addr, &["chat", "search", "who shipped it", "--top-k", "1"], None);
|
||||||
|
assert!(by_meaning.out.starts_with("#swarm:gamecult.org (!r:gamecult.org) $e1..$e2 ") && by_meaning.out.lines().next().unwrap().ends_with(" 0.900"), "{}", by_meaning.out);
|
||||||
|
|
||||||
|
let (spy, seen) = replying(
|
||||||
|
"search_chat",
|
||||||
|
HuginnMindResponse::Chat(huginn_mind::ChatHits { indexed_under: huginn_mind::SourceModel { name: "m".into(), digest: "d".into() }, truncated: false, hits: vec![] }),
|
||||||
|
);
|
||||||
|
let ran = at(spy.addr, &["chat", "search", "a", "b", "--room", "!x:y", "--room", "#z:y", "--after", "2023-11-14T22:13:20Z", "--before", "2023-11-14T22:14:20Z", "--top-k", "3"], None);
|
||||||
|
assert_eq!(ran.code, 0, "{}", ran.err);
|
||||||
|
let HuginnMindRequest::SearchChat { query, .. } = seen.lock().unwrap().remove(0) else { panic!("a chat search") };
|
||||||
|
let texts: Vec<&str> = query.texts.iter().map(|text| text.0.as_str()).collect();
|
||||||
|
let rooms: Vec<&str> = query.rooms.iter().map(|room| room.0.as_str()).collect();
|
||||||
|
assert_eq!((texts, rooms, query.after_ms, query.before_ms, query.top_k), (vec!["a", "b"], vec!["!x:y", "#z:y"], Some(1_700_000_000_000), Some(1_700_000_060_000), 3));
|
||||||
|
|
||||||
|
// A truncated listing is continued by `--after` only inside one room; across rooms the spans overlap and an instant loses chunks.
|
||||||
|
let hit = |end| huginn_mind::ChatHit {
|
||||||
|
room_id: "!r:y".into(),
|
||||||
|
room_alias: None,
|
||||||
|
first_event_id: "$a".into(),
|
||||||
|
last_event_id: "$b".into(),
|
||||||
|
ts_start_ms: 1_700_000_000_000,
|
||||||
|
ts_end_ms: end,
|
||||||
|
score: None,
|
||||||
|
messages: vec![],
|
||||||
|
};
|
||||||
|
let cut_short = HuginnMindResponse::Chat(huginn_mind::ChatHits { indexed_under: huginn_mind::SourceModel { name: "m".into(), digest: "d".into() }, truncated: true, hits: vec![hit(1_700_000_060_000)] });
|
||||||
|
let (listing, _) = replying("search_chat", cut_short);
|
||||||
|
let one = at(listing.addr, &["chat", "search", "--room", "!r:y"], None);
|
||||||
|
assert!(one.out.contains("continue with --after 2023-11-14T22:14:20.001Z"), "{}", one.out);
|
||||||
|
for rooms in [&["chat", "search"][..], &["chat", "search", "--room", "!r:y", "--room", "!s:y"]] {
|
||||||
|
let many = at(listing.addr, rooms, None);
|
||||||
|
assert!(many.out.contains("--after cannot continue a listing of several rooms") && !many.out.contains("continue with --after"), "{rooms:?}: {}", many.out);
|
||||||
|
}
|
||||||
|
|
||||||
|
assert_ne!(at(spy.addr, &["chat", "search", "--after", "yesterday"], None).code, 0);
|
||||||
|
assert!(seen.lock().unwrap().is_empty(), "a bad instant never left the CLI");
|
||||||
|
for bad in [&["chat", "search", "--room", "swarm"][..], &["chat", "search", "--top-k", "0"]] {
|
||||||
|
let ran = at(server.addr, bad, None);
|
||||||
|
assert_eq!(ran.code, 1, "{bad:?} is a refusal by the daemon's door: {}", ran.err);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// `huginn chat window` end to end: an unknown or malformed event id is a
|
||||||
|
/// refusal naming the field; a known one prints the room header and the
|
||||||
|
/// messages with the anchor marked `>`; the flags reach the request.
|
||||||
|
#[test]
|
||||||
|
fn chat_window_prints_the_anchor_marked_and_refuses_what_the_index_lacks() {
|
||||||
|
let server = source_daemon("d1", "d1");
|
||||||
|
for (bad, field) in [("$unknown:gamecult.org", "event_id"), ("e1", "event_id")] {
|
||||||
|
let ran = at(server.addr, &["chat", "window", bad], None);
|
||||||
|
assert_eq!(ran.code, 1, "{}", ran.err);
|
||||||
|
assert!(ran.out.starts_with("Refused SelectionInvalid") && ran.out.contains(field), "no window for an id the index lacks: {}", ran.out);
|
||||||
|
}
|
||||||
|
let found = at(server.addr, &["chat", "window", "$e2"], None);
|
||||||
|
assert_eq!(found.code, 0, "{}", found.err);
|
||||||
|
assert_eq!(
|
||||||
|
found.out,
|
||||||
|
"#swarm:gamecult.org (!r:gamecult.org)\n\
|
||||||
|
\x20 2023-11-14T22:13:20.000Z @nibu:gamecult.org $e1 ship the cut\\nthen rest\n\
|
||||||
|
> 2023-11-14T22:14:20.000Z @eureka:gamecult.org $e2 done\n"
|
||||||
|
);
|
||||||
|
let over = at(server.addr, &["chat", "window", "$e2", "--before", "51"], None);
|
||||||
|
assert_eq!(over.code, 1, "{}", over.err);
|
||||||
|
|
||||||
|
let (spy, seen) = replying(
|
||||||
|
"chat_window",
|
||||||
|
HuginnMindResponse::ChatWindow(huginn_mind::ChatWindow { room_id: "!r:x".into(), room_alias: None, anchor_event_id: "$a".into(), messages: vec![], more_before: false, more_after: false }),
|
||||||
|
);
|
||||||
|
let ran = at(spy.addr, &["chat", "window", "$a", "--before", "3", "--after", "0"], None);
|
||||||
|
assert_eq!(ran.code, 0, "{}", ran.err);
|
||||||
|
let HuginnMindRequest::ChatWindow { query, .. } = seen.lock().unwrap().remove(0) else { panic!("a chat window") };
|
||||||
|
assert_eq!((query.event_id.0.as_str(), query.before, query.after), ("$a", 3, 0));
|
||||||
|
}
|
||||||
|
|
|
||||||
|
|
@ -93,10 +93,18 @@ impl<S: MindStore> IndexSink<S> for Inert {
|
||||||
fn search(&mut self, _text: &str, _top_k: u32, _among: Vec<String>) -> Result<SearchTicket, String> {
|
fn search(&mut self, _text: &str, _top_k: u32, _among: Vec<String>) -> Result<SearchTicket, String> {
|
||||||
Err("this index cannot search".into())
|
Err("this index cannot search".into())
|
||||||
}
|
}
|
||||||
|
fn search_chat(&mut self, _query: &huginn_mind::ChatQuery) -> Result<SearchTicket, String> {
|
||||||
|
Err("this index cannot search".into())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn chat_window(&mut self, _query: &huginn_mind::ChatWindowQuery) -> Result<SearchTicket, String> {
|
||||||
|
Err("this index cannot search".into())
|
||||||
|
}
|
||||||
|
|
||||||
fn search_sources(&mut self, _query: &huginn_mind::SourceQuery) -> Result<SearchTicket, String> {
|
fn search_sources(&mut self, _query: &huginn_mind::SourceQuery) -> Result<SearchTicket, String> {
|
||||||
Err("this index cannot search".into())
|
Err("this index cannot search".into())
|
||||||
}
|
}
|
||||||
fn sources_searched(&mut self) -> Vec<(SearchTicket, Result<huginn_mind::SourceHits, String>)> {
|
fn sources_searched(&mut self) -> Vec<(SearchTicket, huginn_daemon::daemon::Found)> {
|
||||||
Vec::new()
|
Vec::new()
|
||||||
}
|
}
|
||||||
fn search_deadline(&self) -> Duration {
|
fn search_deadline(&self) -> Duration {
|
||||||
|
|
|
||||||
|
|
@ -18,7 +18,7 @@ name = "huginn-daemon"
|
||||||
path = "src/main.rs"
|
path = "src/main.rs"
|
||||||
|
|
||||||
# The source index's one writer: a oneshot job that brings `huginn_sources` to
|
# The source index's one writer: a oneshot job that brings `huginn_sources` to
|
||||||
# the forge's current state and exits. No daemon calls any of its writes.
|
# the forge's and the chat reader's current state and exits. No daemon calls any of its writes.
|
||||||
[[bin]]
|
[[bin]]
|
||||||
name = "huginn-sources"
|
name = "huginn-sources"
|
||||||
path = "src/bin/huginn-sources.rs"
|
path = "src/bin/huginn-sources.rs"
|
||||||
|
|
@ -34,7 +34,8 @@ rmp-serde = "1"
|
||||||
rustix = { version = "1", default-features = false, features = ["std", "net"] }
|
rustix = { version = "1", default-features = false, features = ["std", "net"] }
|
||||||
serde = { version = "1", features = ["derive"] }
|
serde = { version = "1", features = ["derive"] }
|
||||||
# The vector store's and the embedder's HTTP APIs are the xenos boundary, and
|
# The vector store's and the embedder's HTTP APIs are the xenos boundary, and
|
||||||
# JSON exists in `index/ollama.rs` and `index/qdrant.rs` only. `ureq` 3 with
|
# JSON exists in `index/ollama.rs`, `index/qdrant.rs`, `sources/forge.rs` and
|
||||||
|
# `sources/matrix.rs` only. `ureq` 3 with
|
||||||
# default features off is plain HTTP with no TLS (`rustls` and `gzip` are its
|
# default features off is plain HTTP with no TLS (`rustls` and `gzip` are its
|
||||||
# only defaults), which is all the private mesh needs. It stays below 3.4,
|
# only defaults), which is all the private mesh needs. It stays below 3.4,
|
||||||
# whose `base64` 0.23 would put a second `base64` in a tree that has one.
|
# whose `base64` 0.23 would put a second `base64` in a tree that has one.
|
||||||
|
|
|
||||||
|
|
@ -1,21 +1,38 @@
|
||||||
//! The source index's refresh job: one run brings the `huginn_sources`
|
//! The source index's refresh job: one run brings the forge corpus and, when
|
||||||
//! collection to the forge's current state, journals one line per repo, and
|
//! configured, the rooms the chat reader has joined (minus the estate and DM
|
||||||
//! exits nonzero if any repo failed or the collection could not be used.
|
//! exclusion) to current state, journals one line per repo and room, and exits
|
||||||
|
//! nonzero if either failed or the collection could not be used. The
|
||||||
|
//! `chat-purge` subcommand writes a steward's purge tombstone and nothing else.
|
||||||
|
//!
|
||||||
|
//! Exit status: 0 clean, 1 a run failed, 2 bad arguments, 3 another run (or a
|
||||||
|
//! purge) holds the lock, which every run and every purge takes. The Matrix
|
||||||
|
//! token is read from a file (a systemd credential), never from the environment or argv.
|
||||||
|
|
||||||
use std::collections::BTreeMap;
|
use std::collections::{BTreeMap, BTreeSet};
|
||||||
use std::path::PathBuf;
|
use std::fs::{File, OpenOptions, TryLockError};
|
||||||
|
use std::path::{Path, PathBuf};
|
||||||
use std::process::ExitCode;
|
use std::process::ExitCode;
|
||||||
|
|
||||||
use anyhow::{Context, Result, ensure};
|
use anyhow::{Context, Result, ensure};
|
||||||
|
use chrono::{DateTime, Utc};
|
||||||
use huginn_daemon::index::ollama::pool;
|
use huginn_daemon::index::ollama::pool;
|
||||||
use huginn_daemon::index::qdrant::QdrantIndex;
|
use huginn_daemon::index::qdrant::QdrantIndex;
|
||||||
use huginn_daemon::sources::forge::HttpForge;
|
use huginn_daemon::sources::forge::HttpForge;
|
||||||
use huginn_daemon::sources::refresh;
|
use huginn_daemon::sources::job::{self, Chat, Purge};
|
||||||
|
use huginn_daemon::sources::matrix::HttpMatrix;
|
||||||
|
use huginn_daemon::sources::room_scope::{ESTATE_ALIASES, Estate};
|
||||||
|
|
||||||
/// Where the forge answers when the flag is absent: the loopback address of the
|
/// Where the forge answers when the flag is absent: the loopback address of the
|
||||||
/// forge on the host that runs this job.
|
/// forge on the host that runs this job.
|
||||||
const DEFAULT_FORGE_URL: &str = "http://127.0.0.1:3310";
|
const DEFAULT_FORGE_URL: &str = "http://127.0.0.1:3310";
|
||||||
|
|
||||||
|
/// The chat flags, which come all together or not at all.
|
||||||
|
struct ChatOptions {
|
||||||
|
matrix_url: String,
|
||||||
|
token: String,
|
||||||
|
estate: Estate,
|
||||||
|
}
|
||||||
|
|
||||||
struct Options {
|
struct Options {
|
||||||
state_root: PathBuf,
|
state_root: PathBuf,
|
||||||
qdrant_url: String,
|
qdrant_url: String,
|
||||||
|
|
@ -25,75 +42,335 @@ struct Options {
|
||||||
reserve_urls: Vec<String>,
|
reserve_urls: Vec<String>,
|
||||||
embedding_model: String,
|
embedding_model: String,
|
||||||
forge_url: String,
|
forge_url: String,
|
||||||
|
chat: Option<ChatOptions>,
|
||||||
|
/// Every run takes it: the cursors object is replaced whole, with or without chat.
|
||||||
|
lock_file: PathBuf,
|
||||||
}
|
}
|
||||||
|
|
||||||
fn parse(args: impl Iterator<Item = String>) -> Result<Options> {
|
enum Command {
|
||||||
|
Run(Options),
|
||||||
|
Purge { qdrant_url: String, lock_file: PathBuf, purge: Purge },
|
||||||
|
}
|
||||||
|
|
||||||
|
/// How a run ends other than by finishing.
|
||||||
|
enum Stop {
|
||||||
|
Usage(anyhow::Error),
|
||||||
|
Locked,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The single-valued options and the repeated ones, by name.
|
||||||
|
type Collected = (BTreeMap<String, String>, BTreeMap<String, Vec<String>>);
|
||||||
|
|
||||||
|
/// The options named, each once, except those in `repeated`, which keep every value.
|
||||||
|
fn collect(args: impl Iterator<Item = String>, once: &[&str], repeated: &[&str]) -> Result<Collected> {
|
||||||
let mut args = args;
|
let mut args = args;
|
||||||
let mut values: BTreeMap<String, String> = BTreeMap::new();
|
let mut singles = BTreeMap::new();
|
||||||
let mut ollama_urls = Vec::new();
|
let mut many: BTreeMap<String, Vec<String>> = BTreeMap::new();
|
||||||
let mut reserve_urls = Vec::new();
|
|
||||||
while let Some(name) = args.next() {
|
while let Some(name) = args.next() {
|
||||||
let name = name.strip_prefix("--").with_context(|| format!("expected --option, got {name:?}"))?;
|
let name = name.strip_prefix("--").with_context(|| format!("expected --option, got {name:?}"))?;
|
||||||
ensure!(
|
ensure!(once.contains(&name) || repeated.contains(&name), "unsupported huginn-sources option --{name}");
|
||||||
matches!(name, "state-root" | "qdrant-url" | "ollama-url" | "reserve-ollama-url" | "embedding-model" | "forge-url"),
|
|
||||||
"unsupported huginn-sources option --{name}"
|
|
||||||
);
|
|
||||||
let value = args.next().with_context(|| format!("missing value for --{name}"))?;
|
let value = args.next().with_context(|| format!("missing value for --{name}"))?;
|
||||||
if name == "ollama-url" || name == "reserve-ollama-url" {
|
if repeated.contains(&name) {
|
||||||
ensure!(value.starts_with("http://"), "--{name} must be a plain http:// address, got {value:?}");
|
many.entry(name.to_owned()).or_default().push(value);
|
||||||
if name == "ollama-url" { ollama_urls.push(value) } else { reserve_urls.push(value) }
|
|
||||||
} else {
|
} else {
|
||||||
ensure!(values.insert(name.to_owned(), value).is_none(), "duplicate option --{name}");
|
ensure!(singles.insert(name.to_owned(), value).is_none(), "duplicate option --{name}");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
let take = |name: &str| values.get(name).cloned().with_context(|| format!("--{name} is required"));
|
Ok((singles, many))
|
||||||
let url = |name: &str, value: String| {
|
}
|
||||||
ensure!(value.starts_with("http://"), "--{name} must be a plain http:// address, got {value:?}");
|
|
||||||
Ok(value)
|
fn http(name: &str, value: String) -> Result<String> {
|
||||||
};
|
ensure!(value.starts_with("http://"), "--{name} must be a plain http:// address, got {value:?}");
|
||||||
let state_root = PathBuf::from(take("state-root")?);
|
Ok(value)
|
||||||
ensure!(state_root.is_absolute(), "--state-root must be an absolute path");
|
}
|
||||||
let embedding_model = take("embedding-model")?;
|
|
||||||
ensure!(!embedding_model.is_empty(), "--embedding-model must name a model");
|
/// The Matrix reader's bearer travels in clear, and this build speaks no TLS: only a loopback host may be named.
|
||||||
Ok(Options {
|
fn loopback_http(name: &str, value: String) -> Result<String> {
|
||||||
state_root,
|
let value = http(name, value)?;
|
||||||
qdrant_url: url("qdrant-url", take("qdrant-url")?)?,
|
let authority = value["http://".len()..].split(['/', '?', '#']).next().unwrap_or("");
|
||||||
ollama_urls: {
|
let host = match authority.strip_prefix('[') {
|
||||||
ensure!(!ollama_urls.is_empty(), "at least one --ollama-url is required; repeat it for each GPU host");
|
// The bracket must end the authority or be followed by the port, so `[::1].evil.example` names no loopback.
|
||||||
ollama_urls
|
Some(inner) => match inner.split_once(']') {
|
||||||
|
Some((host, "")) => host,
|
||||||
|
Some((host, port)) if port.starts_with(':') => host,
|
||||||
|
_ => "",
|
||||||
},
|
},
|
||||||
|
None => authority.split(':').next().unwrap_or(""),
|
||||||
|
};
|
||||||
|
let loopback = !authority.contains('@') && (host.eq_ignore_ascii_case("localhost") || host.parse::<std::net::IpAddr>().is_ok_and(|ip| ip.is_loopback()));
|
||||||
|
ensure!(loopback, "--{name} must name a loopback host (localhost, 127.0.0.0/8 or [::1]): the bearer token travels in clear and this build has no TLS, got {value:?}");
|
||||||
|
Ok(value)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn absolute(name: &str, value: &str) -> Result<PathBuf> {
|
||||||
|
let path = PathBuf::from(value);
|
||||||
|
ensure!(path.is_absolute(), "--{name} must be an absolute path");
|
||||||
|
Ok(path)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn chat_options(singles: &BTreeMap<String, String>, excluded: Vec<String>) -> Result<Option<ChatOptions>> {
|
||||||
|
let given = [singles.contains_key("matrix-url"), singles.contains_key("matrix-token-file"), !excluded.is_empty()];
|
||||||
|
if !given.contains(&true) {
|
||||||
|
return Ok(None);
|
||||||
|
}
|
||||||
|
for (present, name) in given.iter().zip(["matrix-url", "matrix-token-file", "chat-exclude-room"]) {
|
||||||
|
ensure!(*present, "--{name} is missing: the chat flags (--matrix-url, --matrix-token-file, --chat-exclude-room) come all together or not at all");
|
||||||
|
}
|
||||||
|
let ids: BTreeSet<String> = excluded.iter().cloned().collect();
|
||||||
|
ensure!(ids.len() == excluded.len(), "--chat-exclude-room must not repeat a room id");
|
||||||
|
ensure!(ids.len() == ESTATE_ALIASES.len(), "--chat-exclude-room must be given exactly {} times (the ids of {})", ESTATE_ALIASES.len(), ESTATE_ALIASES.join(", "));
|
||||||
|
ensure!(ids.iter().all(|id| id.starts_with('!')), "--chat-exclude-room takes room ids beginning with !, not aliases");
|
||||||
|
let path = absolute("matrix-token-file", &singles["matrix-token-file"])?;
|
||||||
|
let token = std::fs::read_to_string(&path).with_context(|| format!("reading --matrix-token-file {}", path.display()))?.trim().to_owned();
|
||||||
|
ensure!(!token.is_empty(), "--matrix-token-file {} is empty", path.display());
|
||||||
|
Ok(Some(ChatOptions { matrix_url: loopback_http("matrix-url", singles["matrix-url"].clone())?, token, estate: Estate::new(ids)? }))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parse(mut args: impl Iterator<Item = String>, now: DateTime<Utc>) -> Result<Command> {
|
||||||
|
let mut rest: Vec<String> = Vec::new();
|
||||||
|
let first = args.next();
|
||||||
|
if first.as_deref() == Some("chat-purge") {
|
||||||
|
let (singles, _) = collect(args, &["qdrant-url", "room", "from", "to", "request", "lock-file"], &[])?;
|
||||||
|
let take = |name: &str| singles.get(name).cloned().with_context(|| format!("--{name} is required"));
|
||||||
|
return Ok(Command::Purge {
|
||||||
|
qdrant_url: http("qdrant-url", take("qdrant-url")?)?,
|
||||||
|
lock_file: absolute("lock-file", &take("lock-file")?)?,
|
||||||
|
purge: Purge::check(&take("room")?, &take("from")?, &take("to")?, &take("request")?, now)?,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
rest.extend(first);
|
||||||
|
let (mut singles, mut many) = collect(
|
||||||
|
rest.into_iter().chain(args),
|
||||||
|
&["state-root", "qdrant-url", "embedding-model", "forge-url", "matrix-url", "matrix-token-file", "lock-file"],
|
||||||
|
&["ollama-url", "reserve-ollama-url", "chat-exclude-room"],
|
||||||
|
)?;
|
||||||
|
let mut hosts = |name: &str| -> Result<Vec<String>> { many.remove(name).unwrap_or_default().into_iter().map(|url| http(name, url)).collect() };
|
||||||
|
let ollama_urls = hosts("ollama-url")?;
|
||||||
|
let reserve_urls = hosts("reserve-ollama-url")?;
|
||||||
|
let excluded = many.remove("chat-exclude-room").unwrap_or_default();
|
||||||
|
let take = |singles: &mut BTreeMap<String, String>, name: &str| singles.remove(name).with_context(|| format!("--{name} is required"));
|
||||||
|
let state_root = absolute("state-root", &take(&mut singles, "state-root")?)?;
|
||||||
|
let embedding_model = take(&mut singles, "embedding-model")?;
|
||||||
|
ensure!(!embedding_model.is_empty(), "--embedding-model must name a model");
|
||||||
|
ensure!(!ollama_urls.is_empty(), "at least one --ollama-url is required; repeat it for each GPU host");
|
||||||
|
let chat = chat_options(&singles, excluded)?;
|
||||||
|
let lock_file = absolute("lock-file", &take(&mut singles, "lock-file")?)?;
|
||||||
|
Ok(Command::Run(Options {
|
||||||
|
state_root,
|
||||||
|
qdrant_url: http("qdrant-url", take(&mut singles, "qdrant-url")?)?,
|
||||||
|
ollama_urls,
|
||||||
reserve_urls,
|
reserve_urls,
|
||||||
embedding_model,
|
embedding_model,
|
||||||
forge_url: url("forge-url", values.get("forge-url").cloned().unwrap_or_else(|| DEFAULT_FORGE_URL.to_owned()))?,
|
forge_url: http("forge-url", singles.remove("forge-url").unwrap_or_else(|| DEFAULT_FORGE_URL.to_owned()))?,
|
||||||
})
|
chat,
|
||||||
|
lock_file,
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Takes the job's lock for the life of the returned file. A held lock means a run (or a
|
||||||
|
/// purge request) is in progress and nothing may change: the cursors object is replaced whole.
|
||||||
|
fn take_lock(path: &Path) -> Result<File, Stop> {
|
||||||
|
let file = OpenOptions::new().create(true).truncate(false).write(true).open(path).with_context(|| format!("opening --lock-file {}", path.display())).map_err(Stop::Usage)?;
|
||||||
|
match file.try_lock() {
|
||||||
|
Ok(()) => Ok(file),
|
||||||
|
Err(TryLockError::WouldBlock) => Err(Stop::Locked),
|
||||||
|
Err(TryLockError::Error(error)) => Err(Stop::Usage(anyhow::Error::new(error).context(format!("locking --lock-file {}", path.display())))),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn execute(command: Command) -> Result<u8, Stop> {
|
||||||
|
match command {
|
||||||
|
Command::Purge { qdrant_url, lock_file, purge } => {
|
||||||
|
let _lock = take_lock(&lock_file)?;
|
||||||
|
let mut index = QdrantIndex::new(&qdrant_url);
|
||||||
|
Ok(match job::purge(&mut index, &purge) {
|
||||||
|
Ok(line) => {
|
||||||
|
println!("{line}");
|
||||||
|
0
|
||||||
|
}
|
||||||
|
Err(error) => {
|
||||||
|
eprintln!("huginn-sources: {error:#}");
|
||||||
|
1
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
Command::Run(options) => {
|
||||||
|
let _lock = take_lock(&options.lock_file)?;
|
||||||
|
let mut embedder = pool(&options.ollama_urls, &options.reserve_urls, &options.embedding_model);
|
||||||
|
let mut index = QdrantIndex::new(&options.qdrant_url);
|
||||||
|
let mut forge = HttpForge::new(&options.forge_url);
|
||||||
|
let mut matrix = options.chat.as_ref().map(|chat| HttpMatrix::new(&chat.matrix_url, &chat.token));
|
||||||
|
let chat = matrix.as_mut().zip(options.chat.as_ref()).map(|(history, chat)| Chat { history, estate: &chat.estate });
|
||||||
|
Ok(job::run(&mut embedder, &mut index, &mut forge, &options.state_root, chat, &mut |line| println!("{line}"), &mut |line| eprintln!("{line}")))
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn main() -> ExitCode {
|
fn main() -> ExitCode {
|
||||||
let options = match parse(std::env::args().skip(1)) {
|
match parse(std::env::args().skip(1), Utc::now()).map_err(Stop::Usage).and_then(execute) {
|
||||||
Ok(options) => options,
|
Ok(status) => ExitCode::from(status),
|
||||||
Err(error) => {
|
Err(Stop::Usage(error)) => {
|
||||||
eprintln!("huginn-sources: {error:#}");
|
eprintln!("huginn-sources: {error:#}");
|
||||||
return ExitCode::from(2);
|
ExitCode::from(2)
|
||||||
}
|
}
|
||||||
};
|
Err(Stop::Locked) => {
|
||||||
let mut embedder = pool(&options.ollama_urls, &options.reserve_urls, &options.embedding_model);
|
eprintln!("huginn-sources: huginn-sources is running");
|
||||||
let mut index = QdrantIndex::new(&options.qdrant_url);
|
ExitCode::from(3)
|
||||||
let mut forge = HttpForge::new(&options.forge_url);
|
|
||||||
let mut journal = |report: &huginn_daemon::sources::Report| println!("{report}");
|
|
||||||
match refresh(&mut embedder, &mut index, &mut forge, &options.state_root, &mut journal) {
|
|
||||||
Ok(refreshed) if refreshed.is_clean() => ExitCode::SUCCESS,
|
|
||||||
Ok(refreshed) => {
|
|
||||||
if !refreshed.failed.is_empty() {
|
|
||||||
eprintln!("huginn-sources: {} repos failed: {}", refreshed.failed.len(), refreshed.failed.join(", "));
|
|
||||||
}
|
|
||||||
if let Some(why) = &refreshed.retirement_withheld {
|
|
||||||
eprintln!("huginn-sources: no repo was retired: {why}");
|
|
||||||
}
|
|
||||||
ExitCode::from(1)
|
|
||||||
}
|
|
||||||
Err(error) => {
|
|
||||||
eprintln!("huginn-sources: {error:#}");
|
|
||||||
ExitCode::from(1)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
const NOW: &str = "2026-11-17T00:00:00Z";
|
||||||
|
|
||||||
|
fn args(text: &str) -> impl Iterator<Item = String> {
|
||||||
|
text.split_whitespace().map(str::to_owned).collect::<Vec<_>>().into_iter()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn now() -> DateTime<Utc> {
|
||||||
|
DateTime::parse_from_rfc3339(NOW).unwrap().with_timezone(&Utc)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn error(text: &str) -> String {
|
||||||
|
match parse(args(text), now()) {
|
||||||
|
Ok(_) => panic!("parsed: {text}"),
|
||||||
|
Err(error) => format!("{error:#}"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const BASE: &str = "--state-root /s --qdrant-url http://q --ollama-url http://o --embedding-model m";
|
||||||
|
const IDS: &str = "--chat-exclude-room !a:s --chat-exclude-room !b:s --chat-exclude-room !c:s";
|
||||||
|
|
||||||
|
fn with_token(secret: &str) -> (tempfile::TempDir, String) {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let path = dir.path().join("token");
|
||||||
|
std::fs::write(&path, secret).unwrap();
|
||||||
|
let path = path.display().to_string();
|
||||||
|
(dir, path)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn chat_flags_come_all_or_none() {
|
||||||
|
let (_dir, token) = with_token(" sekret-token\n");
|
||||||
|
let lock = "--lock-file /l";
|
||||||
|
let base = format!("{BASE} {lock}");
|
||||||
|
let full = format!("{BASE} --matrix-url http://localhost --matrix-token-file {token} {IDS} {lock}");
|
||||||
|
let Ok(Command::Run(options)) = parse(args(&full), now()) else { panic!("full chat flags must parse") };
|
||||||
|
assert_eq!(options.chat.as_ref().unwrap().token, "sekret-token", "trimmed");
|
||||||
|
assert!(matches!(parse(args(&base), now()), Ok(Command::Run(Options { chat: None, .. }))));
|
||||||
|
assert!(error(BASE).contains("--lock-file is required"), "a chat-off run needs the lock too");
|
||||||
|
assert!(error(&format!("{base} --matrix-url http://localhost")).contains("--matrix-token-file is missing"));
|
||||||
|
assert!(error(&format!("{base} --matrix-token-file {token} {IDS}")).contains("--matrix-url is missing"));
|
||||||
|
assert!(error(&format!("{base} --matrix-url http://localhost --matrix-token-file {token}")).contains("--chat-exclude-room is missing"));
|
||||||
|
assert!(error(&format!("{BASE} --matrix-url http://localhost --matrix-token-file {token} {IDS}")).contains("--lock-file is required"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn chat_flag_values_are_checked_and_never_print_the_token() {
|
||||||
|
let (_dir, token) = with_token("sekret-token");
|
||||||
|
let on = |extra: &str| format!("{BASE} --matrix-url http://localhost --matrix-token-file {token} --lock-file /l {extra}");
|
||||||
|
assert!(error(&on("--chat-exclude-room !a:s --chat-exclude-room !b:s")).contains("exactly 3"));
|
||||||
|
assert!(error(&on(&format!("{IDS} --chat-exclude-room !d:s"))).contains("exactly 3"));
|
||||||
|
assert!(error(&on("--chat-exclude-room !a:s --chat-exclude-room !a:s --chat-exclude-room !b:s")).contains("repeat"));
|
||||||
|
assert!(error(&on("--chat-exclude-room !a:s --chat-exclude-room !b:s --chat-exclude-room #c:s")).contains("beginning with !"));
|
||||||
|
assert!(error(&format!("{BASE} --matrix-url https://m --matrix-token-file {token} --lock-file /l {IDS}")).contains("plain http://"));
|
||||||
|
assert!(error(&format!("{BASE} --matrix-url http://localhost --matrix-token-file rel --lock-file /l {IDS}")).contains("absolute"));
|
||||||
|
assert!(error(&format!("{BASE} --matrix-url http://localhost --matrix-token-file {token} --lock-file rel {IDS}")).contains("absolute"));
|
||||||
|
let (_empty, empty) = with_token(" \n");
|
||||||
|
let message = error(&format!("{BASE} --matrix-url http://localhost --matrix-token-file {empty} --lock-file /l {IDS}"));
|
||||||
|
assert!(message.contains(&empty) && message.contains("empty"));
|
||||||
|
let missing = error(&format!("{BASE} --matrix-url http://localhost --matrix-token-file /nowhere/t --lock-file /l {IDS}"));
|
||||||
|
assert!(missing.contains("/nowhere/t"));
|
||||||
|
for text in [on(IDS), on("--chat-exclude-room !a:s")] {
|
||||||
|
assert!(!error(&format!("{text} --nope x")).contains("sekret-token"));
|
||||||
|
}
|
||||||
|
let dead = ["--chat", "room"].join("-");
|
||||||
|
assert!(error(&format!("{BASE} {dead} !x:s")).contains(&format!("unsupported huginn-sources option {dead}")));
|
||||||
|
}
|
||||||
|
|
||||||
|
fn purge(extra: &str) -> String {
|
||||||
|
error(&format!("chat-purge --qdrant-url http://q --lock-file /l {extra}"))
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn chat_purge_arguments() {
|
||||||
|
let good = "--room !r:s --from 2026-11-10T00:00:00Z --to 2026-11-11T00:00:00Z --request ref-1";
|
||||||
|
let Ok(Command::Purge { purge: asked, .. }) = parse(args(&format!("chat-purge --qdrant-url http://q --lock-file /l {good}")), now()) else { panic!() };
|
||||||
|
assert_eq!((asked.window.from_ms, asked.window.to_ms), (1_794_268_800_000, 1_794_355_200_000));
|
||||||
|
let bad = |from: &str, to: &str, room: &str, request: &str| purge(&format!("--room {room} --from {from} --to {to} --request {request}"));
|
||||||
|
let (f, t) = ("2026-11-10T00:00:00Z", "2026-11-11T00:00:00Z");
|
||||||
|
assert!(bad(f, t, "#a:s", "r").contains("--room"));
|
||||||
|
assert!(bad("2026-11-10T00:00:00+00:00", t, "!r:s", "r").contains("--from"));
|
||||||
|
assert!(bad(f, "2026-11-11T00:00:00", "!r:s", "r").contains("--to"));
|
||||||
|
assert!(bad(t, f, "!r:s", "r").contains("before"));
|
||||||
|
assert!(bad(f, "2026-11-18T00:00:00Z", "!r:s", "r").contains("after now"));
|
||||||
|
assert!(bad("2025-11-09T00:00:00Z", t, "!r:s", "r").contains("366"));
|
||||||
|
assert!(purge(&format!("--room !r:s --from {f} --to {t} --request {}", "x".repeat(121))).contains("--request"));
|
||||||
|
assert!(purge("--room !r:s --from 2026-11-10T00:00:00Z --to 2026-11-11T00:00:00Z").contains("--request is required"));
|
||||||
|
assert!(parse(args(&format!("chat-purge --qdrant-url http://q --lock-file /l {good} --matrix-url http://localhost")), now()).is_err(), "no Matrix flags");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn matrix_url_must_be_loopback_http() {
|
||||||
|
let (_dir, token) = with_token("sekret-token");
|
||||||
|
let with = |url: &str| format!("{BASE} --matrix-url {url} --matrix-token-file {token} --lock-file /l {IDS}");
|
||||||
|
for good in ["http://localhost", "http://127.0.0.1:8008", "http://127.9.9.9/", "http://[::1]:8008", "http://LOCALHOST:1/x"] {
|
||||||
|
assert!(parse(args(&with(good)), now()).is_ok(), "{good}");
|
||||||
|
}
|
||||||
|
for bad in ["http://matrix.example", "http://10.0.0.5:8008", "http://localhost@evil.example", "http://127.0.0.1.evil.example", "http://localhost.evil.example", "http://[::2]", "https://localhost", "http://127.0.0.1:pw@evil.example", "http://[::1].evil.example", "http://[::1]x"] {
|
||||||
|
assert!(error(&with(bad)).contains("--matrix-url"), "{bad}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_held_lock_exits_3_through_execute_for_both_modes() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let lock = dir.path().join("lock");
|
||||||
|
let held = take_lock(&lock).ok().unwrap();
|
||||||
|
let lock = lock.display().to_string();
|
||||||
|
let good = "--room !r:s --from 2026-11-10T00:00:00Z --to 2026-11-11T00:00:00Z --request r";
|
||||||
|
let purge = parse(args(&format!("chat-purge --qdrant-url http://127.0.0.1:1 --lock-file {lock} {good}")), now()).unwrap();
|
||||||
|
assert!(matches!(execute(purge), Err(Stop::Locked)));
|
||||||
|
let (_t, token) = with_token("sekret-token");
|
||||||
|
let run = parse(args(&format!("{BASE} --matrix-url http://localhost --matrix-token-file {token} {IDS} --lock-file {lock}")), now()).unwrap();
|
||||||
|
assert!(matches!(execute(run), Err(Stop::Locked)));
|
||||||
|
let off = parse(args(&format!("{BASE} --lock-file {lock}")), now()).unwrap();
|
||||||
|
assert!(matches!(execute(off), Err(Stop::Locked)), "a chat-off run must not write cursors under a held lock");
|
||||||
|
drop(held);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn the_lock_is_held_for_the_whole_run() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let lock = dir.path().join("lock");
|
||||||
|
// A qdrant that accepts the run's first request and says nothing: the run is mid-flight while we probe the lock.
|
||||||
|
let qdrant = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
|
||||||
|
let url = format!("http://{}", qdrant.local_addr().unwrap());
|
||||||
|
let run = parse(args(&format!("--state-root /s --qdrant-url {url} --ollama-url http://127.0.0.1:1 --embedding-model m --lock-file {}", lock.display())), now()).unwrap();
|
||||||
|
let worker = std::thread::spawn(move || execute(run));
|
||||||
|
let (connection, _) = qdrant.accept().unwrap();
|
||||||
|
assert!(matches!(take_lock(&lock), Err(Stop::Locked)), "the run must hold the lock while it works");
|
||||||
|
drop(connection);
|
||||||
|
// Refuse any retry the same way until the run gives up.
|
||||||
|
qdrant.set_nonblocking(true).unwrap();
|
||||||
|
while !worker.is_finished() {
|
||||||
|
drop(qdrant.accept());
|
||||||
|
std::thread::sleep(std::time::Duration::from_millis(5));
|
||||||
|
}
|
||||||
|
assert!(worker.join().unwrap().is_ok());
|
||||||
|
assert!(take_lock(&lock).is_ok(), "the lock is released when the run ends");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_held_lock_stops_the_run_untouched() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let path = dir.path().join("lock");
|
||||||
|
let held = take_lock(&path).ok().unwrap();
|
||||||
|
assert!(matches!(take_lock(&path), Err(Stop::Locked)));
|
||||||
|
drop(held);
|
||||||
|
assert!(take_lock(&path).is_ok());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
|
||||||
|
|
@ -6,9 +6,9 @@ use chrono::{DateTime, Utc};
|
||||||
use cultnet_rs::Selection;
|
use cultnet_rs::Selection;
|
||||||
use crate::sources::forge::HttpForge;
|
use crate::sources::forge::HttpForge;
|
||||||
use huginn_mind::desk_wire::{DeskRequest, DeskResponse};
|
use huginn_mind::desk_wire::{DeskRequest, DeskResponse};
|
||||||
use huginn_mind::eureka_pipeline::{PipelineRef, Slug};
|
use huginn_mind::eureka_pipeline::{AccountId, PipelineRef, Short, Slug};
|
||||||
use huginn_mind::wire::{ACTIVITY_REPOS_MAX, HuginnMindRequest, HuginnMindResponse, IndexStatus};
|
use huginn_mind::wire::{ACTIVITY_REPOS_MAX, HuginnMindRequest, HuginnMindResponse, IndexStatus};
|
||||||
use huginn_mind::{Members, Mind, MindRefusal, MindStore, PipelineAdmissionOutcome, SemanticQuery, SourceHits, SourceQuery};
|
use huginn_mind::{ChatHits, ChatQuery, ChatWindow, ChatWindowQuery, Members, Mind, MindRefusal, MindStore, PipelineAdmissionOutcome, SemanticQuery, SourceHits, SourceQuery};
|
||||||
|
|
||||||
/// Where the index attaches. Called after a batch commits, with the refs the
|
/// Where the index attaches. Called after a batch commits, with the refs the
|
||||||
/// batch landed, derived writes included; the sink may read each through
|
/// batch landed, derived writes included; the sink may read each through
|
||||||
|
|
@ -34,8 +34,14 @@ pub trait IndexSink<S: MindStore> {
|
||||||
/// Starts a source search. The source index is not the mind's: it needs no
|
/// Starts a source search. The source index is not the mind's: it needs no
|
||||||
/// member ids, and nothing about the mind's own index state refuses it.
|
/// member ids, and nothing about the mind's own index state refuses it.
|
||||||
fn search_sources(&mut self, query: &SourceQuery) -> Result<SearchTicket, String>;
|
fn search_sources(&mut self, query: &SourceQuery) -> Result<SearchTicket, String>;
|
||||||
/// Every source search that finished since the last call, by ticket.
|
/// Starts a chat search: the same queue and worker as a source search,
|
||||||
fn sources_searched(&mut self) -> Vec<(SearchTicket, Result<SourceHits, String>)>;
|
/// over the same collection.
|
||||||
|
fn search_chat(&mut self, query: &ChatQuery) -> Result<SearchTicket, String>;
|
||||||
|
/// Starts a chat window: the same queue and worker, no embedding.
|
||||||
|
fn chat_window(&mut self, query: &ChatWindowQuery) -> Result<SearchTicket, String>;
|
||||||
|
/// Every source or chat search that finished since the last call, by
|
||||||
|
/// ticket: `Found::Sources`, `Found::Chat` or `Found::Failed`.
|
||||||
|
fn sources_searched(&mut self) -> Vec<(SearchTicket, Found)>;
|
||||||
fn search_deadline(&self) -> std::time::Duration;
|
fn search_deadline(&self) -> std::time::Duration;
|
||||||
/// Every search that finished since the last call, by ticket: its
|
/// Every search that finished since the last call, by ticket: its
|
||||||
/// candidates and scores, or why it failed.
|
/// candidates and scores, or why it failed.
|
||||||
|
|
@ -78,6 +84,10 @@ struct MindSearch {
|
||||||
pub enum Found {
|
pub enum Found {
|
||||||
Mind(Hits),
|
Mind(Hits),
|
||||||
Sources(SourceHits),
|
Sources(SourceHits),
|
||||||
|
Chat(ChatHits),
|
||||||
|
Window(ChatWindow),
|
||||||
|
/// The index answered with a typed refusal rather than failing.
|
||||||
|
Refused(MindRefusal),
|
||||||
Failed(String),
|
Failed(String),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -99,6 +109,15 @@ impl Handled {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// The door checks an account id; the Mind's desk calls take it as the text it is.
|
||||||
|
fn short_id(id: AccountId) -> Short {
|
||||||
|
Short(id.0)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn short_ids(ids: &[AccountId]) -> Vec<Short> {
|
||||||
|
ids.iter().map(|id| Short(id.0.clone())).collect()
|
||||||
|
}
|
||||||
|
|
||||||
/// What the log says of the hits the mind held no document for: how many, and
|
/// What the log says of the hits the mind held no document for: how many, and
|
||||||
/// the first few by name (a restored mind can leave hundreds), or nothing when
|
/// the first few by name (a restored mind can leave hundreds), or nothing when
|
||||||
/// the mind held them all.
|
/// the mind held them all.
|
||||||
|
|
@ -205,6 +224,24 @@ impl<S: MindStore, I: IndexSink<S>> Daemon<S, I> {
|
||||||
Ok(roots) => HuginnMindResponse::RootsTouched(roots),
|
Ok(roots) => HuginnMindResponse::RootsTouched(roots),
|
||||||
Err(refusal) => HuginnMindResponse::Refused(refusal),
|
Err(refusal) => HuginnMindResponse::Refused(refusal),
|
||||||
},
|
},
|
||||||
|
HuginnMindRequest::SearchChat { query, .. } => {
|
||||||
|
if let Err(refusal) = query.check() {
|
||||||
|
return Handled::Answered(HuginnMindResponse::Refused(refusal));
|
||||||
|
}
|
||||||
|
return match self.index.search_chat(&query) {
|
||||||
|
Ok(ticket) => Handled::Searching(Search { ticket, mind: None }),
|
||||||
|
Err(detail) => Handled::Answered(HuginnMindResponse::Refused(MindRefusal::Unavailable { detail })),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
HuginnMindRequest::ChatWindow { query, .. } => {
|
||||||
|
if let Err(refusal) = query.check() {
|
||||||
|
return Handled::Answered(HuginnMindResponse::Refused(refusal));
|
||||||
|
}
|
||||||
|
return match self.index.chat_window(&query) {
|
||||||
|
Ok(ticket) => Handled::Searching(Search { ticket, mind: None }),
|
||||||
|
Err(detail) => Handled::Answered(HuginnMindResponse::Refused(MindRefusal::Unavailable { detail })),
|
||||||
|
};
|
||||||
|
}
|
||||||
HuginnMindRequest::StewardedActivity { instance, repos, after } => {
|
HuginnMindRequest::StewardedActivity { instance, repos, after } => {
|
||||||
let refused = |message: &str| MindRefusal::SelectionInvalid { field: "repos".into(), value: None, message: message.into() };
|
let refused = |message: &str| MindRefusal::SelectionInvalid { field: "repos".into(), value: None, message: message.into() };
|
||||||
if repos.len() > ACTIVITY_REPOS_MAX {
|
if repos.len() > ACTIVITY_REPOS_MAX {
|
||||||
|
|
@ -235,18 +272,18 @@ impl<S: MindStore, I: IndexSink<S>> Daemon<S, I> {
|
||||||
/// One desk request, answered by the Mind method it names. The Mind is the instance's own
|
/// One desk request, answered by the Mind method it names. The Mind is the instance's own
|
||||||
/// (`mind` is checked against it); `deciders` is this call's list, handed to the Mind and
|
/// (`mind` is checked against it); `deciders` is this call's list, handed to the Mind and
|
||||||
/// dropped. A desk answer is not indexed (`index_text` gives it no text), so nothing here tells the index.
|
/// dropped. A desk answer is not indexed (`index_text` gives it no text), so nothing here tells the index.
|
||||||
/// The reads answer with the whole record and nothing here gates them (ruling
|
/// The reads are handed `subject` and `deciders`: the Mind builds a decider's view or a
|
||||||
/// `desk-read-visibility-whole-record`): they are handed `deciders` and not `subject`.
|
/// non-decider's (ruling `desk-read-visibility-withheld-ids`).
|
||||||
pub fn handle_desk(&mut self, request: DeskRequest, now: DateTime<Utc>) -> DeskResponse {
|
pub fn handle_desk(&mut self, request: DeskRequest, now: DateTime<Utc>) -> DeskResponse {
|
||||||
if let Err(refusal) = self.mind.require_instance(request.mind()) {
|
if let Err(refusal) = self.mind.require_instance(request.mind()) {
|
||||||
return DeskResponse::Refused(refusal);
|
return DeskResponse::Refused(refusal);
|
||||||
}
|
}
|
||||||
match request {
|
match request {
|
||||||
DeskRequest::Answer { subject, authority, deciders, request, .. } => {
|
DeskRequest::Answer { subject, authority, deciders, request, .. } => {
|
||||||
self.mind.desk_answer(&deciders, subject, authority, request, now)
|
self.mind.desk_answer(&short_ids(&deciders), short_id(subject), authority, request, now)
|
||||||
}
|
}
|
||||||
DeskRequest::Forks { deciders, status, cursor, limit, .. } => self.mind.desk_forks(&deciders, status, cursor, limit),
|
DeskRequest::Forks { subject, deciders, status, cursor, limit, .. } => self.mind.desk_forks(&short_id(subject), &short_ids(&deciders), status, cursor, limit),
|
||||||
DeskRequest::Fork { deciders, fork_id, .. } => self.mind.desk_fork(&deciders, &fork_id),
|
DeskRequest::Fork { subject, deciders, fork_id, .. } => self.mind.desk_fork(&short_id(subject), &short_ids(&deciders), &fork_id),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -282,8 +319,7 @@ impl<S: MindStore, I: IndexSink<S>> Daemon<S, I> {
|
||||||
/// Searches the index has finished since this was last called.
|
/// Searches the index has finished since this was last called.
|
||||||
pub fn searched(&mut self) -> Vec<(SearchTicket, Found)> {
|
pub fn searched(&mut self) -> Vec<(SearchTicket, Found)> {
|
||||||
let mind = self.index.searched().into_iter().map(|(ticket, found)| (ticket, found.map_or_else(Found::Failed, Found::Mind)));
|
let mind = self.index.searched().into_iter().map(|(ticket, found)| (ticket, found.map_or_else(Found::Failed, Found::Mind)));
|
||||||
let sources = self.index.sources_searched().into_iter().map(|(ticket, found)| (ticket, found.map_or_else(Found::Failed, Found::Sources)));
|
mind.chain(self.index.sources_searched()).collect()
|
||||||
mind.chain(sources).collect()
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// The client has stopped waiting for `ticket`.
|
/// The client has stopped waiting for `ticket`.
|
||||||
|
|
@ -300,6 +336,9 @@ impl<S: MindStore, I: IndexSink<S>> Daemon<S, I> {
|
||||||
let (selection, semantic, members, hits) = match (search.mind, found) {
|
let (selection, semantic, members, hits) = match (search.mind, found) {
|
||||||
(_, Found::Failed(detail)) => return unavailable(detail),
|
(_, Found::Failed(detail)) => return unavailable(detail),
|
||||||
(None, Found::Sources(hits)) => return HuginnMindResponse::Sources(hits),
|
(None, Found::Sources(hits)) => return HuginnMindResponse::Sources(hits),
|
||||||
|
(None, Found::Chat(hits)) => return HuginnMindResponse::Chat(hits),
|
||||||
|
(None, Found::Window(window)) => return HuginnMindResponse::ChatWindow(window),
|
||||||
|
(None, Found::Refused(refusal)) => return HuginnMindResponse::Refused(refusal),
|
||||||
(Some(asked), Found::Mind(hits)) => (asked.selection, asked.semantic, asked.members, hits),
|
(Some(asked), Found::Mind(hits)) => (asked.selection, asked.semantic, asked.members, hits),
|
||||||
_ => return unavailable("the index answered another kind of search than the one asked".into()),
|
_ => return unavailable("the index answered another kind of search than the one asked".into()),
|
||||||
};
|
};
|
||||||
|
|
@ -326,6 +365,7 @@ pub(crate) mod tests {
|
||||||
StructuralDelta, Title, VerificationTest,
|
StructuralDelta, Title, VerificationTest,
|
||||||
};
|
};
|
||||||
use huginn_mind::wire::MindStatus;
|
use huginn_mind::wire::MindStatus;
|
||||||
|
use huginn_mind::ChatQuery;
|
||||||
use huginn_mind::{MindRefusal, OwnedRedbMessagePackBackingStore};
|
use huginn_mind::{MindRefusal, OwnedRedbMessagePackBackingStore};
|
||||||
use std::path::Path;
|
use std::path::Path;
|
||||||
use cultnet_rs::Selection;
|
use cultnet_rs::Selection;
|
||||||
|
|
@ -580,11 +620,19 @@ pub(crate) mod tests {
|
||||||
Err("this index cannot search".into())
|
Err("this index cannot search".into())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn search_chat(&mut self, _query: &ChatQuery) -> Result<SearchTicket, String> {
|
||||||
|
Err("this index cannot search".into())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn chat_window(&mut self, _query: &ChatWindowQuery) -> Result<SearchTicket, String> {
|
||||||
|
Err("this index cannot search".into())
|
||||||
|
}
|
||||||
|
|
||||||
fn search_sources(&mut self, _query: &SourceQuery) -> Result<SearchTicket, String> {
|
fn search_sources(&mut self, _query: &SourceQuery) -> Result<SearchTicket, String> {
|
||||||
Err("this index cannot search".into())
|
Err("this index cannot search".into())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn sources_searched(&mut self) -> Vec<(SearchTicket, Result<SourceHits, String>)> {
|
fn sources_searched(&mut self) -> Vec<(SearchTicket, Found)> {
|
||||||
Vec::new()
|
Vec::new()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -619,13 +667,21 @@ pub(crate) mod tests {
|
||||||
Err("this index cannot search the mind".into())
|
Err("this index cannot search the mind".into())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn search_chat(&mut self, _query: &ChatQuery) -> Result<SearchTicket, String> {
|
||||||
|
Err("this index cannot search".into())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn chat_window(&mut self, _query: &ChatWindowQuery) -> Result<SearchTicket, String> {
|
||||||
|
Err("this index cannot search".into())
|
||||||
|
}
|
||||||
|
|
||||||
fn search_sources(&mut self, _query: &SourceQuery) -> Result<SearchTicket, String> {
|
fn search_sources(&mut self, _query: &SourceQuery) -> Result<SearchTicket, String> {
|
||||||
*self.asked.lock().unwrap() += 1;
|
*self.asked.lock().unwrap() += 1;
|
||||||
Ok(SearchTicket(7))
|
Ok(SearchTicket(7))
|
||||||
}
|
}
|
||||||
|
|
||||||
fn sources_searched(&mut self) -> Vec<(SearchTicket, Result<SourceHits, String>)> {
|
fn sources_searched(&mut self) -> Vec<(SearchTicket, Found)> {
|
||||||
vec![(SearchTicket(7), Ok(self.hits.clone()))]
|
vec![(SearchTicket(7), Found::Sources(self.hits.clone()))]
|
||||||
}
|
}
|
||||||
|
|
||||||
fn search_deadline(&self) -> std::time::Duration {
|
fn search_deadline(&self) -> std::time::Duration {
|
||||||
|
|
@ -685,6 +741,31 @@ pub(crate) mod tests {
|
||||||
assert_eq!(*asked.lock().unwrap(), 3, "a refused query never reached the index");
|
assert_eq!(*asked.lock().unwrap(), 3, "a refused query never reached the index");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// A chat search is checked at the door before the index is asked, is
|
||||||
|
/// refused for another mind, turns an index that cannot be asked into
|
||||||
|
/// `Unavailable`, and its finished hits are the answer as they are.
|
||||||
|
#[test]
|
||||||
|
fn a_chat_search_is_checked_at_the_door_and_answers_its_hits() {
|
||||||
|
use huginn_mind::eureka_pipeline::{Line, Short};
|
||||||
|
use huginn_mind::{ChatHits, SourceModel};
|
||||||
|
let root = tempfile::tempdir().unwrap();
|
||||||
|
let mut daemon = Daemon::new(Mind::open(root.path(), &slug(INSTANCE)).unwrap(), NoIndex);
|
||||||
|
let request = |instance: &str, rooms: &[&str], top_k: u32| HuginnMindRequest::SearchChat {
|
||||||
|
instance: slug(instance),
|
||||||
|
query: ChatQuery { texts: vec![Line("find".into())], rooms: rooms.iter().map(|room| Short((*room).into())).collect(), after_ms: None, before_ms: None, top_k },
|
||||||
|
};
|
||||||
|
for (refused, field) in [(request(INSTANCE, &[], 0), "top_k"), (request(INSTANCE, &["swarm"], 5), "rooms")] {
|
||||||
|
let answer = daemon.handle(refused, now()).answered();
|
||||||
|
assert!(matches!(answer, HuginnMindResponse::Refused(MindRefusal::SelectionInvalid { field: ref named, .. }) if named == field), "{answer:?}");
|
||||||
|
}
|
||||||
|
assert!(matches!(daemon.handle(request(OTHER, &[], 5), now()).answered(), HuginnMindResponse::Refused(_)));
|
||||||
|
let answer = daemon.handle(request(INSTANCE, &["#swarm:x"], 5), now()).answered();
|
||||||
|
assert!(matches!(answer, HuginnMindResponse::Refused(MindRefusal::Unavailable { .. })), "{answer:?}");
|
||||||
|
let hits = ChatHits { indexed_under: SourceModel { name: "m".into(), digest: "d".into() }, truncated: false, hits: vec![] };
|
||||||
|
let search = Search { ticket: SearchTicket(1), mind: None };
|
||||||
|
assert_eq!(daemon.finish(search, Found::Chat(hits.clone())), HuginnMindResponse::Chat(hits));
|
||||||
|
}
|
||||||
|
|
||||||
/// A daemon over the instance's mind under `state_root`, with no index.
|
/// A daemon over the instance's mind under `state_root`, with no index.
|
||||||
pub(crate) fn open_unindexed(
|
pub(crate) fn open_unindexed(
|
||||||
state_root: &Path,
|
state_root: &Path,
|
||||||
|
|
@ -716,11 +797,19 @@ pub(crate) mod tests {
|
||||||
Err("this index cannot search".into())
|
Err("this index cannot search".into())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn search_chat(&mut self, _query: &ChatQuery) -> Result<SearchTicket, String> {
|
||||||
|
Err("this index cannot search".into())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn chat_window(&mut self, _query: &ChatWindowQuery) -> Result<SearchTicket, String> {
|
||||||
|
Err("this index cannot search".into())
|
||||||
|
}
|
||||||
|
|
||||||
fn search_sources(&mut self, _query: &SourceQuery) -> Result<SearchTicket, String> {
|
fn search_sources(&mut self, _query: &SourceQuery) -> Result<SearchTicket, String> {
|
||||||
Err("this index cannot search".into())
|
Err("this index cannot search".into())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn sources_searched(&mut self) -> Vec<(SearchTicket, Result<SourceHits, String>)> {
|
fn sources_searched(&mut self) -> Vec<(SearchTicket, Found)> {
|
||||||
Vec::new()
|
Vec::new()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -750,11 +839,19 @@ pub(crate) mod tests {
|
||||||
Err("this index cannot search".into())
|
Err("this index cannot search".into())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn search_chat(&mut self, _query: &ChatQuery) -> Result<SearchTicket, String> {
|
||||||
|
Err("this index cannot search".into())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn chat_window(&mut self, _query: &ChatWindowQuery) -> Result<SearchTicket, String> {
|
||||||
|
Err("this index cannot search".into())
|
||||||
|
}
|
||||||
|
|
||||||
fn search_sources(&mut self, _query: &SourceQuery) -> Result<SearchTicket, String> {
|
fn search_sources(&mut self, _query: &SourceQuery) -> Result<SearchTicket, String> {
|
||||||
Err("this index cannot search".into())
|
Err("this index cannot search".into())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn sources_searched(&mut self) -> Vec<(SearchTicket, Result<SourceHits, String>)> {
|
fn sources_searched(&mut self) -> Vec<(SearchTicket, Found)> {
|
||||||
Vec::new()
|
Vec::new()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -10,16 +10,22 @@
|
||||||
//! deadline like one that is silent), a reply past `DESK_MAX_RESPONSE_BYTES` refused by name (a read of one
|
//! deadline like one that is silent), a reply past `DESK_MAX_RESPONSE_BYTES` refused by name (a read of one
|
||||||
//! fork past it is that fork's `fork_too_large`).
|
//! fork past it is that fork's `fork_too_large`).
|
||||||
//!
|
//!
|
||||||
//! The trust boundary of the reads (ruling `desk-read-visibility-whole-record`). The peer-uid check
|
//! The trust boundary of the reads (ruling `desk-read-visibility-withheld-ids`). The peer-uid check
|
||||||
//! IS the whole authentication: Huginn must be reachable only by Bifrost's peer-checked socket, so
|
//! IS the whole authentication: Huginn must be reachable only by Bifrost's peer-checked socket, so
|
||||||
//! the configured uid is Bifrost's and no other process's. `Forks` and `Fork` return the WHOLE record
|
//! the configured uid is Bifrost's and no other process's. `Forks` and `Fork` carry every decider's
|
||||||
//! (cards, outcomes, every decider's note) to whoever passes that check; nothing in Huginn strips a
|
//! note to whoever passes that check, and Bifrost strips notes at its own door (a stripped read
|
||||||
//! note or gates a read on `subject` or `deciders` (`deciders` only fills the card and the
|
//! serves `note: null`, CultLib `fork-card.md`); a second reader of this socket is therefore a note
|
||||||
//! showability gate). Bifrost applies visibility at its own door (a stripped read serves `note: null`,
|
//! leak, and a deployment that gives the socket to any uid but Bifrost's has no reader check left.
|
||||||
//! CultLib `fork-card.md`). A second reader of this socket is therefore a note leak, and a deployment
|
//! Account ids are Huginn's to withhold: a reader whose `subject` is not in the call's `deciders`
|
||||||
//! that gives the socket to any uid but Bifrost's has no reader check left. The reads carry no
|
//! gets `deciders` empty, `answerer` nil and the counts instead. The reads carry no `authority` (a
|
||||||
//! `authority` (a payload that names one is refused whole), and the door has no other way to ask who
|
//! payload that names one is refused whole), and the door has no other way to ask who is reading.
|
||||||
//! is reading.
|
//!
|
||||||
|
//! The listing gate: a reader who is not a decider sees a fork only while a `desk_listing` for it is
|
||||||
|
//! in force, and a fork it may not see reads `fork_unknown`, the same bytes as a missing one, on
|
||||||
|
//! every desk operation, a replay of a stored answer included. The door never writes a listing; it
|
||||||
|
//! arrives by the RUDP Admit door only. Unlisting is a `Withdrawn` resolution on the listing, and
|
||||||
|
//! admitting the listing again after that returns `AlreadyAdmitted` and the fork stays hidden:
|
||||||
|
//! relisting is withdrawing the withdrawal, not a second admission.
|
||||||
//!
|
//!
|
||||||
//! It owns no rule. Which Mind method answers is `Daemon::handle_desk`'s; when the door is
|
//! It owns no rule. Which Mind method answers is `Daemon::handle_desk`'s; when the door is
|
||||||
//! served is `serve::run`'s, once per pass in the thread that owns the Mind. A peer that is
|
//! served is `serve::run`'s, once per pass in the thread that owns the Mind. A peer that is
|
||||||
|
|
|
||||||
|
|
@ -20,7 +20,7 @@ use cultnet_rs::{
|
||||||
decode_cultnet_message_from_slice, encode_cultnet_message_to_vec, encode_frame,
|
decode_cultnet_message_from_slice, encode_cultnet_message_to_vec, encode_frame,
|
||||||
};
|
};
|
||||||
use huginn_mind::desk_wire::{
|
use huginn_mind::desk_wire::{
|
||||||
DESK_MAX_RESPONSE_BYTES, DESK_REQUEST_SCHEMA, DESK_SERVICE_ID, DeskRequest, DeskResponse, ForkAnswer, ForkAnswerCode, ForkAnswerRequest, ForkCursor,
|
DESK_REQUEST_SCHEMA, DESK_SERVICE_ID, DeskRequest, DeskResponse, ForkAnswer, ForkAnswerCode, ForkAnswerRequest, ForkCursor,
|
||||||
ForkEntry, ForkLimit, ForkList, ListStatus, RequestSchema,
|
ForkEntry, ForkLimit, ForkList, ListStatus, RequestSchema,
|
||||||
};
|
};
|
||||||
use huginn_mind::envelope::{OperationFailure, decode_desk_response, decode_response, encode_desk_request, encode_request};
|
use huginn_mind::envelope::{OperationFailure, decode_desk_response, decode_response, encode_desk_request, encode_request};
|
||||||
|
|
@ -85,20 +85,26 @@ fn wire(hash: &str, request_id: &str) -> ForkAnswerRequest {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// A test's name for someone as the Bifrost account id the door requires: a deterministic lowercase D-format UUID.
|
||||||
|
fn acct(name: &str) -> huginn_mind::eureka_pipeline::AccountId {
|
||||||
|
let hash = name.bytes().fold(0xcbf2_9ce4_8422_2325_u64, |hash, byte| (hash ^ u64::from(byte)).wrapping_mul(0x0100_0000_01b3));
|
||||||
|
huginn_mind::eureka_pipeline::AccountId(format!("{:08x}-{:04x}-4000-8000-{:012x}", hash >> 32, (hash >> 16) & 0xffff, hash & 0xffff_ffff_ffff))
|
||||||
|
}
|
||||||
|
|
||||||
const R1: &str = "00000000-0000-4000-8000-000000000001";
|
const R1: &str = "00000000-0000-4000-8000-000000000001";
|
||||||
const R2: &str = "00000000-0000-4000-8000-000000000002";
|
const R2: &str = "00000000-0000-4000-8000-000000000002";
|
||||||
const R3: &str = "00000000-0000-4000-8000-000000000003";
|
const R3: &str = "00000000-0000-4000-8000-000000000003";
|
||||||
|
|
||||||
fn answer(who: &str, authority: DeskAuthority, deciders: &[&str], request: ForkAnswerRequest) -> DeskRequest {
|
fn answer(who: &str, authority: DeskAuthority, deciders: &[&str], request: ForkAnswerRequest) -> DeskRequest {
|
||||||
DeskRequest::Answer { mind: slug(INSTANCE), subject: short(who), authority, deciders: deciders.iter().map(|d| short(d)).collect(), request }
|
DeskRequest::Answer { mind: slug(INSTANCE), subject: acct(who), authority, deciders: deciders.iter().map(|d| acct(d)).collect(), request }
|
||||||
}
|
}
|
||||||
|
|
||||||
/// A list read by `who` against `deciders`.
|
/// A list read by `who` against `deciders`.
|
||||||
fn forks(who: &str, deciders: &[&str], status: Option<ListStatus>, cursor: Option<ForkCursor>, limit: u8) -> DeskRequest {
|
fn forks(who: &str, deciders: &[&str], status: Option<ListStatus>, cursor: Option<ForkCursor>, limit: u8) -> DeskRequest {
|
||||||
DeskRequest::Forks {
|
DeskRequest::Forks {
|
||||||
mind: slug(INSTANCE),
|
mind: slug(INSTANCE),
|
||||||
subject: short(who),
|
subject: acct(who),
|
||||||
deciders: deciders.iter().map(|d| short(d)).collect(),
|
deciders: deciders.iter().map(|d| acct(d)).collect(),
|
||||||
status,
|
status,
|
||||||
cursor,
|
cursor,
|
||||||
limit: ForkLimit::new(limit).unwrap(),
|
limit: ForkLimit::new(limit).unwrap(),
|
||||||
|
|
@ -110,7 +116,7 @@ fn forks_all(deciders: &[&str]) -> DeskRequest {
|
||||||
}
|
}
|
||||||
|
|
||||||
fn fork_read(who: &str, deciders: &[&str], fork_id: &str) -> DeskRequest {
|
fn fork_read(who: &str, deciders: &[&str], fork_id: &str) -> DeskRequest {
|
||||||
DeskRequest::Fork { mind: slug(INSTANCE), subject: short(who), deciders: deciders.iter().map(|d| short(d)).collect(), fork_id: fork_id.into() }
|
DeskRequest::Fork { mind: slug(INSTANCE), subject: acct(who), deciders: deciders.iter().map(|d| acct(d)).collect(), fork_id: fork_id.into() }
|
||||||
}
|
}
|
||||||
|
|
||||||
fn page_of(response: DeskResponse) -> (ForkList, u64) {
|
fn page_of(response: DeskResponse) -> (ForkList, u64) {
|
||||||
|
|
@ -205,6 +211,13 @@ fn rig(peer_uid: u32) -> Rig {
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Rig {
|
impl Rig {
|
||||||
|
/// The Mind lists the seeded fork, on its own door, so a non-decider reaches it.
|
||||||
|
fn list_the_fork(&mut self) {
|
||||||
|
let listing = PipelineDocument::DeskListing(huginn_mind::eureka_pipeline::PipelineDeskListing { campaign: slug("eureka-state"), listed: short(FORK), reason: "Listed.".into() });
|
||||||
|
let admitted = self.daemon.handle(HuginnMindRequest::Admit(batch(INSTANCE, vec![listing])), now()).answered();
|
||||||
|
assert!(matches!(admitted, HuginnMindResponse::Admit(huginn_mind::PipelineAdmissionOutcome::Committed { .. })), "{admitted:?}");
|
||||||
|
}
|
||||||
|
|
||||||
/// Runs `client` on its own thread while this one serves the door, then drains what is still waiting.
|
/// Runs `client` on its own thread while this one serves the door, then drains what is still waiting.
|
||||||
fn serving<T: Send>(&mut self, client: impl FnOnce(&Path) -> T + Send) -> T {
|
fn serving<T: Send>(&mut self, client: impl FnOnce(&Path) -> T + Send) -> T {
|
||||||
let path = self.path.clone();
|
let path = self.path.clone();
|
||||||
|
|
@ -567,7 +580,7 @@ mod door_smoke {
|
||||||
claim_in_future: false,
|
claim_in_future: false,
|
||||||
request_id: RequestId(R1.into()),
|
request_id: RequestId(R1.into()),
|
||||||
body_sha256: Sha256Hex("0".repeat(64)),
|
body_sha256: Sha256Hex("0".repeat(64)),
|
||||||
answerer: short("x"),
|
answerer: acct("x"),
|
||||||
authority: DeskAuthority::Operator,
|
authority: DeskAuthority::Operator,
|
||||||
outcome: DeskOutcome::Counted,
|
outcome: DeskOutcome::Counted,
|
||||||
});
|
});
|
||||||
|
|
@ -601,6 +614,7 @@ mod door_smoke {
|
||||||
#[test]
|
#[test]
|
||||||
fn deciders_ride_each_request() {
|
fn deciders_ride_each_request() {
|
||||||
let mut rig = rig(my_uid());
|
let mut rig = rig(my_uid());
|
||||||
|
rig.list_the_fork();
|
||||||
let (before, hash) = (rig.state(), rig.hash.clone());
|
let (before, hash) = (rig.state(), rig.hash.clone());
|
||||||
let out = rig.serving(|path| {
|
let out = rig.serving(|path| {
|
||||||
let ask = |id: &str, who: &str, deciders: &[&str], request_id: &str| {
|
let ask = |id: &str, who: &str, deciders: &[&str], request_id: &str| {
|
||||||
|
|
@ -776,6 +790,7 @@ mod door_smoke {
|
||||||
#[test]
|
#[test]
|
||||||
fn a_mind_refusal_and_a_stored_refusal_are_typed_replies() {
|
fn a_mind_refusal_and_a_stored_refusal_are_typed_replies() {
|
||||||
let mut rig = rig(my_uid());
|
let mut rig = rig(my_uid());
|
||||||
|
rig.list_the_fork();
|
||||||
let (before, hash) = (rig.state(), rig.hash.clone());
|
let (before, hash) = (rig.state(), rig.hash.clone());
|
||||||
let mut foreign = answer("x", DeskAuthority::Member, &["x"], wire(&hash, R1));
|
let mut foreign = answer("x", DeskAuthority::Member, &["x"], wire(&hash, R1));
|
||||||
let DeskRequest::Answer { mind, .. } = &mut foreign else { unreachable!() };
|
let DeskRequest::Answer { mind, .. } = &mut foreign else { unreachable!() };
|
||||||
|
|
@ -801,7 +816,7 @@ mod door_smoke {
|
||||||
assert_eq!((listed_at, read_at), (before.1 as u64, before.1 as u64), "both read at the head the mind held");
|
assert_eq!((listed_at, read_at), (before.1 as u64, before.1 as u64), "both read at the head the mind held");
|
||||||
assert_eq!(list.stamp, format!("s{listed_at}"));
|
assert_eq!(list.stamp, format!("s{listed_at}"));
|
||||||
assert_eq!(list.forks[0], entry);
|
assert_eq!(list.forks[0], entry);
|
||||||
assert_eq!((entry.card.fork_id.as_str(), entry.card.options_hash.as_str(), entry.card.deciders.as_slice()), (FORK, hash.as_str(), ["x".to_string(), "y".to_string()].as_slice()));
|
assert_eq!((entry.card.fork_id.as_str(), entry.card.options_hash.as_str(), entry.card.deciders.as_slice()), (FORK, hash.as_str(), [acct("x").0, acct("y").0].as_slice()));
|
||||||
assert_eq!(rig.state(), before, "a read stores nothing");
|
assert_eq!(rig.state(), before, "a read stores nothing");
|
||||||
assert!(rig.index.seen.lock().unwrap().is_empty());
|
assert!(rig.index.seen.lock().unwrap().is_empty());
|
||||||
// An unknown fork is the answer path's refusal.
|
// An unknown fork is the answer path's refusal.
|
||||||
|
|
@ -855,29 +870,43 @@ mod door_smoke {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Ruling `desk-read-visibility-whole-record`: Huginn returns the whole record, notes included, to
|
/// Ruling `desk-read-visibility-withheld-ids`: a subject the call's deciders do not name gets counts
|
||||||
/// whoever the socket carries. The subject is not in the deciders; the decider's note is there.
|
/// and no account id, over the socket too; notes stay whole (Bifrost strips them). A decider gets the ids.
|
||||||
#[test]
|
#[test]
|
||||||
fn a_subject_not_in_deciders_reads_the_whole_record_with_the_notes() {
|
fn a_subject_not_in_deciders_gets_counts_and_no_account_id() {
|
||||||
let mut rig = rig(my_uid());
|
let mut rig = rig(my_uid());
|
||||||
let hash = rig.hash.clone();
|
let hash = rig.hash.clone();
|
||||||
let note = "Keep it local: the author's call.";
|
let note = "Keep it local: the author's call.";
|
||||||
rig.serving(|path| counted(reply_of(path, "m-1", &DeskRequest::Answer {
|
rig.serving(|path| counted(reply_of(path, "m-1", &DeskRequest::Answer {
|
||||||
mind: slug(INSTANCE),
|
mind: slug(INSTANCE),
|
||||||
subject: short("y"),
|
subject: acct("y"),
|
||||||
authority: DeskAuthority::Member,
|
authority: DeskAuthority::Member,
|
||||||
deciders: vec![short("y")],
|
deciders: vec![acct("y")],
|
||||||
request: noted(&hash, R1, note),
|
request: noted(&hash, R1, note),
|
||||||
})));
|
})));
|
||||||
|
// Unlisted: the outsider is shown nothing at all, and a read is `fork_unknown`.
|
||||||
|
let (hidden, unknown) = rig.serving(|path| (reply_of(path, "m-1", &forks("outsider", &["x"], None, None, 50)), reply_of(path, "m-1b", &fork_read("outsider", &["x"], FORK))));
|
||||||
|
assert_eq!(page_of(hidden).0.forks.len(), 0);
|
||||||
|
assert!(matches!(unknown, DeskResponse::NotStored(_)), "{unknown:?}");
|
||||||
|
// The Mind lists it, on its own door.
|
||||||
|
let listing = PipelineDocument::DeskListing(huginn_mind::eureka_pipeline::PipelineDeskListing { campaign: slug("eureka-state"), listed: short(FORK), reason: "Members may read why.".into() });
|
||||||
|
let admitted = rig.daemon.handle(HuginnMindRequest::Admit(batch(INSTANCE, vec![listing])), now()).answered();
|
||||||
|
assert!(matches!(admitted, HuginnMindResponse::Admit(huginn_mind::PipelineAdmissionOutcome::Committed { .. })), "{admitted:?}");
|
||||||
let (list, read) = rig.serving(|path| (reply_of(path, "m-2", &forks("outsider", &["x"], None, None, 50)), reply_of(path, "m-3", &fork_read("outsider", &["x"], FORK))));
|
let (list, read) = rig.serving(|path| (reply_of(path, "m-2", &forks("outsider", &["x"], None, None, 50)), reply_of(path, "m-3", &fork_read("outsider", &["x"], FORK))));
|
||||||
let (list, _) = page_of(list);
|
let (list, _) = page_of(list);
|
||||||
let (read, _) = entry_of(read);
|
let (read, _) = entry_of(read);
|
||||||
for entry in [&list.forks[0], &read] {
|
for entry in [&list.forks[0], &read] {
|
||||||
assert_eq!(entry.record.answers.len(), 1);
|
assert_eq!(entry.record.answers.len(), 1);
|
||||||
assert_eq!(entry.record.answers[0].note.as_deref(), Some(note));
|
assert_eq!(entry.record.answers[0].note.as_deref(), Some(note));
|
||||||
assert_eq!(entry.record.answers[0].answerer, "y");
|
assert_eq!(entry.record.answers[0].answerer, None);
|
||||||
assert_eq!(entry.card.deciders, ["x"], "the list is the call's, and only fills the card");
|
assert_eq!(entry.record.answerer_count, 1);
|
||||||
|
assert!(entry.card.deciders.is_empty());
|
||||||
|
assert_eq!(entry.card.decider_count, 1);
|
||||||
}
|
}
|
||||||
|
let seen = rig.serving(|path| reply_of(path, "m-4", &fork_read("x", &["x"], FORK)));
|
||||||
|
let (seen, _) = entry_of(seen);
|
||||||
|
assert_eq!(seen.card.deciders, [acct("x").0]);
|
||||||
|
assert_eq!(seen.record.answers[0].answerer.as_deref(), Some(acct("y").0.as_str()));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|
@ -896,7 +925,7 @@ mod door_smoke {
|
||||||
let refused = rig.serving(|path| failure_of(path, &bytes));
|
let refused = rig.serving(|path| failure_of(path, &bytes));
|
||||||
assert_eq!(refused.code, "payload-not-a-request", "{what}");
|
assert_eq!(refused.code, "payload-not-a-request", "{what}");
|
||||||
assert!(refused.message.contains("deciders"), "{what}: {}", refused.message);
|
assert!(refused.message.contains("deciders"), "{what}: {}", refused.message);
|
||||||
assert!(!refused.message.contains("d64"), "{what} echoes a decider: {}", refused.message);
|
assert!(!refused.message.contains(&acct("d64").0), "{what} echoes a decider: {}", refused.message);
|
||||||
}
|
}
|
||||||
assert_eq!(rig.state(), before, "nothing stored, head unchanged");
|
assert_eq!(rig.state(), before, "nothing stored, head unchanged");
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -8,7 +8,7 @@ use std::time::Duration;
|
||||||
|
|
||||||
use anyhow::{Result, bail, ensure};
|
use anyhow::{Result, bail, ensure};
|
||||||
|
|
||||||
use crate::sources::{DescribedSources, SourceIndex, SourcePoint, SourceRow, SourcesMeta};
|
use crate::sources::{ChatFilter, ChatOrder, ChatPayload, ChatPoint, ChatRow, DescribedSources, SourceIndex, SourcePoint, SourceRow, SourcesMeta, StoredChat};
|
||||||
|
|
||||||
use super::{CollectionMeta, Described, Embedder, Hit, ModelIdentity, Point, VectorIndex, point_id};
|
use super::{CollectionMeta, Described, Embedder, Hit, ModelIdentity, Point, VectorIndex, point_id};
|
||||||
|
|
||||||
|
|
@ -253,6 +253,8 @@ pub(crate) fn pair() -> (FakeEmbedder, FakeIndex) {
|
||||||
pub(crate) struct SourcesState {
|
pub(crate) struct SourcesState {
|
||||||
pub(crate) described: Option<DescribedSources>,
|
pub(crate) described: Option<DescribedSources>,
|
||||||
pub(crate) points: BTreeMap<String, SourcePoint>,
|
pub(crate) points: BTreeMap<String, SourcePoint>,
|
||||||
|
/// The chat points, by id: the same collection, the other corpus.
|
||||||
|
pub(crate) chat: BTreeMap<String, ChatPoint>,
|
||||||
/// How many times the collection was recreated.
|
/// How many times the collection was recreated.
|
||||||
pub(crate) recreated: u32,
|
pub(crate) recreated: u32,
|
||||||
pub(crate) upsert_calls: u32,
|
pub(crate) upsert_calls: u32,
|
||||||
|
|
@ -262,6 +264,16 @@ pub(crate) struct SourcesState {
|
||||||
pub(crate) deletes: Vec<(String, Option<Vec<String>>)>,
|
pub(crate) deletes: Vec<(String, Option<Vec<String>>)>,
|
||||||
/// A delete naming this repo refuses.
|
/// A delete naming this repo refuses.
|
||||||
pub(crate) fail_deletes_of: Option<String>,
|
pub(crate) fail_deletes_of: Option<String>,
|
||||||
|
/// Every cursor write refuses once this many have been written.
|
||||||
|
pub(crate) fail_cursor_writes_after: Option<usize>,
|
||||||
|
/// Every chat upsert refuses while this is set.
|
||||||
|
pub(crate) fail_chat_upserts: bool,
|
||||||
|
/// Every chat delete refuses while this is set.
|
||||||
|
pub(crate) fail_chat_deletes: bool,
|
||||||
|
/// A chat delete naming one of these rooms refuses.
|
||||||
|
pub(crate) fail_chat_deletes_for: BTreeSet<String>,
|
||||||
|
/// Every chat delete that went through: the room and the ids (`None` is all of the room).
|
||||||
|
pub(crate) chat_deletes: Vec<(String, Option<Vec<String>>)>,
|
||||||
/// The repo filter and limit of every search, in order.
|
/// The repo filter and limit of every search, in order.
|
||||||
pub(crate) searches: Vec<(Option<String>, u32)>,
|
pub(crate) searches: Vec<(Option<String>, u32)>,
|
||||||
}
|
}
|
||||||
|
|
@ -317,12 +329,16 @@ impl SourceIndex for FakeSources {
|
||||||
let mut state = self.state.lock().unwrap();
|
let mut state = self.state.lock().unwrap();
|
||||||
state.recreated += 1;
|
state.recreated += 1;
|
||||||
state.points.clear();
|
state.points.clear();
|
||||||
|
state.chat.clear();
|
||||||
state.described = Some(DescribedSources::Labelled { meta: meta.clone(), cursors: BTreeMap::new() });
|
state.described = Some(DescribedSources::Labelled { meta: meta.clone(), cursors: BTreeMap::new() });
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn set_cursors(&mut self, cursors: &BTreeMap<String, String>) -> Result<()> {
|
fn set_cursors(&mut self, cursors: &BTreeMap<String, String>) -> Result<()> {
|
||||||
let mut state = self.state.lock().unwrap();
|
let mut state = self.state.lock().unwrap();
|
||||||
|
if state.fail_cursor_writes_after.is_some_and(|after| state.cursor_writes.len() >= after) {
|
||||||
|
bail!("the vector store refused the cursor write");
|
||||||
|
}
|
||||||
state.cursor_writes.push(cursors.clone());
|
state.cursor_writes.push(cursors.clone());
|
||||||
let Some(DescribedSources::Labelled { cursors: held, .. }) = &mut state.described else { bail!("no source collection") };
|
let Some(DescribedSources::Labelled { cursors: held, .. }) = &mut state.described else { bail!("no source collection") };
|
||||||
*held = cursors.clone();
|
*held = cursors.clone();
|
||||||
|
|
@ -369,6 +385,86 @@ impl SourceIndex for FakeSources {
|
||||||
rows.truncate(limit as usize);
|
rows.truncate(limit as usize);
|
||||||
Ok(rows)
|
Ok(rows)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn upsert_chat(&mut self, points: &[ChatPoint]) -> Result<()> {
|
||||||
|
let mut state = self.state.lock().unwrap();
|
||||||
|
ensure!(!state.fail_chat_upserts, "the vector store refused the chat upsert");
|
||||||
|
for point in points {
|
||||||
|
ensure!(!point.payload.messages.is_empty(), "a chat chunk carries no messages");
|
||||||
|
state.chat.insert(point.id.clone(), point.clone());
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn chat_nearest(&mut self, vector: &[f32], filter: &ChatFilter, limit: u32, _within: Duration) -> Result<Vec<ChatRow>> {
|
||||||
|
let state = self.state.lock().unwrap();
|
||||||
|
let mut rows: Vec<ChatRow> = state
|
||||||
|
.chat
|
||||||
|
.values()
|
||||||
|
.filter(|point| chat_kept(filter, &point.payload))
|
||||||
|
.map(|point| ChatRow { score: cosine(vector, &point.vector), payload: point.payload.clone() })
|
||||||
|
.collect();
|
||||||
|
rows.sort_by(|a, b| b.score.total_cmp(&a.score));
|
||||||
|
rows.truncate(limit as usize);
|
||||||
|
Ok(rows)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn chat_scroll(&mut self, filter: &ChatFilter, order: ChatOrder, limit: u32, _within: Duration) -> Result<Vec<ChatPayload>> {
|
||||||
|
let state = self.state.lock().unwrap();
|
||||||
|
let mut kept: Vec<&ChatPayload> = state.chat.values().map(|point| &point.payload).filter(|payload| chat_kept(filter, payload)).collect();
|
||||||
|
kept.sort_by_key(|payload| payload.messages.iter().map(|message| message.ts_ms).min());
|
||||||
|
if order == ChatOrder::NewestFirst {
|
||||||
|
kept.reverse();
|
||||||
|
}
|
||||||
|
Ok(kept.into_iter().take(limit as usize).cloned().collect())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn chat_tail(&mut self, room_id: &str) -> Result<Option<StoredChat>> {
|
||||||
|
let state = self.state.lock().unwrap();
|
||||||
|
Ok(state.chat.iter().filter(|(_, point)| point.payload.room_id == room_id).max_by_key(|(id, point)| (span(&point.payload).1, (*id).clone())).map(stored))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn chat_chunks_holding(&mut self, room_id: &str, event_ids: &[String]) -> Result<Vec<StoredChat>> {
|
||||||
|
let state = self.state.lock().unwrap();
|
||||||
|
Ok(state
|
||||||
|
.chat
|
||||||
|
.iter()
|
||||||
|
.filter(|(_, point)| point.payload.room_id == room_id && point.payload.messages.iter().any(|message| event_ids.contains(&message.event_id)))
|
||||||
|
.map(stored)
|
||||||
|
.collect())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn chat_chunks_overlapping(&mut self, room_id: &str, from_ms: u64, to_ms: u64) -> Result<Vec<StoredChat>> {
|
||||||
|
let state = self.state.lock().unwrap();
|
||||||
|
Ok(state.chat.iter().filter(|(_, point)| point.payload.room_id == room_id && span(&point.payload).1 >= from_ms && span(&point.payload).0 < to_ms).map(stored).collect())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn delete_chat(&mut self, room_id: &str, ids: Option<&[String]>) -> Result<()> {
|
||||||
|
let mut state = self.state.lock().unwrap();
|
||||||
|
ensure!(!state.fail_chat_deletes && !state.fail_chat_deletes_for.contains(room_id), "the vector store refused the chat delete");
|
||||||
|
state.chat_deletes.push((room_id.to_owned(), ids.map(<[String]>::to_vec)));
|
||||||
|
state.chat.retain(|id, point| !(point.payload.room_id == room_id && ids.is_none_or(|ids| ids.contains(id))));
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn stored((id, point): (&String, &ChatPoint)) -> StoredChat {
|
||||||
|
StoredChat { id: id.clone(), payload: point.payload.clone() }
|
||||||
|
}
|
||||||
|
|
||||||
|
/// (start, end) of the times a chunk's messages make.
|
||||||
|
fn span(payload: &ChatPayload) -> (u64, u64) {
|
||||||
|
let times = || payload.messages.iter().map(|message| message.ts_ms);
|
||||||
|
(times().min().unwrap_or(0), times().max().unwrap_or(0))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A chat chunk under a filter, by the span its messages make.
|
||||||
|
fn chat_kept(filter: &ChatFilter, payload: &ChatPayload) -> bool {
|
||||||
|
let times = || payload.messages.iter().map(|message| message.ts_ms);
|
||||||
|
match (times().min(), times().max()) {
|
||||||
|
(Some(start), Some(end)) => filter.event_id.as_ref().is_none_or(|id| payload.messages.iter().any(|message| &message.event_id == id)) && filter.keeps(&payload.room_id, payload.room_alias.as_deref(), start, end),
|
||||||
|
_ => false,
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl SourceIndex for FakeIndex {
|
impl SourceIndex for FakeIndex {
|
||||||
|
|
@ -390,4 +486,25 @@ impl SourceIndex for FakeIndex {
|
||||||
fn search_sources(&mut self, vector: &[f32], repo: Option<&str>, front: bool, limit: u32, within: Duration) -> Result<Vec<SourceRow>> {
|
fn search_sources(&mut self, vector: &[f32], repo: Option<&str>, front: bool, limit: u32, within: Duration) -> Result<Vec<SourceRow>> {
|
||||||
self.sources.search_sources(vector, repo, front, limit, within)
|
self.sources.search_sources(vector, repo, front, limit, within)
|
||||||
}
|
}
|
||||||
|
fn upsert_chat(&mut self, points: &[ChatPoint]) -> Result<()> {
|
||||||
|
self.sources.upsert_chat(points)
|
||||||
|
}
|
||||||
|
fn chat_nearest(&mut self, vector: &[f32], filter: &ChatFilter, limit: u32, within: Duration) -> Result<Vec<ChatRow>> {
|
||||||
|
self.sources.chat_nearest(vector, filter, limit, within)
|
||||||
|
}
|
||||||
|
fn chat_scroll(&mut self, filter: &ChatFilter, order: ChatOrder, limit: u32, within: Duration) -> Result<Vec<ChatPayload>> {
|
||||||
|
self.sources.chat_scroll(filter, order, limit, within)
|
||||||
|
}
|
||||||
|
fn chat_tail(&mut self, room_id: &str) -> Result<Option<StoredChat>> {
|
||||||
|
self.sources.chat_tail(room_id)
|
||||||
|
}
|
||||||
|
fn chat_chunks_holding(&mut self, room_id: &str, event_ids: &[String]) -> Result<Vec<StoredChat>> {
|
||||||
|
self.sources.chat_chunks_holding(room_id, event_ids)
|
||||||
|
}
|
||||||
|
fn chat_chunks_overlapping(&mut self, room_id: &str, from_ms: u64, to_ms: u64) -> Result<Vec<StoredChat>> {
|
||||||
|
self.sources.chat_chunks_overlapping(room_id, from_ms, to_ms)
|
||||||
|
}
|
||||||
|
fn delete_chat(&mut self, room_id: &str, ids: Option<&[String]>) -> Result<()> {
|
||||||
|
self.sources.delete_chat(room_id, ids)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -105,9 +105,9 @@ use huginn_mind::wire::IndexStatus;
|
||||||
use huginn_mind::{INDEX_TEXT_VERSION, IndexEntry, Mind, MindStore};
|
use huginn_mind::{INDEX_TEXT_VERSION, IndexEntry, Mind, MindStore};
|
||||||
use sha2::{Digest, Sha256};
|
use sha2::{Digest, Sha256};
|
||||||
|
|
||||||
use crate::daemon::{Hits, IndexSink, SearchTicket};
|
use crate::daemon::{Found, Hits, IndexSink, SearchTicket};
|
||||||
use crate::sources::{self, SourceIndex};
|
use crate::sources::{self, SourceIndex};
|
||||||
use huginn_mind::{SourceHits, SourceQuery};
|
use huginn_mind::{ChatQuery, ChatWindowQuery, SourceQuery};
|
||||||
use ollama::QUERY_INSTRUCTION;
|
use ollama::QUERY_INSTRUCTION;
|
||||||
|
|
||||||
/// The `managed_by` a collection carries when this organ made it.
|
/// The `managed_by` a collection carries when this organ made it.
|
||||||
|
|
@ -685,11 +685,18 @@ struct Queued {
|
||||||
due: Instant,
|
due: Instant,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// A source search waiting for the worker. It is not the mind's: nothing about
|
/// What a source-collection search asks: forge chunks or chat chunks.
|
||||||
/// the mind's reconciliation gates it.
|
enum SourceAsk {
|
||||||
|
Sources(SourceQuery),
|
||||||
|
Chat(ChatQuery),
|
||||||
|
Window(ChatWindowQuery),
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A source or chat search waiting for the worker. It is not the mind's:
|
||||||
|
/// nothing about the mind's reconciliation gates it.
|
||||||
struct QueuedSource {
|
struct QueuedSource {
|
||||||
ticket: SearchTicket,
|
ticket: SearchTicket,
|
||||||
query: SourceQuery,
|
ask: SourceAsk,
|
||||||
due: Instant,
|
due: Instant,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -706,7 +713,7 @@ struct Shared {
|
||||||
/// Searches the worker finished and the serve loop has not collected.
|
/// Searches the worker finished and the serve loop has not collected.
|
||||||
found: Vec<(SearchTicket, Result<Hits, String>)>,
|
found: Vec<(SearchTicket, Result<Hits, String>)>,
|
||||||
source_searches: VecDeque<QueuedSource>,
|
source_searches: VecDeque<QueuedSource>,
|
||||||
sources_found: Vec<(SearchTicket, Result<SourceHits, String>)>,
|
sources_found: Vec<(SearchTicket, Found)>,
|
||||||
next_ticket: u64,
|
next_ticket: u64,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -821,10 +828,15 @@ fn serve_source_search<E: Embedder, V: VectorIndex + SourceIndex>(projector: &mu
|
||||||
if guard.closed {
|
if guard.closed {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
let Some(QueuedSource { ticket, query, due }) = guard.source_searches.pop_front() else { return };
|
let Some(QueuedSource { ticket, ask, due }) = guard.source_searches.pop_front() else { return };
|
||||||
drop(guard);
|
drop(guard);
|
||||||
let found = sources::search(&mut projector.embedder, &mut projector.index, &query, due.saturating_duration_since(Instant::now()))
|
let within = due.saturating_duration_since(Instant::now());
|
||||||
.map_err(|error| format!("{error:#}"));
|
let found = match ask {
|
||||||
|
SourceAsk::Sources(query) => sources::search(&mut projector.embedder, &mut projector.index, &query, within).map(Found::Sources),
|
||||||
|
SourceAsk::Chat(query) => sources::search_chat(&mut projector.embedder, &mut projector.index, &query, within).map(Found::Chat),
|
||||||
|
SourceAsk::Window(query) => sources::window(&mut projector.index, &query, within).map(|window| window.map_or_else(Found::Refused, Found::Window)),
|
||||||
|
}
|
||||||
|
.unwrap_or_else(|error| Found::Failed(format!("{error:#}")));
|
||||||
lock(shared).sources_found.push((ticket, found));
|
lock(shared).sources_found.push((ticket, found));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -966,6 +978,24 @@ pub struct WorkerSink {
|
||||||
}
|
}
|
||||||
|
|
||||||
impl WorkerSink {
|
impl WorkerSink {
|
||||||
|
/// Queues a source or chat search behind the same bound as every other
|
||||||
|
/// search; the mind's index state does not refuse it.
|
||||||
|
fn queue_source_ask(&mut self, ask: SourceAsk) -> Result<SearchTicket, String> {
|
||||||
|
let mut guard = lock(&self.shared);
|
||||||
|
if guard.searches.len() + guard.source_searches.len() >= SEARCHES_QUEUED_MAX {
|
||||||
|
return Err(format!(
|
||||||
|
"the index has {} searches waiting and takes no more until it has answered some",
|
||||||
|
guard.searches.len() + guard.source_searches.len()
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let ticket = SearchTicket(guard.next_ticket);
|
||||||
|
guard.next_ticket += 1;
|
||||||
|
guard.source_searches.push_back(QueuedSource { ticket, ask, due: Instant::now() + self.deadline });
|
||||||
|
drop(guard);
|
||||||
|
self.shared.1.notify_all();
|
||||||
|
Ok(ticket)
|
||||||
|
}
|
||||||
|
|
||||||
/// Starts the worker. `mind` is the mind's genesis receipt id, `None` for
|
/// Starts the worker. `mind` is the mind's genesis receipt id, `None` for
|
||||||
/// a mind not yet written. `startup` is every indexable document the mind
|
/// a mind not yet written. `startup` is every indexable document the mind
|
||||||
/// holds now; the worker's first act is to reconcile the collection
|
/// holds now; the worker's first act is to reconcile the collection
|
||||||
|
|
@ -1086,22 +1116,18 @@ impl<S: MindStore> IndexSink<S> for WorkerSink {
|
||||||
}
|
}
|
||||||
|
|
||||||
fn search_sources(&mut self, query: &SourceQuery) -> Result<SearchTicket, String> {
|
fn search_sources(&mut self, query: &SourceQuery) -> Result<SearchTicket, String> {
|
||||||
let mut guard = lock(&self.shared);
|
self.queue_source_ask(SourceAsk::Sources(query.clone()))
|
||||||
if guard.searches.len() + guard.source_searches.len() >= SEARCHES_QUEUED_MAX {
|
|
||||||
return Err(format!(
|
|
||||||
"the index has {} searches waiting and takes no more until it has answered some",
|
|
||||||
guard.searches.len() + guard.source_searches.len()
|
|
||||||
));
|
|
||||||
}
|
|
||||||
let ticket = SearchTicket(guard.next_ticket);
|
|
||||||
guard.next_ticket += 1;
|
|
||||||
guard.source_searches.push_back(QueuedSource { ticket, query: query.clone(), due: Instant::now() + self.deadline });
|
|
||||||
drop(guard);
|
|
||||||
self.shared.1.notify_all();
|
|
||||||
Ok(ticket)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
fn sources_searched(&mut self) -> Vec<(SearchTicket, Result<SourceHits, String>)> {
|
fn search_chat(&mut self, query: &ChatQuery) -> Result<SearchTicket, String> {
|
||||||
|
self.queue_source_ask(SourceAsk::Chat(query.clone()))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn chat_window(&mut self, query: &ChatWindowQuery) -> Result<SearchTicket, String> {
|
||||||
|
self.queue_source_ask(SourceAsk::Window(query.clone()))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn sources_searched(&mut self) -> Vec<(SearchTicket, Found)> {
|
||||||
std::mem::take(&mut lock(&self.shared).sources_found)
|
std::mem::take(&mut lock(&self.shared).sources_found)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -7,7 +7,12 @@ use std::time::Duration;
|
||||||
use anyhow::{Context, Result, ensure};
|
use anyhow::{Context, Result, ensure};
|
||||||
use serde_json::{Value, json};
|
use serde_json::{Value, json};
|
||||||
|
|
||||||
use crate::sources::{COLLECTION, FRONT_PAGE_FILES, DescribedSources, SourceIndex, SourcePayload, SourcePoint, SourceRow, SourcesMeta};
|
use huginn_mind::ChatMessage;
|
||||||
|
|
||||||
|
use crate::sources::{
|
||||||
|
COLLECTION, CORPUS_CHAT, CORPUS_FORGE, ChatFilter, ChatOrder, ChatPayload, ChatPoint, ChatRow, DescribedSources, FRONT_PAGE_FILES, SourceIndex, SourcePayload, SourcePoint, SourceRow,
|
||||||
|
SourcesMeta, StoredChat,
|
||||||
|
};
|
||||||
|
|
||||||
use super::{CollectionMeta, INDEX_CALL_TIMEOUT, Described, Hit, Http, Method, Point, VectorIndex};
|
use super::{CollectionMeta, INDEX_CALL_TIMEOUT, Described, Hit, Http, Method, Point, VectorIndex};
|
||||||
|
|
||||||
|
|
@ -175,21 +180,103 @@ impl VectorIndex for QdrantIndex {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// The payload filter of a source search: the repo, and the front-page paths
|
/// The payload filter of a source search: forge points only, then the repo and
|
||||||
/// (exact matches on the keyword-indexed `path`), both when both are asked.
|
/// the front-page paths (exact matches on the keyword-indexed `path`) when
|
||||||
fn source_filter(repo: Option<&str>, front: bool) -> Option<Value> {
|
/// asked. A chat point carries no repo or path and can never pass it.
|
||||||
let mut must = Vec::new();
|
fn source_filter(repo: Option<&str>, front: bool) -> Value {
|
||||||
|
let mut must = vec![json!({ "key": "corpus", "match": { "value": CORPUS_FORGE } })];
|
||||||
if let Some(repo) = repo {
|
if let Some(repo) = repo {
|
||||||
must.push(json!({ "key": "repo", "match": { "value": repo } }));
|
must.push(json!({ "key": "repo", "match": { "value": repo } }));
|
||||||
}
|
}
|
||||||
if front {
|
if front {
|
||||||
must.push(json!({ "key": "path", "match": { "any": FRONT_PAGE_FILES } }));
|
must.push(json!({ "key": "path", "match": { "any": FRONT_PAGE_FILES } }));
|
||||||
}
|
}
|
||||||
(!must.is_empty()).then(|| json!({ "must": must }))
|
json!({ "must": must })
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The payload filter of a chat read: chat points only; the rooms any-of by id
|
||||||
|
/// (`!`) or alias (`#`) as a `should`; the span as ranges that keep a chunk
|
||||||
|
/// overlapping [after, before].
|
||||||
|
fn chat_filter(filter: &ChatFilter) -> Value {
|
||||||
|
let mut must = vec![json!({ "key": "corpus", "match": { "value": CORPUS_CHAT } })];
|
||||||
|
if let Some(after) = filter.after_ms {
|
||||||
|
must.push(json!({ "key": "ts_end_ms", "range": { "gte": after } }));
|
||||||
|
}
|
||||||
|
if let Some(before) = filter.before_ms {
|
||||||
|
must.push(json!({ "key": "ts_start_ms", "range": { "lte": before } }));
|
||||||
|
}
|
||||||
|
if let Some(from) = filter.starts_from_ms {
|
||||||
|
must.push(json!({ "key": "ts_start_ms", "range": { "gte": from } }));
|
||||||
|
}
|
||||||
|
if let Some(to) = filter.starts_to_ms {
|
||||||
|
must.push(json!({ "key": "ts_start_ms", "range": { "lte": to } }));
|
||||||
|
}
|
||||||
|
if let Some(event_id) = &filter.event_id {
|
||||||
|
must.push(json!({ "key": "event_ids", "match": { "value": event_id } }));
|
||||||
|
}
|
||||||
|
let mut out = json!({ "must": must });
|
||||||
|
let (ids, aliases): (Vec<&String>, Vec<&String>) = filter.rooms.iter().partition(|room| room.starts_with('!'));
|
||||||
|
let mut should = Vec::new();
|
||||||
|
if !ids.is_empty() {
|
||||||
|
should.push(json!({ "key": "room_id", "match": { "any": ids } }));
|
||||||
|
}
|
||||||
|
if !aliases.is_empty() {
|
||||||
|
should.push(json!({ "key": "room_alias", "match": { "any": aliases } }));
|
||||||
|
}
|
||||||
|
if !should.is_empty() {
|
||||||
|
out["should"] = Value::Array(should);
|
||||||
|
}
|
||||||
|
out
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A chat chunk's payload. The span, first and last ids and the id list are
|
||||||
|
/// derived from `messages`, so they cannot disagree with them.
|
||||||
|
fn chat_payload_json(payload: &ChatPayload) -> Result<Value> {
|
||||||
|
let (Some(first), Some(last)) = (payload.messages.first(), payload.messages.last()) else {
|
||||||
|
anyhow::bail!("a chat chunk of {} carries no messages", payload.room_id);
|
||||||
|
};
|
||||||
|
let mut out = json!({
|
||||||
|
"corpus": CORPUS_CHAT,
|
||||||
|
"room_id": payload.room_id,
|
||||||
|
"ts_start_ms": payload.messages.iter().map(|message| message.ts_ms).min(),
|
||||||
|
"ts_end_ms": payload.messages.iter().map(|message| message.ts_ms).max(),
|
||||||
|
"first_event_id": first.event_id,
|
||||||
|
"last_event_id": last.event_id,
|
||||||
|
"event_ids": payload.messages.iter().map(|message| &message.event_id).collect::<Vec<_>>(),
|
||||||
|
"messages": payload.messages.iter().map(|message| json!({
|
||||||
|
"event_id": message.event_id, "sender": message.sender, "ts_ms": message.ts_ms, "body": message.body,
|
||||||
|
})).collect::<Vec<_>>(),
|
||||||
|
"text": payload.text,
|
||||||
|
});
|
||||||
|
if let Some(alias) = &payload.room_alias {
|
||||||
|
out["room_alias"] = json!(alias);
|
||||||
|
}
|
||||||
|
Ok(out)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn chat_payload_of(payload: &Value) -> Result<ChatPayload> {
|
||||||
|
ensure!(payload["corpus"] == CORPUS_CHAT, "a chat read was handed a point of another corpus");
|
||||||
|
let text = |value: &Value, key: &str| value[key].as_str().map(str::to_owned).with_context(|| format!("a chat chunk carries no {key}"));
|
||||||
|
let messages = payload["messages"]
|
||||||
|
.as_array()
|
||||||
|
.context("a chat chunk carries no messages")?
|
||||||
|
.iter()
|
||||||
|
.map(|message| {
|
||||||
|
Ok(ChatMessage {
|
||||||
|
event_id: text(message, "event_id")?,
|
||||||
|
sender: text(message, "sender")?,
|
||||||
|
ts_ms: message["ts_ms"].as_u64().context("a chat message carries no ts_ms")?,
|
||||||
|
body: text(message, "body")?,
|
||||||
|
})
|
||||||
|
})
|
||||||
|
.collect::<Result<Vec<_>>>()?;
|
||||||
|
ensure!(!messages.is_empty(), "a chat chunk carries no messages");
|
||||||
|
Ok(ChatPayload { room_id: text(payload, "room_id")?, room_alias: payload["room_alias"].as_str().map(str::to_owned), messages, text: text(payload, "text")? })
|
||||||
}
|
}
|
||||||
|
|
||||||
fn payload_json(payload: &SourcePayload) -> Value {
|
fn payload_json(payload: &SourcePayload) -> Value {
|
||||||
json!({
|
json!({
|
||||||
|
"corpus": CORPUS_FORGE,
|
||||||
"repo": payload.repo,
|
"repo": payload.repo,
|
||||||
"path": payload.path,
|
"path": payload.path,
|
||||||
"commit": payload.commit,
|
"commit": payload.commit,
|
||||||
|
|
@ -270,8 +357,17 @@ impl SourceIndex for QdrantIndex {
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
self.ask(Method::Put, &format!("{path}?wait=true"), Some(&body), &[200])?;
|
self.ask(Method::Put, &format!("{path}?wait=true"), Some(&body), &[200])?;
|
||||||
for field in ["repo", "path"] {
|
for (field, schema) in [
|
||||||
let index = json!({ "field_name": field, "field_schema": "keyword" });
|
("repo", "keyword"),
|
||||||
|
("path", "keyword"),
|
||||||
|
("corpus", "keyword"),
|
||||||
|
("room_id", "keyword"),
|
||||||
|
("room_alias", "keyword"),
|
||||||
|
("event_ids", "keyword"),
|
||||||
|
("ts_start_ms", "integer"),
|
||||||
|
("ts_end_ms", "integer"),
|
||||||
|
] {
|
||||||
|
let index = json!({ "field_name": field, "field_schema": schema });
|
||||||
self.ask(Method::Put, &format!("{path}/index?wait=true"), Some(&index), &[200])?;
|
self.ask(Method::Put, &format!("{path}/index?wait=true"), Some(&index), &[200])?;
|
||||||
}
|
}
|
||||||
Ok(())
|
Ok(())
|
||||||
|
|
@ -315,9 +411,7 @@ impl SourceIndex for QdrantIndex {
|
||||||
let collection = self.sources.clone();
|
let collection = self.sources.clone();
|
||||||
let http = self.http.bounded(within);
|
let http = self.http.bounded(within);
|
||||||
let mut body = json!({ "vector": vector, "limit": limit, "with_payload": true, "with_vector": false });
|
let mut body = json!({ "vector": vector, "limit": limit, "with_payload": true, "with_vector": false });
|
||||||
if let Some(filter) = source_filter(repo, front) {
|
body["filter"] = source_filter(repo, front);
|
||||||
body["filter"] = filter;
|
|
||||||
}
|
|
||||||
let (_, answer) =
|
let (_, answer) =
|
||||||
self.ask_with(&http, Method::Post, &format!("/collections/{collection}/points/search"), Some(&body), &[200])?;
|
self.ask_with(&http, Method::Post, &format!("/collections/{collection}/points/search"), Some(&body), &[200])?;
|
||||||
let found = answer["result"].as_array().context("Qdrant's search has no result")?;
|
let found = answer["result"].as_array().context("Qdrant's search has no result")?;
|
||||||
|
|
@ -331,6 +425,113 @@ impl SourceIndex for QdrantIndex {
|
||||||
})
|
})
|
||||||
.collect()
|
.collect()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn upsert_chat(&mut self, points: &[ChatPoint]) -> Result<()> {
|
||||||
|
let collection = self.sources.clone();
|
||||||
|
let points: Vec<Value> = points
|
||||||
|
.iter()
|
||||||
|
.map(|point| Ok(json!({ "id": point.id, "vector": point.vector, "payload": chat_payload_json(&point.payload)? })))
|
||||||
|
.collect::<Result<_>>()?;
|
||||||
|
self.ask(Method::Put, &format!("/collections/{collection}/points?wait=true"), Some(&json!({ "points": points })), &[200])?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn chat_nearest(&mut self, vector: &[f32], filter: &ChatFilter, limit: u32, within: Duration) -> Result<Vec<ChatRow>> {
|
||||||
|
let collection = self.sources.clone();
|
||||||
|
let http = self.http.bounded(within);
|
||||||
|
let body = json!({ "vector": vector, "limit": limit, "filter": chat_filter(filter), "with_payload": true, "with_vector": false });
|
||||||
|
let (_, answer) =
|
||||||
|
self.ask_with(&http, Method::Post, &format!("/collections/{collection}/points/search"), Some(&body), &[200])?;
|
||||||
|
let found = answer["result"].as_array().context("Qdrant's search has no result")?;
|
||||||
|
found
|
||||||
|
.iter()
|
||||||
|
.map(|point| Ok(ChatRow { score: point["score"].as_f64().context("a hit carries no score")? as f32, payload: chat_payload_of(&point["payload"])? }))
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn chat_scroll(&mut self, filter: &ChatFilter, order: ChatOrder, limit: u32, within: Duration) -> Result<Vec<ChatPayload>> {
|
||||||
|
let collection = self.sources.clone();
|
||||||
|
let http = self.http.bounded(within);
|
||||||
|
let body = json!({
|
||||||
|
"filter": chat_filter(filter),
|
||||||
|
"limit": limit,
|
||||||
|
"order_by": { "key": "ts_start_ms", "direction": match order { ChatOrder::OldestFirst => "asc", ChatOrder::NewestFirst => "desc" } },
|
||||||
|
"with_payload": true,
|
||||||
|
"with_vector": false,
|
||||||
|
});
|
||||||
|
let (_, answer) =
|
||||||
|
self.ask_with(&http, Method::Post, &format!("/collections/{collection}/points/scroll"), Some(&body), &[200])?;
|
||||||
|
let found = answer["result"]["points"].as_array().context("Qdrant's scroll has no points")?;
|
||||||
|
found.iter().map(|point| chat_payload_of(&point["payload"])).collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn chat_tail(&mut self, room_id: &str) -> Result<Option<StoredChat>> {
|
||||||
|
let order = json!({ "key": "ts_end_ms", "direction": "desc" });
|
||||||
|
let Some(newest) = self.scroll_chat(chat_room_filter(room_id, vec![]), Some((order, 1)))?.into_iter().next() else { return Ok(None) };
|
||||||
|
// Chunks tied on the greatest end are all read and the greatest id wins, so the tail is the same chunk on every run.
|
||||||
|
let end = newest.payload.messages.iter().map(|message| message.ts_ms).max().unwrap_or(0);
|
||||||
|
let tied = self.scroll_chat(chat_room_filter(room_id, vec![json!({ "key": "ts_end_ms", "range": { "gte": end } })]), None)?;
|
||||||
|
Ok(tied.into_iter().max_by(|a, b| a.id.cmp(&b.id)))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn chat_chunks_holding(&mut self, room_id: &str, event_ids: &[String]) -> Result<Vec<StoredChat>> {
|
||||||
|
if event_ids.is_empty() {
|
||||||
|
return Ok(Vec::new());
|
||||||
|
}
|
||||||
|
self.scroll_chat(chat_room_filter(room_id, vec![json!({ "key": "event_ids", "match": { "any": event_ids } })]), None)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn chat_chunks_overlapping(&mut self, room_id: &str, from_ms: u64, to_ms: u64) -> Result<Vec<StoredChat>> {
|
||||||
|
let span = vec![json!({ "key": "ts_end_ms", "range": { "gte": from_ms } }), json!({ "key": "ts_start_ms", "range": { "lt": to_ms } })];
|
||||||
|
self.scroll_chat(chat_room_filter(room_id, span), None)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn delete_chat(&mut self, room_id: &str, ids: Option<&[String]>) -> Result<()> {
|
||||||
|
let collection = self.sources.clone();
|
||||||
|
let body = json!({ "filter": chat_delete_filter(room_id, ids) });
|
||||||
|
self.ask(Method::Post, &format!("/collections/{collection}/points/delete?wait=true"), Some(&body), &[200])?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl QdrantIndex {
|
||||||
|
/// Every chat chunk `filter` keeps, with its point id, paged; or, given an
|
||||||
|
/// order and a limit, the first of them in that order.
|
||||||
|
fn scroll_chat(&mut self, filter: Value, ordered: Option<(Value, u64)>) -> Result<Vec<StoredChat>> {
|
||||||
|
let collection = self.sources.clone();
|
||||||
|
let mut out = Vec::new();
|
||||||
|
let mut offset = Value::Null;
|
||||||
|
loop {
|
||||||
|
let mut body = json!({ "filter": filter, "limit": SCROLL_PAGE, "with_payload": true, "with_vector": false });
|
||||||
|
if let Some((order, limit)) = &ordered {
|
||||||
|
body["order_by"] = order.clone();
|
||||||
|
body["limit"] = json!(limit);
|
||||||
|
} else if !offset.is_null() {
|
||||||
|
body["offset"] = offset;
|
||||||
|
}
|
||||||
|
let (_, answer) = self.ask(Method::Post, &format!("/collections/{collection}/points/scroll"), Some(&body), &[200])?;
|
||||||
|
for point in answer["result"]["points"].as_array().context("Qdrant's scroll has no points")? {
|
||||||
|
out.push(StoredChat { id: point["id"].as_str().context("a point id is not a string")?.to_owned(), payload: chat_payload_of(&point["payload"])? });
|
||||||
|
}
|
||||||
|
offset = answer["result"]["next_page_offset"].clone();
|
||||||
|
if ordered.is_some() || offset.is_null() {
|
||||||
|
return Ok(out);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// What a chat delete removes: the room's chat points, all of them or those with `ids`. The corpus
|
||||||
|
/// and room conditions hold either way, so a forge point or another room's point cannot match.
|
||||||
|
fn chat_delete_filter(room_id: &str, ids: Option<&[String]>) -> Value {
|
||||||
|
chat_room_filter(room_id, ids.map(|ids| json!({ "has_id": ids })).into_iter().collect())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The chat points of one room, narrowed by `more` conditions.
|
||||||
|
fn chat_room_filter(room_id: &str, more: Vec<Value>) -> Value {
|
||||||
|
let mut must = vec![json!({ "key": "corpus", "match": { "value": CORPUS_CHAT } }), json!({ "key": "room_id", "match": { "value": room_id } })];
|
||||||
|
must.extend(more);
|
||||||
|
json!({ "must": must })
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
|
|
@ -339,9 +540,10 @@ mod source_filter_tests {
|
||||||
|
|
||||||
/// No ask, no filter; a repo alone is the repo's match.
|
/// No ask, no filter; a repo alone is the repo's match.
|
||||||
#[test]
|
#[test]
|
||||||
fn no_ask_is_no_filter_and_a_repo_is_its_own_match() {
|
fn no_ask_is_the_forge_corpus_and_a_repo_is_its_own_match() {
|
||||||
assert_eq!(source_filter(None, false), None);
|
let forge = json!({ "key": "corpus", "match": { "value": "forge" } });
|
||||||
assert_eq!(source_filter(Some("Huginn"), false), Some(json!({ "must": [{ "key": "repo", "match": { "value": "Huginn" } }] })));
|
assert_eq!(source_filter(None, false), json!({ "must": [forge] }));
|
||||||
|
assert_eq!(source_filter(Some("Huginn"), false), json!({ "must": [forge, { "key": "repo", "match": { "value": "Huginn" } }] }));
|
||||||
}
|
}
|
||||||
|
|
||||||
/// The front filter matches the path against exactly the front-page files
|
/// The front filter matches the path against exactly the front-page files
|
||||||
|
|
@ -354,15 +556,104 @@ mod source_filter_tests {
|
||||||
assert_eq!(path.len(), 1);
|
assert_eq!(path.len(), 1);
|
||||||
path[0]["match"]["any"].as_array().unwrap().iter().map(|p| p.as_str().unwrap().to_owned()).collect()
|
path[0]["match"]["any"].as_array().unwrap().iter().map(|p| p.as_str().unwrap().to_owned()).collect()
|
||||||
};
|
};
|
||||||
let front = source_filter(None, true).unwrap();
|
let front = source_filter(None, true);
|
||||||
assert_eq!(any(&front), ["README.md", "README", "readme.md", "README.markdown", "CLAUDE.md", "AGENTS.md"]);
|
assert_eq!(any(&front), ["README.md", "README", "readme.md", "README.markdown", "CLAUDE.md", "AGENTS.md"]);
|
||||||
assert_eq!(front["must"].as_array().unwrap().len(), 1, "no repo condition when none was asked");
|
assert_eq!(front["must"].as_array().unwrap().len(), 2, "the corpus and the path; no repo condition when none was asked");
|
||||||
|
|
||||||
let both = source_filter(Some("Zyphos"), true).unwrap();
|
let both = source_filter(Some("Zyphos"), true);
|
||||||
assert_eq!(any(&both), any(&front));
|
assert_eq!(any(&both), any(&front));
|
||||||
let repos: Vec<&Value> = both["must"].as_array().unwrap().iter().filter(|c| c["key"] == "repo").collect();
|
let repos: Vec<&Value> = both["must"].as_array().unwrap().iter().filter(|c| c["key"] == "repo").collect();
|
||||||
assert_eq!(repos.len(), 1);
|
assert_eq!(repos.len(), 1);
|
||||||
assert_eq!(repos[0]["match"]["value"], "Zyphos");
|
assert_eq!(repos[0]["match"]["value"], "Zyphos");
|
||||||
assert_eq!(both["must"].as_array().unwrap().len(), 2);
|
assert_eq!(both["must"].as_array().unwrap().len(), 3);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn payload() -> ChatPayload {
|
||||||
|
let message = |id: &str, ts: u64| ChatMessage { event_id: id.into(), sender: "@a:gamecult.org".into(), ts_ms: ts, body: format!("body {id}") };
|
||||||
|
ChatPayload { room_id: "!r:gamecult.org".into(), room_alias: Some("#swarm:gamecult.org".into()), messages: vec![message("$a", 10), message("$b", 20)], text: "swarm\n10 @a: body".into() }
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The chat codec is its own inverse, writes the contract's fields derived
|
||||||
|
/// from the messages, and refuses a chunk with none and a point of another corpus.
|
||||||
|
#[test]
|
||||||
|
fn the_chat_payload_round_trips_the_contract() {
|
||||||
|
let written = chat_payload_json(&payload()).unwrap();
|
||||||
|
assert_eq!(written["corpus"], "chat");
|
||||||
|
assert_eq!((written["ts_start_ms"].as_u64(), written["ts_end_ms"].as_u64()), (Some(10), Some(20)));
|
||||||
|
assert_eq!((written["first_event_id"].as_str(), written["last_event_id"].as_str()), (Some("$a"), Some("$b")));
|
||||||
|
assert_eq!(written["event_ids"], json!(["$a", "$b"]));
|
||||||
|
assert_eq!(written["room_alias"], "#swarm:gamecult.org");
|
||||||
|
assert_eq!(chat_payload_of(&written).unwrap(), payload());
|
||||||
|
let mut no_alias = payload();
|
||||||
|
no_alias.room_alias = None;
|
||||||
|
assert!(chat_payload_json(&no_alias).unwrap().get("room_alias").is_none());
|
||||||
|
assert_eq!(chat_payload_of(&chat_payload_json(&no_alias).unwrap()).unwrap(), no_alias);
|
||||||
|
assert!(chat_payload_json(&ChatPayload { messages: vec![], ..payload() }).is_err());
|
||||||
|
let forge = payload_json(&SourcePayload { repo: "r".into(), path: "p".into(), commit: "c".into(), blob: "b".into(), line_start: 1, line_end: 2, chunk_index: 0, ext: "rs".into(), text: "t".into() });
|
||||||
|
assert_eq!(forge["corpus"], "forge");
|
||||||
|
assert!(chat_payload_of(&forge).is_err());
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The writer-side filters name the chat corpus and the room, so a forge point never matches.
|
||||||
|
#[test]
|
||||||
|
fn the_writer_filters_are_always_chat_corpus_and_the_room() {
|
||||||
|
let filter = chat_room_filter("!r:x", vec![json!({ "key": "event_ids", "match": { "any": ["$a"] } })]);
|
||||||
|
assert_eq!(
|
||||||
|
filter,
|
||||||
|
json!({ "must": [
|
||||||
|
{ "key": "corpus", "match": { "value": "chat" } },
|
||||||
|
{ "key": "room_id", "match": { "value": "!r:x" } },
|
||||||
|
{ "key": "event_ids", "match": { "any": ["$a"] } },
|
||||||
|
] })
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_chat_delete_names_the_corpus_and_room_with_or_without_ids() {
|
||||||
|
let all = chat_delete_filter("!r:x", None);
|
||||||
|
assert_eq!(all["must"].as_array().unwrap().len(), 2);
|
||||||
|
let some = chat_delete_filter("!r:x", Some(&["p1".to_owned()]));
|
||||||
|
assert_eq!(some["must"][2], json!({ "has_id": ["p1"] }));
|
||||||
|
assert_eq!(some["must"][0], json!({ "key": "corpus", "match": { "value": "chat" } }));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The chat filter is the chat corpus, the rooms as an any-of `should` split
|
||||||
|
/// by id and alias, and the two ranges that keep an overlapping span.
|
||||||
|
#[test]
|
||||||
|
fn the_chat_filter_is_corpus_rooms_and_overlap() {
|
||||||
|
let filter = chat_filter(&ChatFilter { rooms: vec!["!r:x".into(), "#a:x".into(), "#b:x".into()], after_ms: Some(5), before_ms: Some(9), ..ChatFilter::default() });
|
||||||
|
assert_eq!(
|
||||||
|
filter,
|
||||||
|
json!({
|
||||||
|
"must": [
|
||||||
|
{ "key": "corpus", "match": { "value": "chat" } },
|
||||||
|
{ "key": "ts_end_ms", "range": { "gte": 5 } },
|
||||||
|
{ "key": "ts_start_ms", "range": { "lte": 9 } },
|
||||||
|
],
|
||||||
|
"should": [
|
||||||
|
{ "key": "room_id", "match": { "any": ["!r:x"] } },
|
||||||
|
{ "key": "room_alias", "match": { "any": ["#a:x", "#b:x"] } },
|
||||||
|
],
|
||||||
|
})
|
||||||
|
);
|
||||||
|
assert_eq!(chat_filter(&ChatFilter::default()), json!({ "must": [{ "key": "corpus", "match": { "value": "chat" } }] }));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// An event id is a match on the keyword array, and the start bounds are
|
||||||
|
/// ranges on the chunk's own start.
|
||||||
|
#[test]
|
||||||
|
fn the_chat_filter_matches_an_event_and_bounds_chunk_starts() {
|
||||||
|
let filter = chat_filter(&ChatFilter { event_id: Some("$e".into()), starts_from_ms: Some(3), starts_to_ms: Some(8), ..ChatFilter::default() });
|
||||||
|
assert_eq!(
|
||||||
|
filter["must"],
|
||||||
|
json!([
|
||||||
|
{ "key": "corpus", "match": { "value": "chat" } },
|
||||||
|
{ "key": "ts_start_ms", "range": { "gte": 3 } },
|
||||||
|
{ "key": "ts_start_ms", "range": { "lte": 8 } },
|
||||||
|
{ "key": "event_ids", "match": { "value": "$e" } },
|
||||||
|
])
|
||||||
|
);
|
||||||
|
let corpus = json!({ "key": "corpus", "match": { "value": "chat" } });
|
||||||
|
assert_eq!(filter["must"][0], corpus, "a point with no corpus, or another, does not match: the chat corpus clause is always first");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1999,10 +1999,19 @@ fn ask_sources(sink: &mut WorkerSink, text: &str) -> Result<SearchTicket, String
|
||||||
<WorkerSink as IndexSink<OwnedRedbMessagePackBackingStore>>::search_sources(sink, &query)
|
<WorkerSink as IndexSink<OwnedRedbMessagePackBackingStore>>::search_sources(sink, &query)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn sources_answers(sink: &mut WorkerSink, count: usize) -> Vec<(SearchTicket, Result<SourceHits, String>)> {
|
fn sources_answers(sink: &mut WorkerSink, count: usize) -> Vec<(SearchTicket, Result<huginn_mind::SourceHits, String>)> {
|
||||||
let mut all = Vec::new();
|
let mut all = Vec::new();
|
||||||
eventually("the source searches being answered", || {
|
eventually("the source searches being answered", || {
|
||||||
all.extend(<WorkerSink as IndexSink<OwnedRedbMessagePackBackingStore>>::sources_searched(sink));
|
all.extend(<WorkerSink as IndexSink<OwnedRedbMessagePackBackingStore>>::sources_searched(sink).into_iter().map(|(ticket, found)| {
|
||||||
|
(
|
||||||
|
ticket,
|
||||||
|
match found {
|
||||||
|
crate::daemon::Found::Sources(hits) => Ok(hits),
|
||||||
|
crate::daemon::Found::Failed(why) => Err(why),
|
||||||
|
other => panic!("a source search was answered with another kind: {}", matches!(other, crate::daemon::Found::Chat(_))),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
}));
|
||||||
all.len() >= count
|
all.len() >= count
|
||||||
});
|
});
|
||||||
all
|
all
|
||||||
|
|
@ -2046,7 +2055,7 @@ fn a_worker_stuck_on_the_minds_collection_still_answers_a_source_search() {
|
||||||
index.plant(&collection_name(&slug(INSTANCE)), mind_label);
|
index.plant(&collection_name(&slug(INSTANCE)), mind_label);
|
||||||
let meta = SourcesMeta {
|
let meta = SourcesMeta {
|
||||||
managed_by: MANAGED_BY.into(),
|
managed_by: MANAGED_BY.into(),
|
||||||
corpus: "forge:GameCult".into(),
|
corpus: crate::sources::CORPUS.into(),
|
||||||
model: "model".into(),
|
model: "model".into(),
|
||||||
model_digest: "d1".into(),
|
model_digest: "d1".into(),
|
||||||
dimensions: 4,
|
dimensions: 4,
|
||||||
|
|
@ -2081,6 +2090,60 @@ fn a_worker_stuck_on_the_minds_collection_still_answers_a_source_search() {
|
||||||
assert_eq!(answered.indexed_under.digest, "d1");
|
assert_eq!(answered.indexed_under.digest, "d1");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// A chat search rides the same queue and worker as a source search, answers
|
||||||
|
/// from the same collection while the mind's own collection is stuck, and comes
|
||||||
|
/// back as chat hits.
|
||||||
|
#[test]
|
||||||
|
fn a_worker_answers_a_chat_search_from_the_shared_collection() {
|
||||||
|
use crate::sources::{ChatPayload, ChatPoint, DescribedSources, SourcesMeta};
|
||||||
|
use huginn_mind::ChatMessage;
|
||||||
|
let (embedder, index) = pair();
|
||||||
|
index.plant(
|
||||||
|
&collection_name(&slug(INSTANCE)),
|
||||||
|
Described::Labelled(CollectionMeta {
|
||||||
|
managed_by: MANAGED_BY.into(),
|
||||||
|
instance: INSTANCE.into(),
|
||||||
|
mind: "another-mind".into(),
|
||||||
|
model: "model".into(),
|
||||||
|
model_digest: "d1".into(),
|
||||||
|
dimensions: 4,
|
||||||
|
index_text_version: INDEX_TEXT_VERSION,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
let meta = SourcesMeta { managed_by: MANAGED_BY.into(), corpus: crate::sources::CORPUS.into(), model: "model".into(), model_digest: "d1".into(), dimensions: 4, chunker_version: 1 };
|
||||||
|
index.sources.plant(DescribedSources::Labelled { meta, cursors: Default::default() });
|
||||||
|
index
|
||||||
|
.sources
|
||||||
|
.state
|
||||||
|
.lock()
|
||||||
|
.unwrap()
|
||||||
|
.chat
|
||||||
|
.insert(
|
||||||
|
"c1".into(),
|
||||||
|
ChatPoint {
|
||||||
|
id: "c1".into(),
|
||||||
|
vector: vec![1.0; 4],
|
||||||
|
payload: ChatPayload {
|
||||||
|
room_id: "!r:x".into(),
|
||||||
|
room_alias: None,
|
||||||
|
messages: vec![ChatMessage { event_id: "$e".into(), sender: "@a:x".into(), ts_ms: 5, body: "hi".into() }],
|
||||||
|
text: "t".into(),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
);
|
||||||
|
let mut sink = WorkerSink::spawn(embedder, index.clone(), &slug(INSTANCE), Some(MIND.into()), entries_of(1), quick());
|
||||||
|
sink.wait_status(|status| matches!(status, IndexStatus::Refused { .. }));
|
||||||
|
let query = huginn_mind::ChatQuery { texts: vec![], rooms: vec![], after_ms: None, before_ms: None, top_k: 5 };
|
||||||
|
let ticket = <WorkerSink as IndexSink<OwnedRedbMessagePackBackingStore>>::search_chat(&mut sink, &query).unwrap();
|
||||||
|
let mut found = Vec::new();
|
||||||
|
eventually("the chat search being answered", || {
|
||||||
|
found.extend(<WorkerSink as IndexSink<OwnedRedbMessagePackBackingStore>>::sources_searched(&mut sink));
|
||||||
|
!found.is_empty()
|
||||||
|
});
|
||||||
|
let (answered, Found::Chat(hits)) = found.remove(0) else { panic!("a chat ask is answered with chat hits") };
|
||||||
|
assert_eq!((answered, hits.hits.len(), hits.hits[0].first_event_id.as_str(), hits.truncated), (ticket, 1, "$e", false));
|
||||||
|
}
|
||||||
|
|
||||||
/// Source searches take their own tickets, share the queue's bound with the
|
/// Source searches take their own tickets, share the queue's bound with the
|
||||||
/// mind's searches, and an abandon removes the one ticket named from either.
|
/// mind's searches, and an abandon removes the one ticket named from either.
|
||||||
#[test]
|
#[test]
|
||||||
|
|
@ -2107,3 +2170,47 @@ fn source_searches_have_distinct_tickets_share_the_queue_bound_and_abandon_by_ti
|
||||||
abandon(&mut sink, second);
|
abandon(&mut sink, second);
|
||||||
assert!(ask_sink(&mut sink, "now", 3).is_ok(), "a freed slot is a slot");
|
assert!(ask_sink(&mut sink, "now", 3).is_ok(), "a freed slot is a slot");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// The chat reads state their request: a scroll is the filter, oldest chunk
|
||||||
|
/// start first; a nearest search is the filter too.
|
||||||
|
#[test]
|
||||||
|
fn qdrant_chat_scroll_and_nearest_send_the_filter_and_the_order() {
|
||||||
|
use crate::sources::{ChatFilter, ChatOrder, SourceIndex};
|
||||||
|
let filter = ChatFilter { rooms: vec!["!r:x".into()], after_ms: Some(5), before_ms: None, ..ChatFilter::default() };
|
||||||
|
let scrolling = stub(&[("POST /collections/", 200, r#"{"result":{"points":[]}}"#)]);
|
||||||
|
QdrantIndex::new(&scrolling.url).chat_scroll(&filter, ChatOrder::OldestFirst, 7, TIME).unwrap();
|
||||||
|
let scroll: serde_json::Value = serde_json::from_str(&scrolling.seen.lock().unwrap()[0].2).unwrap();
|
||||||
|
assert_eq!(scroll["order_by"], serde_json::json!({ "key": "ts_start_ms", "direction": "asc" }));
|
||||||
|
assert_eq!(scroll["filter"]["must"][0], serde_json::json!({ "key": "corpus", "match": { "value": "chat" } }));
|
||||||
|
assert_eq!(scroll["filter"]["must"][1]["range"], serde_json::json!({ "gte": 5 }));
|
||||||
|
assert_eq!(scroll["limit"], 7);
|
||||||
|
|
||||||
|
let searching = stub(&[("POST /collections/", 200, r#"{"result":[]}"#)]);
|
||||||
|
QdrantIndex::new(&searching.url).chat_nearest(&[1.0, 0.0], &filter, 3, TIME).unwrap();
|
||||||
|
let near: serde_json::Value = serde_json::from_str(&searching.seen.lock().unwrap()[0].2).unwrap();
|
||||||
|
assert_eq!(near["filter"]["must"][0], serde_json::json!({ "key": "corpus", "match": { "value": "chat" } }));
|
||||||
|
assert_eq!(near["filter"]["must"][1]["range"], serde_json::json!({ "gte": 5 }));
|
||||||
|
assert_eq!(near["limit"], 3);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A new source collection indexes `corpus` as a keyword and both times as
|
||||||
|
/// integers, which the range filters and ordered scroll need.
|
||||||
|
#[test]
|
||||||
|
fn qdrant_recreate_sources_creates_the_corpus_and_time_indexes() {
|
||||||
|
use crate::sources::{CORPUS, SourceIndex, SourcesMeta};
|
||||||
|
let building = stub(&[("DELETE /collections/", 200, "{}"), ("PUT /collections/", 200, r#"{"result":true}"#)]);
|
||||||
|
let meta = SourcesMeta { managed_by: MANAGED_BY.into(), corpus: CORPUS.into(), model: "m".into(), model_digest: "d".into(), dimensions: 4, chunker_version: 1 };
|
||||||
|
QdrantIndex::new(&building.url).recreate_sources(&meta).unwrap();
|
||||||
|
let seen = building.seen.lock().unwrap();
|
||||||
|
let indexed: Vec<(String, String)> = seen
|
||||||
|
.iter()
|
||||||
|
.filter(|(_, path, _)| path.contains("/index"))
|
||||||
|
.map(|(_, _, body)| {
|
||||||
|
let body: serde_json::Value = serde_json::from_str(body).unwrap();
|
||||||
|
(body["field_name"].as_str().unwrap().to_owned(), body["field_schema"].as_str().unwrap().to_owned())
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
let has = |field: &str, schema: &str| indexed.iter().any(|(f, s)| f == field && s == schema);
|
||||||
|
assert!(has("corpus", "keyword"), "{indexed:?}");
|
||||||
|
assert!(has("ts_start_ms", "integer") && has("ts_end_ms", "integer"), "{indexed:?}");
|
||||||
|
}
|
||||||
|
|
|
||||||
672
crates/huginn-daemon/src/sources/chat.rs
Normal file
672
crates/huginn-daemon/src/sources/chat.rs
Normal file
|
|
@ -0,0 +1,672 @@
|
||||||
|
//! Chat ingest: joined-room history into the shared source collection, beside
|
||||||
|
//! the forge's repos. `refresh_chat` is the only writer of `corpus=chat` points
|
||||||
|
//! and of the cursors object's `chat:` keys; `chat_purge::request_purge` is the
|
||||||
|
//! only writer of `chat-purge:` keys, and `refresh_chat` the only applier.
|
||||||
|
//!
|
||||||
|
//! The Matrix client that implements `RoomHistory` is the next cut's and holds no
|
||||||
|
//! policy. What is in scope is `room_scope`'s; what may be embedded is
|
||||||
|
//! `huginn_mind::redact::scrub`'s; this file folds events into chunks and writes
|
||||||
|
//! them, write-then-delete and cursor-last, so any crash replays to the same state.
|
||||||
|
|
||||||
|
use std::collections::{BTreeMap, BTreeSet, HashSet};
|
||||||
|
use std::time::Instant;
|
||||||
|
|
||||||
|
use anyhow::{Context, Result, bail, ensure};
|
||||||
|
use huginn_mind::ChatMessage;
|
||||||
|
use huginn_mind::redact::{ScrubKind, scrub};
|
||||||
|
|
||||||
|
use super::chat_chunk::{chat_version, chunk_id, chunk_messages, chunk_payload, cut_body};
|
||||||
|
use super::chat_purge::{PurgeWindow, covers, parse_windows, purge_key};
|
||||||
|
use super::room_scope::{Estate, Refusal, RoomFacts, admit_room, refuse_event};
|
||||||
|
use super::{CORPUS, ChatPoint, DescribedSources, SourceIndex, StoredChat};
|
||||||
|
use crate::index::{BATCH, BULK_EMBED_TIMEOUT, Embedder, INDEX_CALL_TIMEOUT};
|
||||||
|
|
||||||
|
/// Events asked of the port per page.
|
||||||
|
pub const CHAT_PAGE: u32 = 200;
|
||||||
|
|
||||||
|
/// A chat cursor is the key `chat:<room_id>`; a forge repo name cannot contain
|
||||||
|
/// a colon. The only definition of the prefix.
|
||||||
|
pub const CHAT_CURSOR_PREFIX: &str = "chat:";
|
||||||
|
|
||||||
|
/// A steward's purge tombstone is the key `chat-purge:<room_id>`. The only
|
||||||
|
/// definition of the prefix.
|
||||||
|
pub const CHAT_PURGE_PREFIX: &str = "chat-purge:";
|
||||||
|
|
||||||
|
/// Whether a cursors-object key is chat's (a room cursor or a purge tombstone)
|
||||||
|
/// and not a forge repo name. The only test that tells the two apart.
|
||||||
|
pub fn is_chat_key(key: &str) -> bool {
|
||||||
|
key.starts_with(CHAT_CURSOR_PREFIX) || key.starts_with(CHAT_PURGE_PREFIX)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// What one room event is to the ingest. Only text, notice and emote messages
|
||||||
|
/// become `Message`; everything else (encrypted, media, state, server-redacted)
|
||||||
|
/// is `Other` and is never stored.
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||||
|
pub enum EventKind {
|
||||||
|
Message { body: String },
|
||||||
|
/// A replacement of `target`'s body.
|
||||||
|
Edit { target: String, body: String },
|
||||||
|
Redaction { target: String },
|
||||||
|
Other,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One event of a room, as the port hands it over: the Matrix client maps its
|
||||||
|
/// JSON to this and filters nothing. `content_keys` are the top-level keys of
|
||||||
|
/// the event's content (the scope reads them).
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||||
|
pub struct RoomEvent {
|
||||||
|
pub event_id: String,
|
||||||
|
pub sender: String,
|
||||||
|
pub ts_ms: u64,
|
||||||
|
pub content_keys: Vec<String>,
|
||||||
|
pub kind: EventKind,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One page of a room's events after a cursor event, oldest first.
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||||
|
pub struct EventPage {
|
||||||
|
pub events: Vec<RoomEvent>,
|
||||||
|
/// More events follow this page.
|
||||||
|
pub more: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The reader user's view of Synapse as ingest uses it. Holds no policy: it maps
|
||||||
|
/// the server's answers to these types and filters nothing.
|
||||||
|
pub trait RoomHistory {
|
||||||
|
/// The rooms the reader has joined. The only source of rooms; there is no allow list.
|
||||||
|
fn joined_rooms(&mut self) -> Result<Vec<String>>;
|
||||||
|
fn facts(&mut self, room_id: &str) -> Result<RoomFacts>;
|
||||||
|
/// Up to `limit` events after the event `after` (from the room's start when `None`).
|
||||||
|
/// `Ok(None)` means `after` is unknown to the server.
|
||||||
|
fn events_after(&mut self, room_id: &str, after: Option<&str>, limit: u32) -> Result<Option<EventPage>>;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The cursors-object key of a room's chat cursor.
|
||||||
|
pub fn cursor_key(room_id: &str) -> String {
|
||||||
|
format!("{CHAT_CURSOR_PREFIX}{room_id}")
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The room a chat cursor key belongs to; `None` for any other key (a purge tombstone included).
|
||||||
|
pub fn cursor_room(key: &str) -> Option<&str> {
|
||||||
|
key.strip_prefix(CHAT_CURSOR_PREFIX)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A cursor value, `<version>:<last event id>`; the event is empty before a room's first upsert.
|
||||||
|
pub fn parse_cursor(value: &str) -> Option<(&str, &str)> {
|
||||||
|
value.split_once(':')
|
||||||
|
}
|
||||||
|
|
||||||
|
/// What one admitted room's run did, for the journal. Never carries a body or a sender.
|
||||||
|
#[derive(Clone, Debug, PartialEq)]
|
||||||
|
pub struct ChatReport {
|
||||||
|
pub room_id: String,
|
||||||
|
pub alias: Option<String>,
|
||||||
|
pub first: Option<String>,
|
||||||
|
pub last: Option<String>,
|
||||||
|
pub events: u32,
|
||||||
|
pub chunks: u32,
|
||||||
|
pub scrubbed: u32,
|
||||||
|
pub purged: u32,
|
||||||
|
pub seconds: f64,
|
||||||
|
/// Why the room could not be read or written this run; what the index held for it stays as it was.
|
||||||
|
pub failure: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl std::fmt::Display for ChatReport {
|
||||||
|
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||||
|
write!(
|
||||||
|
formatter,
|
||||||
|
"{} ({}) {}..{} events={} chunks={} scrubbed={} purged={} seconds={:.1}",
|
||||||
|
self.alias.as_deref().unwrap_or("-"),
|
||||||
|
self.room_id,
|
||||||
|
self.first.as_deref().unwrap_or("-"),
|
||||||
|
self.last.as_deref().unwrap_or("-"),
|
||||||
|
self.events,
|
||||||
|
self.chunks,
|
||||||
|
self.scrubbed,
|
||||||
|
self.purged,
|
||||||
|
self.seconds
|
||||||
|
)?;
|
||||||
|
match &self.failure {
|
||||||
|
Some(why) => write!(formatter, " failed: {why}"),
|
||||||
|
None => Ok(()),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One line of a chat run's journal, typed so the job routes by what happened and not by how the line reads.
|
||||||
|
#[derive(Clone, Debug, PartialEq)]
|
||||||
|
pub enum Entry {
|
||||||
|
/// A room's report; a failed one is an alarm.
|
||||||
|
Report(ChatReport),
|
||||||
|
Refused(RefusedRoom),
|
||||||
|
Scrubbed(ScrubHit),
|
||||||
|
/// A failure outside any one room's report: retiring rooms.
|
||||||
|
Trouble(String),
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Entry {
|
||||||
|
/// The operator must see this on stderr: a failure, a refusal, a scrub hit.
|
||||||
|
pub fn is_alarm(&self) -> bool {
|
||||||
|
match self {
|
||||||
|
Self::Report(report) => report.failure.is_some(),
|
||||||
|
Self::Refused(_) | Self::Scrubbed(_) | Self::Trouble(_) => true,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl std::fmt::Display for Entry {
|
||||||
|
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||||
|
match self {
|
||||||
|
Self::Report(report) => report.fmt(formatter),
|
||||||
|
Self::Refused(RefusedRoom { room_id, reason, event_id: None }) => write!(formatter, "refused: {room_id} {reason}"),
|
||||||
|
Self::Refused(RefusedRoom { room_id, reason, event_id: Some(event) }) => write!(formatter, "refused: {room_id} {reason} [{event}]"),
|
||||||
|
Self::Scrubbed(hit) => write!(formatter, "scrubbed: {} {} {}", hit.room_id, hit.event_id, hit.kind),
|
||||||
|
Self::Trouble(why) => write!(formatter, "huginn-sources: {why}"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A joined room the scope refused, whole, and why; `event_id` when an event did it.
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||||
|
pub struct RefusedRoom {
|
||||||
|
pub room_id: String,
|
||||||
|
pub reason: Refusal,
|
||||||
|
pub event_id: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A credential the scrub replaced: where, never what.
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||||
|
pub struct ScrubHit {
|
||||||
|
pub room_id: String,
|
||||||
|
pub event_id: String,
|
||||||
|
pub kind: ScrubKind,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// What a chat refresh did.
|
||||||
|
#[derive(Clone, Debug, Default, PartialEq)]
|
||||||
|
pub struct ChatRefreshed {
|
||||||
|
pub rooms: Vec<ChatReport>,
|
||||||
|
/// Rooms (or steps) that failed; the others went on.
|
||||||
|
pub failed: Vec<String>,
|
||||||
|
pub refused: Vec<RefusedRoom>,
|
||||||
|
pub scrubbed: Vec<ScrubHit>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ChatReport {
|
||||||
|
fn empty(room_id: &str, alias: Option<String>) -> Self {
|
||||||
|
Self { room_id: room_id.to_owned(), alias, first: None, last: None, events: 0, chunks: 0, scrubbed: 0, purged: 0, seconds: 0.0, failure: None }
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A room that could not be handled this run. Its points and cursor are untouched: an error is not a reclassification.
|
||||||
|
fn failed(room_id: &str, alias: Option<String>, error: &anyhow::Error) -> Self {
|
||||||
|
Self { failure: Some(format!("{error:#}")), ..Self::empty(room_id, alias) }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ChatRefreshed {
|
||||||
|
/// Nothing failed and no joined room had to be refused: a joined estate, direct,
|
||||||
|
/// encrypted or bridged room means the reader's roster is wrong. A scrub hit does not make a run unclean.
|
||||||
|
pub fn is_clean(&self) -> bool {
|
||||||
|
self.failed.is_empty() && self.refused.is_empty()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn read_cursors<S: SourceIndex>(index: &mut S) -> Result<BTreeMap<String, String>> {
|
||||||
|
match index.describe_sources(INDEX_CALL_TIMEOUT).context("describing the source collection")? {
|
||||||
|
DescribedSources::Labelled { cursors, .. } => Ok(cursors),
|
||||||
|
_ => bail!("the source collection is gone; the forge refresh creates it"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn put_cursor<S: SourceIndex>(index: &mut S, key: &str, value: &str) -> Result<()> {
|
||||||
|
let mut cursors = read_cursors(index)?;
|
||||||
|
if cursors.insert(key.to_owned(), value.to_owned()).as_deref() != Some(value) {
|
||||||
|
index.set_cursors(&cursors).context("writing a chat cursor")?;
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn drop_cursor<S: SourceIndex>(index: &mut S, key: &str) -> Result<()> {
|
||||||
|
let mut cursors = read_cursors(index)?;
|
||||||
|
if cursors.remove(key).is_some() {
|
||||||
|
index.set_cursors(&cursors).context("dropping a chat cursor")?;
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A room's points, then its `chat:` cursor. Its purge tombstone stays.
|
||||||
|
fn purge_room<S: SourceIndex>(index: &mut S, room_id: &str) -> Result<()> {
|
||||||
|
index.delete_chat(room_id, None).with_context(|| format!("deleting the chunks of {room_id}"))?;
|
||||||
|
drop_cursor(index, &cursor_key(room_id))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// What `retire_chat` did: the rooms it removed, and those it could not (each with why), which it went on past.
|
||||||
|
#[derive(Debug, Default, PartialEq, Eq)]
|
||||||
|
pub struct Retired {
|
||||||
|
pub retired: Vec<String>,
|
||||||
|
pub failed: Vec<(String, String)>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Deletes the points and `chat:` cursor of every room not in `keep` (the reader left it,
|
||||||
|
/// or it is refused or excluded). A room whose delete fails is reported and the others go on. Purge
|
||||||
|
/// tombstones and forge repos are never touched. With the empty set it removes the whole chat corpus.
|
||||||
|
pub fn retire_chat<S: SourceIndex>(index: &mut S, keep: &BTreeSet<String>) -> Result<Retired> {
|
||||||
|
let rooms: Vec<String> = read_cursors(index)?.keys().filter_map(|key| cursor_room(key)).filter(|room| !keep.contains(*room)).map(str::to_owned).collect();
|
||||||
|
let mut done = Retired::default();
|
||||||
|
for room in rooms {
|
||||||
|
match purge_room(index, &room) {
|
||||||
|
Ok(()) => done.retired.push(room),
|
||||||
|
Err(error) => done.failed.push((room, format!("{error:#}"))),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(done)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A change to chunks already stored, in event order.
|
||||||
|
enum Op {
|
||||||
|
/// An edit by `sender` of `target`'s body; applies only to the original's sender.
|
||||||
|
Edit { id: String, target: String, sender: String, body: String },
|
||||||
|
/// A held message's body is what the page says it is (a replay).
|
||||||
|
Set { target: String, body: String },
|
||||||
|
Redact { target: String },
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Op {
|
||||||
|
fn target(&self) -> &str {
|
||||||
|
match self {
|
||||||
|
Self::Edit { target, .. } | Self::Set { target, .. } | Self::Redact { target } => target,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Applies to `messages`; whether they changed.
|
||||||
|
fn apply(&self, messages: &mut Vec<ChatMessage>) -> bool {
|
||||||
|
let Some(at) = messages.iter().position(|message| message.event_id == self.target()) else { return false };
|
||||||
|
match self {
|
||||||
|
Self::Edit { sender, body, .. } if messages[at].sender == *sender && messages[at].body != *body => messages[at].body.clone_from(body),
|
||||||
|
Self::Set { body, .. } if messages[at].body != *body => messages[at].body.clone_from(body),
|
||||||
|
Self::Redact { .. } => {
|
||||||
|
messages.remove(at);
|
||||||
|
}
|
||||||
|
_ => return false,
|
||||||
|
}
|
||||||
|
true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A message of the page with the edits of the page that apply to it: the body is the last edit's, and a
|
||||||
|
/// redacted edit leaves the list, so the body falls back to the one before it.
|
||||||
|
struct Pending {
|
||||||
|
message: ChatMessage,
|
||||||
|
original: String,
|
||||||
|
edits: Vec<(String, String)>,
|
||||||
|
}
|
||||||
|
|
||||||
|
struct Folded {
|
||||||
|
pending: Vec<ChatMessage>,
|
||||||
|
ops: Vec<Op>,
|
||||||
|
/// Every event id of the page: a redaction of one of them is known.
|
||||||
|
seen: HashSet<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
struct Room<'a> {
|
||||||
|
facts: &'a RoomFacts,
|
||||||
|
label: String,
|
||||||
|
windows: Vec<PurgeWindow>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The page folded in order: the purge filter, then the credential scrub, then the
|
||||||
|
/// event's effect. After this no raw body exists.
|
||||||
|
fn fold(room: &Room, events: &[RoomEvent], report: &mut ChatReport, out: &mut ChatRefreshed) -> Folded {
|
||||||
|
let mut pending: Vec<Pending> = Vec::new();
|
||||||
|
let mut ops: Vec<Op> = Vec::new();
|
||||||
|
let mut scrubbed = |event: &RoomEvent, body: &str| {
|
||||||
|
let clean = scrub(body);
|
||||||
|
for kind in clean.hits {
|
||||||
|
report.scrubbed += 1;
|
||||||
|
out.scrubbed.push(ScrubHit { room_id: room.facts.room_id.clone(), event_id: event.event_id.clone(), kind });
|
||||||
|
}
|
||||||
|
cut_body(&clean.text)
|
||||||
|
};
|
||||||
|
for event in events {
|
||||||
|
match &event.kind {
|
||||||
|
EventKind::Message { body } => {
|
||||||
|
if covers(&room.windows, event.ts_ms) {
|
||||||
|
report.purged += 1;
|
||||||
|
} else {
|
||||||
|
let body = scrubbed(event, body);
|
||||||
|
if !body.trim().is_empty() {
|
||||||
|
let message = ChatMessage { event_id: event.event_id.clone(), sender: event.sender.clone(), ts_ms: event.ts_ms, body: body.clone() };
|
||||||
|
pending.push(Pending { message, original: body, edits: Vec::new() });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
EventKind::Edit { target, body } => {
|
||||||
|
if covers(&room.windows, event.ts_ms) {
|
||||||
|
report.purged += 1;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let body = scrubbed(event, body);
|
||||||
|
match pending.iter_mut().find(|held| held.message.event_id == *target) {
|
||||||
|
Some(held) if held.message.sender == event.sender && !body.trim().is_empty() => {
|
||||||
|
held.edits.push((event.event_id.clone(), body.clone()));
|
||||||
|
held.message.body = body;
|
||||||
|
}
|
||||||
|
Some(_) => {}
|
||||||
|
None => ops.push(Op::Edit { id: event.event_id.clone(), target: target.clone(), sender: event.sender.clone(), body }),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
EventKind::Redaction { target } => {
|
||||||
|
// The target may be a message, or an edit whose own text must stop applying.
|
||||||
|
pending.retain(|held| held.message.event_id != *target);
|
||||||
|
for held in &mut pending {
|
||||||
|
held.edits.retain(|(id, _)| id != target);
|
||||||
|
held.message.body = held.edits.last().map_or_else(|| held.original.clone(), |(_, body)| body.clone());
|
||||||
|
}
|
||||||
|
ops.retain(|op| !matches!(op, Op::Edit { id, .. } if id == target));
|
||||||
|
ops.push(Op::Redact { target: target.clone() });
|
||||||
|
}
|
||||||
|
EventKind::Other => {}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Folded { pending: pending.into_iter().map(|held| held.message).collect(), ops, seen: events.iter().map(|event| event.event_id.clone()).collect() }
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Chunks to rewrite: the stored chunk ids they replace and the messages that now make them.
|
||||||
|
type Rewrite = (Vec<String>, Vec<ChatMessage>);
|
||||||
|
|
||||||
|
/// Chunks `rewrites`, embeds and writes the new chunks (acknowledged), and only then deletes the old
|
||||||
|
/// ids that are not among the new. Returns how many chunks were written.
|
||||||
|
///
|
||||||
|
/// A chunk written by an earlier run whose delete failed can still sit beside the chunk that replaced
|
||||||
|
/// it, and a rewrite of the stale one then names the same first event, hence the same id, as the
|
||||||
|
/// newer chunk. The newer chunk (`standing`, the other chunks the page found) stays; the stale one is deleted.
|
||||||
|
fn write_rewrites<E: Embedder, S: SourceIndex>(
|
||||||
|
embedder: &mut E,
|
||||||
|
index: &mut S,
|
||||||
|
dimensions: u32,
|
||||||
|
room: &Room,
|
||||||
|
rewrites: Vec<Rewrite>,
|
||||||
|
standing: &BTreeSet<String>,
|
||||||
|
) -> Result<u32> {
|
||||||
|
let mut old: BTreeSet<String> = BTreeSet::new();
|
||||||
|
let mut points: Vec<(String, super::ChatPayload)> = Vec::new();
|
||||||
|
for (ids, messages) in rewrites {
|
||||||
|
for group in chunk_messages(&room.label, &messages) {
|
||||||
|
let id = chunk_id(&room.facts.room_id, &group[0].event_id);
|
||||||
|
if standing.contains(&id) && !ids.contains(&id) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
points.push((id, chunk_payload(&room.facts.room_id, room.facts.alias.as_deref(), &room.label, group)));
|
||||||
|
}
|
||||||
|
old.extend(ids);
|
||||||
|
}
|
||||||
|
let new: BTreeSet<&String> = points.iter().map(|(id, _)| id).collect();
|
||||||
|
let stale: Vec<String> = old.iter().filter(|id| !new.contains(id)).cloned().collect();
|
||||||
|
let written = u32::try_from(points.len()).unwrap_or(u32::MAX);
|
||||||
|
for slice in points.chunks(BATCH) {
|
||||||
|
let texts: Vec<String> = slice.iter().map(|(_, payload)| payload.text.clone()).collect();
|
||||||
|
let vectors = embedder.embed(&texts, BULK_EMBED_TIMEOUT).context("embedding chat chunks")?;
|
||||||
|
ensure!(vectors.len() == texts.len(), "the embedder returned {} vectors for {} chunks", vectors.len(), texts.len());
|
||||||
|
let mut batch = Vec::with_capacity(slice.len());
|
||||||
|
for ((id, payload), vector) in slice.iter().zip(vectors) {
|
||||||
|
ensure!(vector.len() == dimensions as usize, "the embedder returned a vector of {} where the collection holds {dimensions}", vector.len());
|
||||||
|
batch.push(ChatPoint { id: id.clone(), vector, payload: payload.clone() });
|
||||||
|
}
|
||||||
|
index.upsert_chat(&batch).context("writing chat chunks")?;
|
||||||
|
}
|
||||||
|
if !stale.is_empty() {
|
||||||
|
index.delete_chat(&room.facts.room_id, Some(&stale)).context("deleting superseded chat chunks")?;
|
||||||
|
}
|
||||||
|
Ok(written)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Applies the room's purge windows to the chunks it already holds.
|
||||||
|
fn apply_purge<E: Embedder, S: SourceIndex>(embedder: &mut E, index: &mut S, dimensions: u32, room: &Room, report: &mut ChatReport) -> Result<()> {
|
||||||
|
let mut hit: BTreeMap<String, StoredChat> = BTreeMap::new();
|
||||||
|
for window in &room.windows {
|
||||||
|
for chunk in index.chat_chunks_overlapping(&room.facts.room_id, window.from_ms, window.to_ms)? {
|
||||||
|
hit.insert(chunk.id.clone(), chunk);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let mut rewrites = Vec::new();
|
||||||
|
for chunk in hit.into_values() {
|
||||||
|
let kept: Vec<ChatMessage> = chunk.payload.messages.iter().filter(|message| !covers(&room.windows, message.ts_ms)).cloned().collect();
|
||||||
|
let removed = chunk.payload.messages.len() - kept.len();
|
||||||
|
if removed > 0 {
|
||||||
|
report.purged += u32::try_from(removed).unwrap_or(u32::MAX);
|
||||||
|
rewrites.push((vec![chunk.id], kept));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
report.chunks += write_rewrites(embedder, index, dimensions, room, rewrites, &BTreeSet::new())?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Folds one page into the stored chunks: edits and redactions rewrite the chunk that held
|
||||||
|
/// the target alone, new messages extend the open tail.
|
||||||
|
fn write_page<E: Embedder, S: SourceIndex>(
|
||||||
|
embedder: &mut E,
|
||||||
|
index: &mut S,
|
||||||
|
dimensions: u32,
|
||||||
|
room: &Room,
|
||||||
|
mut folded: Folded,
|
||||||
|
may_rebuild: bool,
|
||||||
|
) -> Result<Option<u32>> {
|
||||||
|
let room_id = &room.facts.room_id;
|
||||||
|
let mut asked: Vec<String> = folded.pending.iter().map(|message| message.event_id.clone()).collect();
|
||||||
|
asked.extend(folded.ops.iter().map(|op| op.target().to_owned()));
|
||||||
|
if asked.is_empty() {
|
||||||
|
return Ok(Some(0));
|
||||||
|
}
|
||||||
|
let holding = index.chat_chunks_holding(room_id, &asked)?;
|
||||||
|
// A message the index already holds is a replay: its page body is asserted, never appended again.
|
||||||
|
let held: HashSet<&str> = holding.iter().flat_map(|chunk| chunk.payload.messages.iter().map(|message| message.event_id.as_str())).collect();
|
||||||
|
// A redaction of an event this room neither holds nor shows on the page may be of an edit whose own id is
|
||||||
|
// never stored, so a body it changed cannot be taken back here: the room is rebuilt from the history, which
|
||||||
|
// serves a redacted edit without its text.
|
||||||
|
if may_rebuild && folded.ops.iter().any(|op| matches!(op, Op::Redact { target } if !folded.seen.contains(target) && !held.contains(target.as_str()))) {
|
||||||
|
return Ok(None);
|
||||||
|
}
|
||||||
|
let (replayed, fresh): (Vec<ChatMessage>, Vec<ChatMessage>) = folded.pending.drain(..).partition(|message| held.contains(message.event_id.as_str()));
|
||||||
|
let mut ops: Vec<Op> = replayed.into_iter().map(|message| Op::Set { target: message.event_id, body: message.body }).collect();
|
||||||
|
ops.append(&mut folded.ops);
|
||||||
|
|
||||||
|
let tail = index.chat_tail(room_id)?;
|
||||||
|
let tail_id = tail.as_ref().map(|tail| tail.id.clone());
|
||||||
|
let standing: BTreeSet<String> = holding.iter().map(|chunk| chunk.id.clone()).chain(tail_id.clone()).collect();
|
||||||
|
let mut tail_messages = tail.map(|tail| tail.payload.messages).unwrap_or_default();
|
||||||
|
let mut tail_changed = false;
|
||||||
|
let mut rewrites: Vec<Rewrite> = Vec::new();
|
||||||
|
for chunk in holding {
|
||||||
|
let mut messages = chunk.payload.messages.clone();
|
||||||
|
let mut changed = false;
|
||||||
|
for op in &ops {
|
||||||
|
changed |= op.apply(&mut messages);
|
||||||
|
}
|
||||||
|
if Some(&chunk.id) == tail_id.as_ref() {
|
||||||
|
(tail_messages, tail_changed) = (messages, changed);
|
||||||
|
} else if changed {
|
||||||
|
rewrites.push((vec![chunk.id], messages));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !fresh.is_empty() || tail_changed {
|
||||||
|
tail_messages.extend(fresh);
|
||||||
|
rewrites.push((tail_id.into_iter().collect(), tail_messages));
|
||||||
|
}
|
||||||
|
write_rewrites(embedder, index, dimensions, room, rewrites, &standing).map(Some)
|
||||||
|
}
|
||||||
|
|
||||||
|
enum Outcome {
|
||||||
|
Done(ChatReport),
|
||||||
|
Refused(RefusedRoom),
|
||||||
|
}
|
||||||
|
|
||||||
|
#[allow(clippy::too_many_arguments)]
|
||||||
|
fn ingest_room<E: Embedder, S: SourceIndex, H: RoomHistory>(
|
||||||
|
embedder: &mut E,
|
||||||
|
index: &mut S,
|
||||||
|
history: &mut H,
|
||||||
|
dimensions: u32,
|
||||||
|
facts: &RoomFacts,
|
||||||
|
out: &mut ChatRefreshed,
|
||||||
|
) -> Result<Outcome> {
|
||||||
|
let started = Instant::now();
|
||||||
|
let room_id = facts.room_id.as_str();
|
||||||
|
let (key, version) = (cursor_key(room_id), chat_version());
|
||||||
|
let cursors = read_cursors(index)?;
|
||||||
|
let mut may_rebuild = false;
|
||||||
|
let mut after: Option<String> = match cursors.get(&key).map(|value| parse_cursor(value)) {
|
||||||
|
None => None,
|
||||||
|
Some(Some((held, event))) if held == version => {
|
||||||
|
may_rebuild = true;
|
||||||
|
Some(event.to_owned()).filter(|event| !event.is_empty())
|
||||||
|
}
|
||||||
|
// Another chunker or scrub version, or a value nobody here wrote: this room alone starts over.
|
||||||
|
Some(_) => {
|
||||||
|
index.delete_chat(room_id, None).with_context(|| format!("deleting the chunks of {room_id} for a re-ingest"))?;
|
||||||
|
None
|
||||||
|
}
|
||||||
|
};
|
||||||
|
if after.is_none() {
|
||||||
|
put_cursor(index, &key, &format!("{version}:"))?;
|
||||||
|
}
|
||||||
|
let windows = cursors.get(&purge_key(room_id)).map(|value| parse_windows(value)).transpose()?.unwrap_or_default();
|
||||||
|
let mut report = ChatReport::empty(room_id, facts.alias.clone());
|
||||||
|
// The room's name is embedded and stored in every chunk, so it passes the scrub like a body.
|
||||||
|
let label = match &facts.name {
|
||||||
|
Some(name) => {
|
||||||
|
let clean = scrub(name);
|
||||||
|
for kind in clean.hits {
|
||||||
|
report.scrubbed += 1;
|
||||||
|
out.scrubbed.push(ScrubHit { room_id: room_id.to_owned(), event_id: "(name)".to_owned(), kind });
|
||||||
|
}
|
||||||
|
clean.text
|
||||||
|
}
|
||||||
|
None => facts.alias.clone().unwrap_or_else(|| room_id.to_owned()),
|
||||||
|
};
|
||||||
|
let room = Room { facts, label, windows };
|
||||||
|
apply_purge(embedder, index, dimensions, &room, &mut report)?;
|
||||||
|
let mut restarted = false;
|
||||||
|
loop {
|
||||||
|
let Some(page) = history.events_after(room_id, after.as_deref(), CHAT_PAGE)? else {
|
||||||
|
ensure!(after.is_some() && !restarted, "the server does not know the start of {room_id}'s history");
|
||||||
|
restarted = true;
|
||||||
|
purge_room(index, room_id)?;
|
||||||
|
put_cursor(index, &key, &format!("{version}:"))?;
|
||||||
|
after = None;
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
let Some(last) = page.events.last().map(|event| event.event_id.clone()) else { break };
|
||||||
|
// A refusing event refuses the room before anything of the page is chunked or embedded.
|
||||||
|
if let Some((event, reason)) = page.events.iter().find_map(|event| refuse_event(event).map(|reason| (event, reason))) {
|
||||||
|
purge_room(index, room_id)?;
|
||||||
|
return Ok(Outcome::Refused(RefusedRoom { room_id: room_id.to_owned(), reason, event_id: Some(event.event_id.clone()) }));
|
||||||
|
}
|
||||||
|
report.first.get_or_insert_with(|| page.events[0].event_id.clone());
|
||||||
|
report.events += u32::try_from(page.events.len()).unwrap_or(u32::MAX);
|
||||||
|
let folded = fold(&room, &page.events, &mut report, out);
|
||||||
|
let Some(written) = write_page(embedder, index, dimensions, &room, folded, may_rebuild)? else {
|
||||||
|
// Start this room over from the history; nothing it read is trusted to be complete.
|
||||||
|
purge_room(index, room_id)?;
|
||||||
|
put_cursor(index, &key, &format!("{version}:"))?;
|
||||||
|
(after, may_rebuild) = (None, false);
|
||||||
|
report = ChatReport { scrubbed: report.scrubbed, ..ChatReport::empty(room_id, facts.alias.clone()) };
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
report.chunks += written;
|
||||||
|
put_cursor(index, &key, &format!("{version}:{last}"))?;
|
||||||
|
report.last = Some(last.clone());
|
||||||
|
after = Some(last);
|
||||||
|
if !page.more {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
report.seconds = started.elapsed().as_secs_f64();
|
||||||
|
Ok(Outcome::Done(report))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Brings the chat corpus to the reader's joined rooms. The collection must already exist and be
|
||||||
|
/// Huginn's, built under this embedder's model: the forge refresh creates and rebuilds it, and this never does.
|
||||||
|
/// A joined room the scope refuses is purged and named; a room the reader left is retired; the rest
|
||||||
|
/// are ingested from their cursors. One room's failure is journaled and the others go on.
|
||||||
|
/// `journal` receives every line the run prints: reports, scrub hits, refusals and failures.
|
||||||
|
pub fn refresh_chat<E: Embedder, S: SourceIndex, H: RoomHistory>(
|
||||||
|
embedder: &mut E,
|
||||||
|
index: &mut S,
|
||||||
|
history: &mut H,
|
||||||
|
estate: &Estate,
|
||||||
|
journal: &mut dyn FnMut(Entry),
|
||||||
|
) -> Result<ChatRefreshed> {
|
||||||
|
let DescribedSources::Labelled { meta, .. } = index.describe_sources(INDEX_CALL_TIMEOUT).context("describing the source collection")? else {
|
||||||
|
bail!("the source collection is absent or not Huginn's; the forge refresh creates it and chat never does");
|
||||||
|
};
|
||||||
|
ensure!(meta.corpus == CORPUS, "the source collection was built for corpus {} and chat ingest writes {CORPUS}; the forge refresh rebuilds it", meta.corpus);
|
||||||
|
embedder.built_under(Some(&meta.model_identity()));
|
||||||
|
let identity = embedder.model_identity(BULK_EMBED_TIMEOUT).context("reading the embedding model's identity")?;
|
||||||
|
ensure!(meta.model_identity() == identity, "the source collection was built under {} {} and chat embeds under {} {}", meta.model, meta.model_digest, identity.name, identity.digest);
|
||||||
|
|
||||||
|
let mut out = ChatRefreshed::default();
|
||||||
|
let mut joined = history.joined_rooms().context("listing the reader's joined rooms")?;
|
||||||
|
let mut seen = HashSet::new();
|
||||||
|
joined.retain(|room| seen.insert(room.clone()));
|
||||||
|
let mut keep: BTreeSet<String> = joined.iter().cloned().collect();
|
||||||
|
let mut admitted: Vec<RoomFacts> = Vec::new();
|
||||||
|
for room_id in &joined {
|
||||||
|
let verdict = if estate.holds_id(room_id) {
|
||||||
|
Ok(Err(Refusal::Estate))
|
||||||
|
} else {
|
||||||
|
history.facts(room_id).map(|facts| admit_room(&facts, estate).map(|()| facts))
|
||||||
|
};
|
||||||
|
match verdict {
|
||||||
|
Ok(Ok(facts)) => admitted.push(facts),
|
||||||
|
Ok(Err(reason)) => {
|
||||||
|
// Not kept, so the retirement below purges whatever it held.
|
||||||
|
keep.remove(room_id);
|
||||||
|
let refused = RefusedRoom { room_id: room_id.clone(), reason, event_id: None };
|
||||||
|
journal(Entry::Refused(refused.clone()));
|
||||||
|
out.refused.push(refused);
|
||||||
|
}
|
||||||
|
// An error is not a reclassification: the room keeps its points and cursor, and the report says why.
|
||||||
|
Err(error) => {
|
||||||
|
let report = ChatReport::failed(room_id, None, &error.context("reading the room's facts"));
|
||||||
|
journal(Entry::Report(report.clone()));
|
||||||
|
out.failed.push(room_id.clone());
|
||||||
|
out.rooms.push(report);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
match retire_chat(index, &keep) {
|
||||||
|
Ok(done) => {
|
||||||
|
for (room, why) in done.failed {
|
||||||
|
journal(Entry::Trouble(format!("retiring {room} failed, the others go on: {why}")));
|
||||||
|
out.failed.push(room);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(error) => {
|
||||||
|
journal(Entry::Trouble(format!("retiring rooms failed: {error:#}")));
|
||||||
|
out.failed.push("retire".to_owned());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for facts in admitted {
|
||||||
|
match ingest_room(embedder, index, history, identity.dimensions, &facts, &mut out) {
|
||||||
|
Ok(Outcome::Done(report)) => {
|
||||||
|
journal(Entry::Report(report.clone()));
|
||||||
|
out.rooms.push(report);
|
||||||
|
}
|
||||||
|
Ok(Outcome::Refused(refused)) => {
|
||||||
|
journal(Entry::Refused(refused.clone()));
|
||||||
|
out.refused.push(refused);
|
||||||
|
}
|
||||||
|
Err(error) => {
|
||||||
|
let report = ChatReport::failed(&facts.room_id, facts.alias.clone(), &error);
|
||||||
|
journal(Entry::Report(report.clone()));
|
||||||
|
out.failed.push(facts.room_id.clone());
|
||||||
|
out.rooms.push(report);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for hit in &out.scrubbed {
|
||||||
|
journal(Entry::Scrubbed(hit.clone()));
|
||||||
|
}
|
||||||
|
Ok(out)
|
||||||
|
}
|
||||||
166
crates/huginn-daemon/src/sources/chat_chunk.rs
Normal file
166
crates/huginn-daemon/src/sources/chat_chunk.rs
Normal file
|
|
@ -0,0 +1,166 @@
|
||||||
|
//! The chat chunker: greedy, deterministic and local. A chunk is consecutive
|
||||||
|
//! messages of one room; it closes at a time gap, at a message count, or when
|
||||||
|
//! its embedded text would pass a character bound. A message is never split.
|
||||||
|
|
||||||
|
use chrono::DateTime;
|
||||||
|
use huginn_mind::redact::SCRUB_VERSION;
|
||||||
|
use huginn_mind::ChatMessage;
|
||||||
|
|
||||||
|
use super::ChatPayload;
|
||||||
|
use crate::index::point_id;
|
||||||
|
|
||||||
|
/// Bumped whenever chunk boundaries or the embedded text would change for the
|
||||||
|
/// same messages. Part of `chat_version`.
|
||||||
|
pub const CHAT_CHUNKER_VERSION: u32 = 1;
|
||||||
|
|
||||||
|
/// A chunk closes when the next message is more than this much later.
|
||||||
|
pub const CHAT_GAP_MS: u64 = 30 * 60 * 1000;
|
||||||
|
/// The most messages a chunk holds.
|
||||||
|
pub const CHAT_MESSAGES_MAX: usize = 24;
|
||||||
|
/// The most characters a chunk's embedded text holds.
|
||||||
|
pub const CHAT_CHARS_MAX: usize = 1_600;
|
||||||
|
/// The most characters a stored message body holds, marker included.
|
||||||
|
pub const CHAT_BODY_CHARS_MAX: usize = 1_200;
|
||||||
|
|
||||||
|
/// Ends a body that was cut.
|
||||||
|
const CUT_MARKER: &str = "…[cut]";
|
||||||
|
|
||||||
|
/// What the room's cursor and chunk ids are stamped with: the chunker and the
|
||||||
|
/// scrub. Either changing re-ingests the room.
|
||||||
|
pub fn chat_version() -> String {
|
||||||
|
format!("{CHAT_CHUNKER_VERSION}.{SCRUB_VERSION}")
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A chunk's id: where it starts and under which version it was cut.
|
||||||
|
pub fn chunk_id(room_id: &str, first_event_id: &str) -> String {
|
||||||
|
point_id(&format!("{room_id}\0{first_event_id}\0{}", chat_version()))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A scrubbed body cut to `CHAT_BODY_CHARS_MAX` characters on a character boundary, with a marker.
|
||||||
|
pub fn cut_body(body: &str) -> String {
|
||||||
|
if body.chars().count() <= CHAT_BODY_CHARS_MAX {
|
||||||
|
return body.to_owned();
|
||||||
|
}
|
||||||
|
let keep = CHAT_BODY_CHARS_MAX - CUT_MARKER.chars().count();
|
||||||
|
body.chars().take(keep).chain(CUT_MARKER.chars()).collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn line_of(message: &ChatMessage) -> String {
|
||||||
|
let when = DateTime::from_timestamp_millis(i64::try_from(message.ts_ms).unwrap_or(i64::MAX)).map_or_else(String::new, |at| at.format("%Y-%m-%d %H:%M").to_string());
|
||||||
|
format!("{when} {}: {}", message.sender, message.body)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// What is embedded for a chunk: the room's label (its name, else its alias,
|
||||||
|
/// else its id), then one `YYYY-MM-DD HH:MM sender: body` line per message, UTC.
|
||||||
|
pub fn embedded_text(label: &str, messages: &[ChatMessage]) -> String {
|
||||||
|
let mut text = label.to_owned();
|
||||||
|
for message in messages {
|
||||||
|
text.push('\n');
|
||||||
|
text.push_str(&line_of(message));
|
||||||
|
}
|
||||||
|
text
|
||||||
|
}
|
||||||
|
|
||||||
|
/// `messages` (one room, in order) cut into chunks, each a run of consecutive messages.
|
||||||
|
pub fn chunk_messages(label: &str, messages: &[ChatMessage]) -> Vec<Vec<ChatMessage>> {
|
||||||
|
let mut chunks: Vec<Vec<ChatMessage>> = Vec::new();
|
||||||
|
let mut chars = 0_usize;
|
||||||
|
for message in messages {
|
||||||
|
let line = line_of(message).chars().count() + 1;
|
||||||
|
let closes = chunks.last().is_some_and(|open| {
|
||||||
|
let last = open.last().map_or(0, |last| last.ts_ms);
|
||||||
|
message.ts_ms.saturating_sub(last) > CHAT_GAP_MS || open.len() >= CHAT_MESSAGES_MAX || chars + line > CHAT_CHARS_MAX
|
||||||
|
});
|
||||||
|
if closes || chunks.is_empty() {
|
||||||
|
chunks.push(Vec::new());
|
||||||
|
chars = label.chars().count();
|
||||||
|
}
|
||||||
|
chunks.last_mut().expect("a chunk is open").push(message.clone());
|
||||||
|
chars += line;
|
||||||
|
}
|
||||||
|
chunks
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The point payload of one chunk.
|
||||||
|
pub fn chunk_payload(room_id: &str, alias: Option<&str>, label: &str, messages: Vec<ChatMessage>) -> ChatPayload {
|
||||||
|
ChatPayload { room_id: room_id.to_owned(), room_alias: alias.map(str::to_owned), text: embedded_text(label, &messages), messages }
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
fn message(n: u64, ts_ms: u64, body: &str) -> ChatMessage {
|
||||||
|
ChatMessage { event_id: format!("$e{n}"), sender: "@a:s".into(), ts_ms, body: body.into() }
|
||||||
|
}
|
||||||
|
|
||||||
|
fn ids(chunks: &[Vec<ChatMessage>]) -> Vec<Vec<&str>> {
|
||||||
|
chunks.iter().map(|chunk| chunk.iter().map(|message| message.event_id.as_str()).collect()).collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_gap_a_count_and_a_character_bound_each_close_a_chunk() {
|
||||||
|
let gap = [message(1, 0, "a"), message(2, CHAT_GAP_MS, "b"), message(3, 2 * CHAT_GAP_MS + 1, "c")];
|
||||||
|
assert_eq!(ids(&chunk_messages("r", &gap)), [["$e1", "$e2"].as_slice(), ["$e3"].as_slice()], "a gap of exactly the bound does not close");
|
||||||
|
let many: Vec<ChatMessage> = (0..25).map(|n| message(n, n, "x")).collect();
|
||||||
|
let chunks = chunk_messages("r", &many);
|
||||||
|
assert_eq!((chunks[0].len(), chunks[1].len()), (CHAT_MESSAGES_MAX, 1));
|
||||||
|
let long: Vec<ChatMessage> = (0..4).map(|n| message(n, n, &"w".repeat(500))).collect();
|
||||||
|
let chunks = chunk_messages("r", &long);
|
||||||
|
assert!(chunks.iter().all(|chunk| embedded_text("r", chunk).chars().count() <= CHAT_CHARS_MAX), "{:?}", ids(&chunks));
|
||||||
|
assert!(chunks.len() > 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn the_embedded_text_is_the_label_then_utc_lines() {
|
||||||
|
let text = embedded_text("Swarm", &[message(1, 1_700_000_000_000, "hello")]);
|
||||||
|
assert_eq!(text, "Swarm\n2023-11-14 22:13 @a:s: hello");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_long_body_is_cut_at_the_bound_with_a_marker_on_a_char_boundary_and_a_short_one_is_whole() {
|
||||||
|
let cut = cut_body(&"é".repeat(CHAT_BODY_CHARS_MAX + 50));
|
||||||
|
assert_eq!(cut.chars().count(), CHAT_BODY_CHARS_MAX);
|
||||||
|
assert!(cut.ends_with(CUT_MARKER));
|
||||||
|
assert_eq!(cut_body(&"é".repeat(CHAT_BODY_CHARS_MAX)), "é".repeat(CHAT_BODY_CHARS_MAX));
|
||||||
|
let one = chunk_messages("r", &[message(1, 0, &cut)]);
|
||||||
|
assert_eq!(one.len(), 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_chunk_id_is_its_room_first_event_and_version() {
|
||||||
|
assert_eq!(chunk_id("!r:s", "$a"), chunk_id("!r:s", "$a"));
|
||||||
|
assert_ne!(chunk_id("!r:s", "$a"), chunk_id("!r:s", "$b"));
|
||||||
|
assert_ne!(chunk_id("!r:s", "$a"), chunk_id("!q:s", "$a"));
|
||||||
|
assert_eq!(chat_version(), format!("{CHAT_CHUNKER_VERSION}.{SCRUB_VERSION}"));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Chunking the whole sequence equals re-chunking the open tail with each new batch appended.
|
||||||
|
#[test]
|
||||||
|
fn appending_to_the_open_tail_gives_the_boundaries_of_chunking_all_at_once() {
|
||||||
|
let mut seed = 0x2545_f491_4f6c_dd1d_u64;
|
||||||
|
let mut next = move |bound: u64| {
|
||||||
|
seed ^= seed << 13;
|
||||||
|
seed ^= seed >> 7;
|
||||||
|
seed ^= seed << 17;
|
||||||
|
seed % bound
|
||||||
|
};
|
||||||
|
let mut ts = 0;
|
||||||
|
let all: Vec<ChatMessage> = (0..300)
|
||||||
|
.map(|n| {
|
||||||
|
ts += if next(7) == 0 { CHAT_GAP_MS + 1 } else { next(60_000) };
|
||||||
|
message(n, ts, &"z".repeat(next(700) as usize + 1))
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
let whole = chunk_messages("r", &all);
|
||||||
|
for step in [1_usize, 5, 17, 100] {
|
||||||
|
let mut held: Vec<Vec<ChatMessage>> = Vec::new();
|
||||||
|
for batch in all.chunks(step) {
|
||||||
|
let mut tail: Vec<ChatMessage> = held.pop().unwrap_or_default();
|
||||||
|
tail.extend_from_slice(batch);
|
||||||
|
held.extend(chunk_messages("r", &tail));
|
||||||
|
}
|
||||||
|
assert_eq!(held, whole, "step {step}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
103
crates/huginn-daemon/src/sources/chat_purge.rs
Normal file
103
crates/huginn-daemon/src/sources/chat_purge.rs
Normal file
|
|
@ -0,0 +1,103 @@
|
||||||
|
//! A steward's purge of a room's time window. The tombstone is the one chat
|
||||||
|
//! state that is not derived from Synapse: a durable request, kept in the
|
||||||
|
//! collection's cursors object under `chat-purge:<room_id>`, that must outlive
|
||||||
|
//! the points it removes, so a re-ingest never brings the window back.
|
||||||
|
//!
|
||||||
|
//! `request_purge` is the only writer; `refresh_chat` reads the tombstone and
|
||||||
|
//! applies it at its next run. The code does not authenticate the steward: whoever
|
||||||
|
//! can run the job as the unit's user (the host admin) may ask, and the request
|
||||||
|
//! carries a ref in the job's journal for a trace. Tombstones live in the
|
||||||
|
//! collection, so a full collection rebuild forgets them; re-issue them from
|
||||||
|
//! their refs.
|
||||||
|
|
||||||
|
use anyhow::{Context, Result, bail};
|
||||||
|
|
||||||
|
use super::chat::CHAT_PURGE_PREFIX;
|
||||||
|
use super::{DescribedSources, SourceIndex};
|
||||||
|
use crate::index::INDEX_CALL_TIMEOUT;
|
||||||
|
|
||||||
|
/// The most windows a room's tombstone holds.
|
||||||
|
pub const PURGE_WINDOWS_MAX: usize = 32;
|
||||||
|
|
||||||
|
/// Messages with `from_ms <= ts_ms < to_ms` are purged.
|
||||||
|
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
|
||||||
|
pub struct PurgeWindow {
|
||||||
|
pub from_ms: u64,
|
||||||
|
pub to_ms: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl PurgeWindow {
|
||||||
|
pub fn covers(&self, ts_ms: u64) -> bool {
|
||||||
|
self.from_ms <= ts_ms && ts_ms < self.to_ms
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The tombstone key of a room.
|
||||||
|
pub fn purge_key(room_id: &str) -> String {
|
||||||
|
format!("{CHAT_PURGE_PREFIX}{room_id}")
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A tombstone value, `<from>..<to>[,<from>..<to>...]`.
|
||||||
|
pub fn parse_windows(value: &str) -> Result<Vec<PurgeWindow>> {
|
||||||
|
value
|
||||||
|
.split(',')
|
||||||
|
.map(|part| {
|
||||||
|
let (from, to) = part.split_once("..").with_context(|| format!("purge window {part:?} is not <from>..<to>"))?;
|
||||||
|
let window = PurgeWindow { from_ms: from.parse().context("a purge window's start is not milliseconds")?, to_ms: to.parse().context("a purge window's end is not milliseconds")? };
|
||||||
|
if window.from_ms >= window.to_ms {
|
||||||
|
bail!("purge window {part:?} is empty");
|
||||||
|
}
|
||||||
|
Ok(window)
|
||||||
|
})
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Windows sorted, with overlapping and touching ones joined.
|
||||||
|
pub fn merge(mut windows: Vec<PurgeWindow>) -> Vec<PurgeWindow> {
|
||||||
|
windows.sort();
|
||||||
|
let mut merged: Vec<PurgeWindow> = Vec::new();
|
||||||
|
for window in windows {
|
||||||
|
match merged.last_mut() {
|
||||||
|
Some(last) if window.from_ms <= last.to_ms => last.to_ms = last.to_ms.max(window.to_ms),
|
||||||
|
_ => merged.push(window),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
merged
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn encode(windows: &[PurgeWindow]) -> String {
|
||||||
|
windows.iter().map(|window| format!("{}..{}", window.from_ms, window.to_ms)).collect::<Vec<_>>().join(",")
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Whether any window covers `ts_ms`.
|
||||||
|
pub fn covers(windows: &[PurgeWindow], ts_ms: u64) -> bool {
|
||||||
|
windows.iter().any(|window| window.covers(ts_ms))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Records that `window` of `room_id` must leave the index and never return. Writes the cursors
|
||||||
|
/// object only if the tombstone changed, and refuses a room's 33rd window. Writes no point and
|
||||||
|
/// embeds nothing; the next `refresh_chat` applies it. The job's `chat-purge` subcommand calls
|
||||||
|
/// it under the job's lock, because `set_cursors` replaces the cursors object whole.
|
||||||
|
/// Returns whether the tombstone changed.
|
||||||
|
pub fn request_purge<S: SourceIndex>(index: &mut S, room_id: &str, window: PurgeWindow) -> Result<bool> {
|
||||||
|
if window.from_ms >= window.to_ms {
|
||||||
|
bail!("a purge window must start before it ends");
|
||||||
|
}
|
||||||
|
let DescribedSources::Labelled { mut cursors, .. } = index.describe_sources(INDEX_CALL_TIMEOUT).context("describing the source collection")? else {
|
||||||
|
bail!("the source collection is absent or not Huginn's; the forge refresh creates it");
|
||||||
|
};
|
||||||
|
let key = purge_key(room_id);
|
||||||
|
let mut windows = cursors.get(&key).map(|held| parse_windows(held)).transpose()?.unwrap_or_default();
|
||||||
|
windows.push(window);
|
||||||
|
let windows = merge(windows);
|
||||||
|
if windows.len() > PURGE_WINDOWS_MAX {
|
||||||
|
bail!("room {room_id} already holds {PURGE_WINDOWS_MAX} purge windows; widen one instead");
|
||||||
|
}
|
||||||
|
let encoded = encode(&windows);
|
||||||
|
if cursors.get(&key) == Some(&encoded) {
|
||||||
|
return Ok(false);
|
||||||
|
}
|
||||||
|
cursors.insert(key, encoded);
|
||||||
|
index.set_cursors(&cursors).context("writing the purge tombstone")?;
|
||||||
|
Ok(true)
|
||||||
|
}
|
||||||
964
crates/huginn-daemon/src/sources/chat_tests.rs
Normal file
964
crates/huginn-daemon/src/sources/chat_tests.rs
Normal file
|
|
@ -0,0 +1,964 @@
|
||||||
|
//! Chat ingest against a scripted room history, a fake vector store and the fake embedder.
|
||||||
|
//! The refusals come first: what must never be read, embedded or stored.
|
||||||
|
|
||||||
|
use std::collections::{BTreeMap, BTreeSet};
|
||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
|
use anyhow::{Result, bail};
|
||||||
|
use huginn_mind::ChatMessage;
|
||||||
|
use huginn_mind::redact::ScrubKind;
|
||||||
|
|
||||||
|
use super::chat::*;
|
||||||
|
use super::chat_chunk::{CHAT_BODY_CHARS_MAX, chat_version, chunk_id};
|
||||||
|
use super::chat_purge::{PURGE_WINDOWS_MAX, PurgeWindow, request_purge};
|
||||||
|
use super::room_scope::{ESTATE_ALIASES, Estate, Refusal, RoomFacts};
|
||||||
|
use super::*;
|
||||||
|
use crate::index::MANAGED_BY;
|
||||||
|
use crate::index::fakes::{FakeEmbedder, FakeSources};
|
||||||
|
|
||||||
|
// ---- the rig ----
|
||||||
|
|
||||||
|
fn plain(id: &str) -> RoomFacts {
|
||||||
|
RoomFacts { room_id: id.into(), alias: None, alt_aliases: vec![], name: None, encrypted: false, direct: false, bridge: None }
|
||||||
|
}
|
||||||
|
|
||||||
|
fn estate(ids: &[&str]) -> Estate {
|
||||||
|
Estate::new(ids.iter().chain(["!pad1:s", "!pad2:s", "!pad3:s"].iter()).take(3).map(|id| (*id).to_owned()).collect()).unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
|
struct Scripted {
|
||||||
|
facts: RoomFacts,
|
||||||
|
events: Vec<RoomEvent>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Default)]
|
||||||
|
struct FakeHistory {
|
||||||
|
joined: Vec<String>,
|
||||||
|
rooms: BTreeMap<String, Scripted>,
|
||||||
|
calls: Vec<String>,
|
||||||
|
/// Rooms whose state cannot be read.
|
||||||
|
facts_fail: BTreeSet<String>,
|
||||||
|
/// Serve an event a redaction targets as the server does, without its content.
|
||||||
|
prune: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl FakeHistory {
|
||||||
|
fn room(&mut self, id: &str, events: Vec<RoomEvent>) -> &mut Self {
|
||||||
|
self.rooms.insert(id.into(), Scripted { facts: plain(id), events });
|
||||||
|
self.joined.push(id.into());
|
||||||
|
self
|
||||||
|
}
|
||||||
|
|
||||||
|
fn facts(&mut self, id: &str, edit: impl FnOnce(&mut RoomFacts)) -> &mut Self {
|
||||||
|
edit(&mut self.rooms.get_mut(id).unwrap().facts);
|
||||||
|
self
|
||||||
|
}
|
||||||
|
|
||||||
|
fn push(&mut self, id: &str, events: Vec<RoomEvent>) {
|
||||||
|
self.rooms.get_mut(id).unwrap().events.extend(events);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn called(&self, prefix: &str, room: &str) -> bool {
|
||||||
|
self.calls.iter().any(|call| call == &format!("{prefix}:{room}"))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl RoomHistory for FakeHistory {
|
||||||
|
fn joined_rooms(&mut self) -> Result<Vec<String>> {
|
||||||
|
self.calls.push("joined".into());
|
||||||
|
Ok(self.joined.clone())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn facts(&mut self, room_id: &str) -> Result<RoomFacts> {
|
||||||
|
self.calls.push(format!("facts:{room_id}"));
|
||||||
|
if self.facts_fail.contains(room_id) {
|
||||||
|
bail!("the server did not answer for the state of {room_id}");
|
||||||
|
}
|
||||||
|
match self.rooms.get(room_id) {
|
||||||
|
Some(room) => Ok(room.facts.clone()),
|
||||||
|
None => bail!("no such room"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn events_after(&mut self, room_id: &str, after: Option<&str>, limit: u32) -> Result<Option<EventPage>> {
|
||||||
|
self.calls.push(format!("events:{room_id}"));
|
||||||
|
let Some(room) = self.rooms.get(room_id) else { bail!("no such room") };
|
||||||
|
let from = match after {
|
||||||
|
None => 0,
|
||||||
|
Some(after) => match room.events.iter().position(|event| event.event_id == after) {
|
||||||
|
Some(at) => at + 1,
|
||||||
|
None => return Ok(None),
|
||||||
|
},
|
||||||
|
};
|
||||||
|
let redacted: BTreeSet<&str> = room.events.iter().filter_map(|event| if let EventKind::Redaction { target } = &event.kind { Some(target.as_str()) } else { None }).collect();
|
||||||
|
let served: Vec<RoomEvent> = room.events[from..]
|
||||||
|
.iter()
|
||||||
|
.map(|event| if self.prune && redacted.contains(event.event_id.as_str()) { RoomEvent { kind: EventKind::Other, content_keys: vec![], ..event.clone() } } else { event.clone() })
|
||||||
|
.collect();
|
||||||
|
let take = served.len().min(limit as usize);
|
||||||
|
Ok(Some(EventPage { events: served[..take].to_vec(), more: take < served.len() }))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn event(id: &str, sender: &str, ts_ms: u64, kind: EventKind) -> RoomEvent {
|
||||||
|
RoomEvent { event_id: id.into(), sender: sender.into(), ts_ms, content_keys: vec!["body".into(), "msgtype".into()], kind }
|
||||||
|
}
|
||||||
|
|
||||||
|
fn said(id: &str, ts_ms: u64, body: &str) -> RoomEvent {
|
||||||
|
event(id, "@a:s", ts_ms, EventKind::Message { body: body.into() })
|
||||||
|
}
|
||||||
|
|
||||||
|
fn edit(id: &str, sender: &str, ts_ms: u64, target: &str, body: &str) -> RoomEvent {
|
||||||
|
event(id, sender, ts_ms, EventKind::Edit { target: target.into(), body: body.into() })
|
||||||
|
}
|
||||||
|
|
||||||
|
fn redaction(id: &str, ts_ms: u64, target: &str) -> RoomEvent {
|
||||||
|
event(id, "@a:s", ts_ms, EventKind::Redaction { target: target.into() })
|
||||||
|
}
|
||||||
|
|
||||||
|
struct Rig {
|
||||||
|
embedder: FakeEmbedder,
|
||||||
|
index: FakeSources,
|
||||||
|
estate: Estate,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Rig {
|
||||||
|
fn new() -> Self {
|
||||||
|
let mut embedder = FakeEmbedder::new("d1");
|
||||||
|
let index = FakeSources::default();
|
||||||
|
let identity = crate::index::Embedder::model_identity(&mut embedder, Duration::from_secs(1)).unwrap();
|
||||||
|
let meta = SourcesMeta {
|
||||||
|
managed_by: MANAGED_BY.into(),
|
||||||
|
corpus: CORPUS.into(),
|
||||||
|
model: identity.name,
|
||||||
|
model_digest: identity.digest,
|
||||||
|
dimensions: identity.dimensions,
|
||||||
|
chunker_version: chunk::CHUNKER_VERSION,
|
||||||
|
};
|
||||||
|
index.plant(DescribedSources::Labelled { meta, cursors: BTreeMap::new() });
|
||||||
|
Self { embedder, index, estate: estate(&[]) }
|
||||||
|
}
|
||||||
|
|
||||||
|
fn run(&mut self, history: &mut FakeHistory) -> (Result<ChatRefreshed>, Vec<String>) {
|
||||||
|
let mut journal = Vec::new();
|
||||||
|
let out = refresh_chat(&mut self.embedder, &mut self.index, history, &self.estate, &mut |line| journal.push(line.to_string()));
|
||||||
|
(out, journal)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn ok(&mut self, history: &mut FakeHistory) -> ChatRefreshed {
|
||||||
|
self.run(history).0.unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn points(&self) -> Vec<ChatPoint> {
|
||||||
|
self.index.state.lock().unwrap().chat.values().cloned().collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn ids(&self) -> BTreeSet<String> {
|
||||||
|
self.index.state.lock().unwrap().chat.keys().cloned().collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn bodies(&self) -> Vec<String> {
|
||||||
|
let mut bodies: Vec<String> = self.points().iter().flat_map(|point| point.payload.messages.iter().map(|message| message.body.clone())).collect();
|
||||||
|
bodies.sort();
|
||||||
|
bodies
|
||||||
|
}
|
||||||
|
|
||||||
|
fn held_ids(&self, room: &str) -> Vec<String> {
|
||||||
|
let mut ids: Vec<String> =
|
||||||
|
self.points().iter().filter(|point| point.payload.room_id == room).flat_map(|point| point.payload.messages.iter().map(|message| message.event_id.clone())).collect();
|
||||||
|
ids.sort();
|
||||||
|
ids
|
||||||
|
}
|
||||||
|
|
||||||
|
fn cursor(&self, room: &str) -> Option<String> {
|
||||||
|
self.index.cursors().get(&cursor_key(room)).cloned()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn embedded(&self) -> Vec<String> {
|
||||||
|
self.embedder.texts()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn seed(&mut self, room: &str, event_id: &str, ts: u64) {
|
||||||
|
let mut seeded = vec![ChatPoint {
|
||||||
|
id: chunk_id(room, event_id),
|
||||||
|
vector: vec![1.0; 4],
|
||||||
|
payload: chunk_payload_of(room, &[(event_id, ts)]),
|
||||||
|
}];
|
||||||
|
self.index.upsert_chat(&seeded.split_off(0)).unwrap();
|
||||||
|
let mut cursors = self.index.cursors();
|
||||||
|
cursors.insert(cursor_key(room), format!("{}:{event_id}", chat_version()));
|
||||||
|
self.index.set_cursors(&cursors).unwrap();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn chunk_payload_of(room: &str, messages: &[(&str, u64)]) -> ChatPayload {
|
||||||
|
let messages: Vec<ChatMessage> = messages.iter().map(|(id, ts)| ChatMessage { event_id: (*id).into(), sender: "@a:s".into(), ts_ms: *ts, body: format!("seeded {id}") }).collect();
|
||||||
|
ChatPayload { room_id: room.into(), room_alias: None, text: "seeded".into(), messages }
|
||||||
|
}
|
||||||
|
|
||||||
|
fn tombstone(rig: &Rig, room: &str) -> Option<String> {
|
||||||
|
rig.index.cursors().get(&format!("{CHAT_PURGE_PREFIX}{room}")).cloned()
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- scope: refusals first ----
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn only_joined_rooms_are_looked_at_and_a_join_rule_is_not_the_test() {
|
||||||
|
let mut history = FakeHistory::default();
|
||||||
|
history.room("!a:s", vec![said("$a1", 1, "in a")]).room("!b:s", vec![said("$b1", 2, "in b")]);
|
||||||
|
// A room the reader has not joined: scripted, but never listed.
|
||||||
|
history.rooms.insert("!c:s".into(), Scripted { facts: plain("!c:s"), events: vec![said("$c1", 3, "in c")] });
|
||||||
|
let mut rig = Rig::new();
|
||||||
|
let done = rig.ok(&mut history);
|
||||||
|
assert!(done.is_clean());
|
||||||
|
assert!(!history.calls.iter().any(|call| call.contains("!c:s")), "{:?}", history.calls);
|
||||||
|
assert_eq!(rig.held_ids("!a:s"), ["$a1"]);
|
||||||
|
assert_eq!(rig.held_ids("!b:s"), ["$b1"]);
|
||||||
|
assert!(rig.held_ids("!c:s").is_empty() && done.rooms.iter().all(|report| report.room_id != "!c:s"));
|
||||||
|
let code = include_str!("room_scope.rs");
|
||||||
|
assert!(!code[..code.find("#[cfg(test)]").unwrap()].contains(&["join", "_rule"].concat()));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_dm_room_is_refused_unread_unindexed_and_purged() {
|
||||||
|
let mut history = FakeHistory::default();
|
||||||
|
history.room("!dm:s", vec![said("$d1", 1, "private")]).facts("!dm:s", |facts| facts.direct = true);
|
||||||
|
let mut rig = Rig::new();
|
||||||
|
rig.seed("!dm:s", "$old", 5);
|
||||||
|
let (done, journal) = rig.run(&mut history);
|
||||||
|
let done = done.unwrap();
|
||||||
|
assert!(!history.called("events", "!dm:s"));
|
||||||
|
assert!(rig.points().is_empty() && rig.cursor("!dm:s").is_none() && rig.embedded().is_empty());
|
||||||
|
assert_eq!(done.refused, [RefusedRoom { room_id: "!dm:s".into(), reason: Refusal::Direct, event_id: None }]);
|
||||||
|
assert!(!done.is_clean() && journal.iter().any(|line| line == "refused: !dm:s direct room"), "{journal:?}");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn each_estate_room_is_refused_by_id_and_by_alias_and_an_excluded_room_in_the_index_is_purged() {
|
||||||
|
let mut rig = Rig::new();
|
||||||
|
rig.estate = estate(&["!byid:s"]);
|
||||||
|
let mut history = FakeHistory::default();
|
||||||
|
history.room("!byid:s", vec![said("$i1", 1, "x")]);
|
||||||
|
for (n, alias) in ESTATE_ALIASES.iter().map(|alias| alias.to_string()).chain(["#USERS:GAMECULT.ORG".to_owned()]).enumerate() {
|
||||||
|
history.room(&format!("!al{n}:s"), vec![said("$x", 1, "x")]).facts(&format!("!al{n}:s"), |facts| facts.alias = Some(alias));
|
||||||
|
}
|
||||||
|
history.room("!alt:s", vec![said("$x", 1, "x")]).facts("!alt:s", |facts| facts.alt_aliases = vec!["#other:s".into(), "#patrons:gamecult.org".into()]);
|
||||||
|
history.room("!ok:s", vec![said("$ok1", 1, "fine")]);
|
||||||
|
history.room("!guide:s", vec![said("$g1", 1, "fine")]).facts("!guide:s", |facts| facts.alias = Some("#users-guide:gamecult.org".into()));
|
||||||
|
history.room("!club:s", vec![said("$c1", 1, "fine")]).facts("!club:s", |facts| facts.alias = Some("#patrons-club:gamecult.org".into()));
|
||||||
|
for room in ["!byid:s", "!al0:s", "!al1:s", "!al2:s", "!al3:s", "!alt:s"] {
|
||||||
|
rig.seed(room, "$old", 5);
|
||||||
|
}
|
||||||
|
let mut cursors = rig.index.cursors();
|
||||||
|
cursors.insert(format!("{CHAT_PURGE_PREFIX}!al0:s"), "1..2".into());
|
||||||
|
rig.index.set_cursors(&cursors).unwrap();
|
||||||
|
let done = rig.ok(&mut history);
|
||||||
|
assert!(!history.called("facts", "!byid:s") && !history.called("events", "!byid:s"));
|
||||||
|
for room in ["!al0:s", "!al1:s", "!al2:s", "!al3:s", "!alt:s"] {
|
||||||
|
assert!(history.called("facts", room) && !history.called("events", room), "{room}");
|
||||||
|
}
|
||||||
|
let refused: BTreeSet<&str> = done.refused.iter().map(|refused| refused.room_id.as_str()).collect();
|
||||||
|
assert_eq!(refused, ["!al0:s", "!al1:s", "!al2:s", "!al3:s", "!alt:s", "!byid:s"].into());
|
||||||
|
assert!(done.refused.iter().all(|refused| refused.reason == Refusal::Estate) && !done.is_clean());
|
||||||
|
let held: BTreeSet<String> = rig.points().iter().map(|point| point.payload.room_id.clone()).collect();
|
||||||
|
assert_eq!(held, ["!guide:s".to_owned(), "!club:s".to_owned(), "!ok:s".to_owned()].into(), "lookalikes and plain rooms are in; every estate room is out");
|
||||||
|
assert!(rig.index.cursors().keys().all(|key| !key.starts_with("chat:!al") && !key.starts_with("chat:!byid") && !key.starts_with("chat:!alt")));
|
||||||
|
assert_eq!(tombstone(&rig, "!al0:s").as_deref(), Some("1..2"), "a tombstone is never touched by a refusal");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn an_encrypted_room_is_refused_and_an_encrypted_event_in_a_plain_room_is_never_stored() {
|
||||||
|
let mut history = FakeHistory::default();
|
||||||
|
history.room("!enc:s", vec![said("$e1", 1, "x")]).facts("!enc:s", |facts| facts.encrypted = true);
|
||||||
|
history.room("!plain:s", vec![said("$p1", 1, "visible"), event("$p2", "@a:s", 2, EventKind::Other), said("$p3", 3, "also visible")]);
|
||||||
|
let mut rig = Rig::new();
|
||||||
|
let done = rig.ok(&mut history);
|
||||||
|
assert_eq!(done.refused[0].reason, Refusal::Encrypted);
|
||||||
|
assert!(!history.called("events", "!enc:s"));
|
||||||
|
assert_eq!(rig.held_ids("!plain:s"), ["$p1", "$p3"]);
|
||||||
|
assert_eq!(rig.cursor("!plain:s").unwrap(), format!("{}:$p3", chat_version()), "the cursor passes an Other event");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_bridged_room_is_refused_whole_before_any_event_is_read_and_its_chunks_are_purged() {
|
||||||
|
let mut history = FakeHistory::default();
|
||||||
|
for (n, state_type) in ["m.bridge", "uk.half-shot.bridge"].into_iter().enumerate() {
|
||||||
|
let room = format!("!br{n}:s");
|
||||||
|
history.room(&room, vec![said("$x", 1, "x")]).facts(&room, |facts| facts.bridge = Some(state_type.into()));
|
||||||
|
}
|
||||||
|
let mut rig = Rig::new();
|
||||||
|
rig.seed("!br0:s", "$old", 5);
|
||||||
|
let done = rig.ok(&mut history);
|
||||||
|
assert_eq!(done.refused.len(), 2);
|
||||||
|
assert!(done.refused.iter().all(|refused| matches!(refused.reason, Refusal::Bridged(_))));
|
||||||
|
assert!(!history.calls.iter().any(|call| call.starts_with("events:")));
|
||||||
|
assert!(rig.points().is_empty() && rig.index.cursors().is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_bridged_platforms_sender_or_content_refuses_the_room_before_anything_is_embedded_and_purges_what_it_held() {
|
||||||
|
let cases = [
|
||||||
|
event("$bad", "@discord_123:s", 2, EventKind::Message { body: "SECRET-BODY-TEXT".into() }),
|
||||||
|
event("$bad", "@_discord_9:s", 2, EventKind::Message { body: "SECRET-BODY-TEXT".into() }),
|
||||||
|
event("$bad", "@discordbot:s", 2, EventKind::Message { body: "SECRET-BODY-TEXT".into() }),
|
||||||
|
event("$bad", "@mautrix_discord:s", 2, EventKind::Message { body: "SECRET-BODY-TEXT".into() }),
|
||||||
|
event("$bad", "@_bridge_discordbot:s", 2, EventKind::Message { body: "SECRET-BODY-TEXT".into() }),
|
||||||
|
RoomEvent { content_keys: vec!["body".into(), "fi.mau.discord".into()], ..said("$bad", 2, "SECRET-BODY-TEXT") },
|
||||||
|
RoomEvent { content_keys: vec!["uk.half-shot.discord".into()], ..said("$bad", 2, "SECRET-BODY-TEXT") },
|
||||||
|
];
|
||||||
|
for bad in cases {
|
||||||
|
let mut history = FakeHistory::default();
|
||||||
|
history.room("!r:s", vec![said("$ok", 1, "before"), bad]);
|
||||||
|
history.room("!other:s", vec![said("$o1", 1, "other room")]);
|
||||||
|
let mut rig = Rig::new();
|
||||||
|
rig.seed("!r:s", "$prev", 0);
|
||||||
|
let (done, journal) = rig.run(&mut history);
|
||||||
|
let done = done.unwrap();
|
||||||
|
assert!(!rig.embedded().iter().any(|text| text.contains("SECRET-BODY-TEXT") || text.contains("before")), "nothing of the page is embedded");
|
||||||
|
assert!(rig.held_ids("!r:s").is_empty() && rig.cursor("!r:s").is_none());
|
||||||
|
assert_eq!(rig.held_ids("!other:s"), ["$o1"]);
|
||||||
|
assert_eq!(done.refused.len(), 1);
|
||||||
|
assert_eq!((done.refused[0].room_id.as_str(), done.refused[0].event_id.as_deref()), ("!r:s", Some("$bad")));
|
||||||
|
assert!(!done.is_clean());
|
||||||
|
assert!(journal.iter().all(|line| !line.contains("SECRET-BODY-TEXT")), "{journal:?}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_lookalike_of_the_bridged_platform_is_kept() {
|
||||||
|
let mut history = FakeHistory::default();
|
||||||
|
let kept = ["@discourse:s", "@disco:s", "@eureka:s"].map(|sender| event(&format!("${}", &sender[1..6]), sender, 1, EventKind::Message { body: "hi".into() }));
|
||||||
|
let mut keyed = said("$k", 2, "hi");
|
||||||
|
keyed.content_keys.push("discourse".into());
|
||||||
|
let mut events = kept.to_vec();
|
||||||
|
events.push(keyed);
|
||||||
|
history.room("!r:s", events);
|
||||||
|
let mut rig = Rig::new();
|
||||||
|
assert!(rig.ok(&mut history).is_clean());
|
||||||
|
assert_eq!(rig.held_ids("!r:s").len(), 4);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn the_collection_is_never_created_by_chat() {
|
||||||
|
let mut history = FakeHistory::default();
|
||||||
|
history.room("!r:s", vec![said("$a", 1, "x")]);
|
||||||
|
for described in [None, Some(DescribedSources::Absent), Some(DescribedSources::Unlabelled)] {
|
||||||
|
let mut rig = Rig::new();
|
||||||
|
match described {
|
||||||
|
None => rig.index.state.lock().unwrap().described = None,
|
||||||
|
Some(described) => rig.index.plant(described),
|
||||||
|
}
|
||||||
|
let error = format!("{:#}", rig.run(&mut history).0.unwrap_err());
|
||||||
|
assert!(error.contains("forge refresh creates it"), "{error}");
|
||||||
|
assert_eq!(rig.index.state.lock().unwrap().recreated, 0);
|
||||||
|
assert!(rig.points().is_empty() && rig.embedded().is_empty());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- redactions, edits, replays ----
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_redacted_event_is_absent_after_the_next_refresh_and_a_first_or_only_message_changes_or_drops_its_chunk() {
|
||||||
|
let mut history = FakeHistory::default();
|
||||||
|
history.room("!r:s", vec![said("$a", 10, "alpha"), said("$b", 20, "beta"), said("$c", 30, "gamma")]);
|
||||||
|
let mut rig = Rig::new();
|
||||||
|
rig.ok(&mut history);
|
||||||
|
let first_id = chunk_id("!r:s", "$a");
|
||||||
|
assert_eq!(rig.ids(), [first_id.clone()].into());
|
||||||
|
history.push("!r:s", vec![redaction("$r1", 40, "$b")]);
|
||||||
|
rig.ok(&mut history);
|
||||||
|
assert_eq!(rig.held_ids("!r:s"), ["$a", "$c"]);
|
||||||
|
assert!(!rig.points()[0].payload.text.contains("beta"));
|
||||||
|
history.push("!r:s", vec![redaction("$r2", 50, "$a")]);
|
||||||
|
rig.ok(&mut history);
|
||||||
|
assert_eq!(rig.ids(), [chunk_id("!r:s", "$c")].into(), "the first message left, so the id changed and the old one is gone");
|
||||||
|
history.push("!r:s", vec![redaction("$r3", 60, "$c")]);
|
||||||
|
rig.ok(&mut history);
|
||||||
|
assert!(rig.points().is_empty(), "a chunk the redaction empties is deleted");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_redaction_in_the_same_page_as_its_message_never_reaches_the_embedder() {
|
||||||
|
let mut history = FakeHistory::default();
|
||||||
|
history.room("!r:s", vec![said("$a", 10, "keep me"), said("$gone", 20, "REDACTED-BODY"), redaction("$r", 30, "$gone")]);
|
||||||
|
let mut rig = Rig::new();
|
||||||
|
rig.ok(&mut history);
|
||||||
|
assert!(rig.embedded().iter().all(|text| !text.contains("REDACTED-BODY")));
|
||||||
|
assert_eq!(rig.held_ids("!r:s"), ["$a"]);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn an_edit_replaces_the_body_in_place_and_a_stranger_an_unknown_target_or_an_edit_of_an_edit_change_nothing() {
|
||||||
|
let mut history = FakeHistory::default();
|
||||||
|
history.room("!r:s", vec![said("$a", 10, "old words"), said("$b", 20, "other")]);
|
||||||
|
let mut rig = Rig::new();
|
||||||
|
rig.ok(&mut history);
|
||||||
|
let before = rig.points();
|
||||||
|
history.push("!r:s", vec![edit("$e1", "@a:s", 30, "$a", "new words")]);
|
||||||
|
rig.ok(&mut history);
|
||||||
|
let after = rig.points();
|
||||||
|
assert_eq!(after.len(), 1);
|
||||||
|
let first = &after[0].payload.messages[0];
|
||||||
|
assert_eq!((first.event_id.as_str(), first.sender.as_str(), first.ts_ms, first.body.as_str()), ("$a", "@a:s", 10, "new words"));
|
||||||
|
assert_eq!(after[0].id, before[0].id);
|
||||||
|
assert!(after[0].payload.text.contains("new words") && !after[0].payload.text.contains("old words"));
|
||||||
|
assert!(rig.embedded().last().unwrap().contains("new words"));
|
||||||
|
let calls = rig.embedder.state.lock().unwrap().embedded.len();
|
||||||
|
history.push("!r:s", vec![edit("$e2", "@stranger:s", 40, "$a", "hijack"), edit("$e3", "@a:s", 41, "$nothing", "lost"), edit("$e4", "@a:s", 42, "$e1", "edit of an edit")]);
|
||||||
|
rig.ok(&mut history);
|
||||||
|
assert_eq!(rig.points(), after);
|
||||||
|
assert_eq!(rig.embedder.state.lock().unwrap().embedded.len(), calls, "no embed call for edits that change nothing");
|
||||||
|
assert_eq!(rig.cursor("!r:s").unwrap(), format!("{}:$e4", chat_version()), "the cursor is the last event of any kind");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn replay_after_a_crash_between_the_upsert_and_the_cursor_neither_duplicates_nor_loses() {
|
||||||
|
let mut history = FakeHistory::default();
|
||||||
|
history.room("!r:s", vec![said("$a", 10, "alpha"), said("$b", 20, "beta"), edit("$e", "@a:s", 25, "$a", "alpha two"), said("$c", 30, "gamma")]);
|
||||||
|
let mut rig = Rig::new();
|
||||||
|
// One cursor write (the empty start) succeeds, then the page's upsert is acknowledged and the next write fails.
|
||||||
|
rig.index.state.lock().unwrap().fail_cursor_writes_after = Some(1);
|
||||||
|
assert!(!rig.run(&mut history).0.unwrap().failed.is_empty());
|
||||||
|
let (ids, held) = (rig.ids(), rig.held_ids("!r:s"));
|
||||||
|
assert_eq!(held, ["$a", "$b", "$c"]);
|
||||||
|
rig.index.state.lock().unwrap().fail_cursor_writes_after = None;
|
||||||
|
let done = rig.ok(&mut history);
|
||||||
|
assert!(done.is_clean());
|
||||||
|
assert_eq!((rig.ids(), rig.held_ids("!r:s")), (ids, held));
|
||||||
|
assert!(rig.bodies().contains(&"alpha two".to_owned()));
|
||||||
|
assert_eq!(rig.cursor("!r:s").unwrap(), format!("{}:$c", chat_version()));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_failed_upsert_leaves_the_cursor_and_a_failed_embed_or_delete_deletes_nothing_the_index_held() {
|
||||||
|
let mut history = FakeHistory::default();
|
||||||
|
history.room("!r:s", vec![said("$a", 10, "alpha"), said("$b", 20, "beta")]);
|
||||||
|
let mut rig = Rig::new();
|
||||||
|
rig.ok(&mut history);
|
||||||
|
let (cursor, points) = (rig.cursor("!r:s"), rig.points());
|
||||||
|
history.push("!r:s", vec![said("$c", 30, "gamma")]);
|
||||||
|
rig.index.state.lock().unwrap().fail_chat_upserts = true;
|
||||||
|
assert_eq!(rig.run(&mut history).0.unwrap().failed, ["!r:s"]);
|
||||||
|
assert_eq!((rig.cursor("!r:s"), rig.points()), (cursor.clone(), points.clone()));
|
||||||
|
rig.index.state.lock().unwrap().fail_chat_upserts = false;
|
||||||
|
rig.embedder.state.lock().unwrap().embed_fails = true;
|
||||||
|
assert_eq!(rig.run(&mut history).0.unwrap().failed, ["!r:s"]);
|
||||||
|
assert_eq!((rig.cursor("!r:s"), rig.points()), (cursor.clone(), points.clone()));
|
||||||
|
rig.embedder.state.lock().unwrap().embed_fails = false;
|
||||||
|
// A redaction of the first message writes a new chunk, then fails to delete the old one: the cursor stays.
|
||||||
|
history.push("!r:s", vec![redaction("$r", 40, "$a")]);
|
||||||
|
rig.index.state.lock().unwrap().fail_chat_deletes = true;
|
||||||
|
assert_eq!(rig.run(&mut history).0.unwrap().failed, ["!r:s"]);
|
||||||
|
assert_eq!(rig.cursor("!r:s"), cursor);
|
||||||
|
rig.index.state.lock().unwrap().fail_chat_deletes = false;
|
||||||
|
assert!(rig.ok(&mut history).is_clean());
|
||||||
|
assert_eq!(rig.held_ids("!r:s"), ["$b", "$c"]);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- chunking through the job ----
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn appending_to_the_tail_over_many_runs_equals_one_run_over_all() {
|
||||||
|
let all: Vec<RoomEvent> = (0..80).map(|n| said(&format!("$m{n:03}"), 1_000 * n + if n >= 40 { 5_000_000 } else { 0 }, &format!("message number {n} {}", "x".repeat((n as usize * 37) % 400)))).collect();
|
||||||
|
let mut whole_history = FakeHistory::default();
|
||||||
|
whole_history.room("!r:s", all.clone());
|
||||||
|
let mut whole = Rig::new();
|
||||||
|
whole.ok(&mut whole_history);
|
||||||
|
let mut grown_history = FakeHistory::default();
|
||||||
|
grown_history.room("!r:s", vec![]);
|
||||||
|
let mut grown = Rig::new();
|
||||||
|
for batch in all.chunks(7) {
|
||||||
|
grown_history.push("!r:s", batch.to_vec());
|
||||||
|
grown.ok(&mut grown_history);
|
||||||
|
}
|
||||||
|
assert!(whole.points().len() > 2);
|
||||||
|
let shape = |rig: &Rig| rig.points().into_iter().map(|point| (point.id, point.payload)).collect::<Vec<_>>();
|
||||||
|
assert_eq!(shape(&grown), shape(&whole));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_long_message_is_cut_with_a_marker_and_every_point_satisfies_the_payload_contract() {
|
||||||
|
let mut history = FakeHistory::default();
|
||||||
|
history.room("!r:s", vec![said("$a", 10, &"é".repeat(CHAT_BODY_CHARS_MAX * 2)), said("$b", 20, "short"), said("$c", 99_000_000, "later")]);
|
||||||
|
history.facts("!r:s", |facts| {
|
||||||
|
facts.alias = Some("#swarm:gamecult.org".into());
|
||||||
|
facts.name = Some("Swarm".into());
|
||||||
|
});
|
||||||
|
let mut rig = Rig::new();
|
||||||
|
rig.ok(&mut history);
|
||||||
|
let points = rig.points();
|
||||||
|
assert_eq!(points.len(), 2);
|
||||||
|
for point in &points {
|
||||||
|
let payload = &point.payload;
|
||||||
|
assert_eq!((payload.room_id.as_str(), payload.room_alias.as_deref()), ("!r:s", Some("#swarm:gamecult.org")));
|
||||||
|
assert_eq!(point.id, chunk_id("!r:s", &payload.messages[0].event_id));
|
||||||
|
assert_eq!(payload.text, super::chat_chunk::embedded_text("Swarm", &payload.messages));
|
||||||
|
assert!(!payload.messages.is_empty());
|
||||||
|
}
|
||||||
|
let long = &points.iter().find(|point| point.payload.messages[0].event_id == "$a").unwrap().payload.messages[0];
|
||||||
|
assert_eq!(long.body.chars().count(), CHAT_BODY_CHARS_MAX);
|
||||||
|
assert!(long.body.ends_with("[cut]"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_lost_cursor_event_or_a_version_bump_re_ingests_that_room_alone() {
|
||||||
|
let mut history = FakeHistory::default();
|
||||||
|
history.room("!r:s", vec![said("$a", 10, "alpha"), said("$b", 20, "beta")]).room("!q:s", vec![said("$q1", 10, "quiet")]);
|
||||||
|
let mut rig = Rig::new();
|
||||||
|
rig.ok(&mut history);
|
||||||
|
let q_ids: BTreeSet<String> = rig.points().iter().filter(|point| point.payload.room_id == "!q:s").map(|point| point.id.clone()).collect();
|
||||||
|
let recreated = rig.index.state.lock().unwrap().recreated;
|
||||||
|
for bump in [Some("0.0".to_owned()), None] {
|
||||||
|
let mut cursors = rig.index.cursors();
|
||||||
|
let value = match &bump {
|
||||||
|
Some(old) => format!("{old}:$b"),
|
||||||
|
None => format!("{}:$gone-from-the-server", chat_version()),
|
||||||
|
};
|
||||||
|
cursors.insert(cursor_key("!r:s"), value);
|
||||||
|
rig.index.set_cursors(&cursors).unwrap();
|
||||||
|
let before = rig.embedded().len();
|
||||||
|
assert!(rig.ok(&mut history).is_clean());
|
||||||
|
assert_eq!(rig.held_ids("!r:s"), ["$a", "$b"]);
|
||||||
|
assert!(rig.embedded().len() > before, "the room was re-embedded");
|
||||||
|
assert_eq!(rig.points().iter().filter(|point| point.payload.room_id == "!q:s").map(|point| point.id.clone()).collect::<BTreeSet<_>>(), q_ids);
|
||||||
|
assert_eq!(rig.cursor("!r:s").unwrap(), format!("{}:$b", chat_version()));
|
||||||
|
}
|
||||||
|
assert_eq!(rig.index.state.lock().unwrap().recreated, recreated);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn retire_chat_removes_rooms_no_longer_kept_and_given_none_all_chat_and_touches_nothing_else() {
|
||||||
|
let mut rig = Rig::new();
|
||||||
|
rig.index.upsert_sources(&[SourcePoint {
|
||||||
|
id: "forge-1".into(),
|
||||||
|
vector: vec![1.0; 4],
|
||||||
|
payload: SourcePayload { repo: "Repo".into(), path: "a".into(), commit: "c".into(), blob: "b".into(), line_start: 1, line_end: 1, chunk_index: 0, ext: "rs".into(), text: "t".into() },
|
||||||
|
}])
|
||||||
|
.unwrap();
|
||||||
|
for room in ["!a:s", "!b:s"] {
|
||||||
|
rig.seed(room, "$e", 1);
|
||||||
|
}
|
||||||
|
let mut cursors = rig.index.cursors();
|
||||||
|
cursors.insert("Repo".into(), "c".into());
|
||||||
|
cursors.insert(format!("{CHAT_PURGE_PREFIX}!a:s"), "1..2".into());
|
||||||
|
rig.index.set_cursors(&cursors).unwrap();
|
||||||
|
let retired = retire_chat(&mut rig.index, &["!a:s".to_owned()].into()).unwrap();
|
||||||
|
assert_eq!(retired.retired, ["!b:s"]);
|
||||||
|
assert_eq!(rig.held_ids("!a:s"), ["$e"]);
|
||||||
|
assert!(rig.held_ids("!b:s").is_empty() && rig.cursor("!b:s").is_none());
|
||||||
|
assert_eq!(retire_chat(&mut rig.index, &BTreeSet::new()).unwrap().retired, ["!a:s"]);
|
||||||
|
assert!(rig.points().is_empty());
|
||||||
|
assert_eq!(rig.index.state.lock().unwrap().points.len(), 1, "a forge point stays");
|
||||||
|
let kept: Vec<String> = rig.index.cursors().into_keys().collect();
|
||||||
|
assert_eq!(kept, ["Repo".to_owned(), format!("{CHAT_PURGE_PREFIX}!a:s")]);
|
||||||
|
// A room the reader left is retired by the refresh itself.
|
||||||
|
let mut history = FakeHistory::default();
|
||||||
|
history.room("!r:s", vec![said("$a", 1, "x")]);
|
||||||
|
rig.seed("!gone:s", "$e", 1);
|
||||||
|
rig.ok(&mut history);
|
||||||
|
assert!(rig.held_ids("!gone:s").is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- the credential scrub through the job ----
|
||||||
|
|
||||||
|
fn planted() -> Vec<(ScrubKind, String, String)> {
|
||||||
|
let t = |kind, secret: String, needle: &str| (kind, secret, needle.to_owned());
|
||||||
|
vec![
|
||||||
|
t(ScrubKind::AuthorizationHeader, format!("Authorization: Bearer {}", "a1B2".repeat(8)), "a1B2a1B2a1B2"),
|
||||||
|
t(ScrubKind::Bearer, format!("Bearer {}", "z9Y8".repeat(6)), "z9Y8z9Y8z9Y8"),
|
||||||
|
t(ScrubKind::Bearer, format!("bearer:{}", "z9Y8".repeat(6)), "z9Y8z9Y8z9Y8"),
|
||||||
|
t(ScrubKind::KeyShape, format!("sk-{}", "q7W3".repeat(8)), "q7W3q7W3q7W3"),
|
||||||
|
t(ScrubKind::KeyShape, format!("sk_live_{}", "m4N5".repeat(6)), "m4N5m4N5m4N5"),
|
||||||
|
t(ScrubKind::KeyShape, format!("ghp_{}", "k2L9".repeat(9)), "k2L9k2L9k2L9"),
|
||||||
|
t(ScrubKind::KeyShape, format!("xxx-ghp_{}", "k2L9".repeat(9)), "k2L9k2L9k2L9"),
|
||||||
|
t(ScrubKind::KeyShape, format!("github_pat_{}_{}", "A1b2C3".repeat(4) + "ab", "d4E5f6".repeat(9) + "abcde"), "A1b2C3A1b2C3"),
|
||||||
|
t(ScrubKind::KeyShape, format!("glpat-{}", "r6T7".repeat(5)), "r6T7r6T7r6T7"),
|
||||||
|
t(ScrubKind::KeyShape, "xoxb-123456789012-987654321098-aBcDeFgHiJkLmNoP".into(), "aBcDeFgHiJkL"),
|
||||||
|
t(ScrubKind::KeyShape, "xoxa-2-123456789012-aBcDeFgHiJkLmNoP".into(), "aBcDeFgHiJkL"),
|
||||||
|
t(ScrubKind::KeyShape, "xoxp-123456789012-987654321098-aBcDeFgHiJkLmNoP".into(), "aBcDeFgHiJkL"),
|
||||||
|
t(ScrubKind::KeyShape, "xoxr-123456789012-987654321098-aBcDeFgHiJkLmNoP".into(), "aBcDeFgHiJkL"),
|
||||||
|
t(ScrubKind::KeyShape, "xoxs-123456789012-987654321098-aBcDeFgHiJkLmNoP".into(), "aBcDeFgHiJkL"),
|
||||||
|
t(ScrubKind::KeyShape, "AKIA0123456789ABCDEF".into(), "0123456789ABCDEF"),
|
||||||
|
t(ScrubKind::KeyShape, "xAKIA0123456789ABCDEF".into(), "0123456789ABCDEF"),
|
||||||
|
t(ScrubKind::KeyShape, format!("AIza{}", "Sy1_".repeat(9)), "Sy1_Sy1_Sy1_"),
|
||||||
|
t(ScrubKind::KeyShape, format!("syt_{}_{}_{}", "YWxpY2U", "aBcDeFgHiJkLmNoPqR", "0a1B2c"), "aBcDeFgHiJkLmNoPqR"),
|
||||||
|
t(ScrubKind::KeyShape, format!("mct_{}", "w4X5".repeat(8)), "w4X5w4X5w4X5"),
|
||||||
|
t(ScrubKind::AccessToken, "https://h/_matrix/client/v3/sync?access_token=abcDEF123456xyz&since=s1".into(), "abcDEF123456xyz"),
|
||||||
|
t(ScrubKind::AccessToken, "{\"access_token\": \"tokenValue987654\", \"user\": 1}".into(), "tokenValue987654"),
|
||||||
|
t(ScrubKind::Jwt, "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJmYWtlIn0.c2lnbmF0dXJlZmFrZQ".into(), "c2lnbmF0dXJlZmFrZQ"),
|
||||||
|
t(ScrubKind::PemBlock, "-----BEGIN RSA PRIVATE KEY-----\nMIIEowIBAAKCAQEAfake\n-----END RSA PRIVATE KEY-----".into(), "MIIEowIBAAKCAQEAfake"),
|
||||||
|
t(ScrubKind::PemBlock, "-----BEGIN PRIVATE KEY-----\nMIIBVQIBADANfakeNOEND".into(), "MIIBVQIBADANfake"),
|
||||||
|
t(ScrubKind::PemBlock, "-----BEGIN PGP PRIVATE KEY BLOCK-----\nlQOYBFakeFakeFake\n-----END PGP PRIVATE KEY BLOCK-----".into(), "lQOYBFakeFake"),
|
||||||
|
t(ScrubKind::UrlUserinfo, "https://user:hunter2hunter2@host.example/r.git".into(), "hunter2hunter2"),
|
||||||
|
t(ScrubKind::UrlUserinfo, "https://tokenvalue12345@host.example/".into(), "tokenvalue12345"),
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn planted_credentials_never_reach_the_chunk_the_embedder_or_the_journal_in_a_message_or_an_edit() {
|
||||||
|
for (kind, secret, needle) in planted() {
|
||||||
|
for shape in ["see {} now", "see {}", "{}\nnext line"] {
|
||||||
|
let (dirty, clean) = (shape.replace("{}", &secret), "nothing here".to_owned());
|
||||||
|
let mut history = FakeHistory::default();
|
||||||
|
history.room("!r:s", vec![said("$m", 10, &dirty), said("$e", 20, &clean), edit("$ed", "@a:s", 30, "$e", &dirty)]);
|
||||||
|
let mut rig = Rig::new();
|
||||||
|
let (done, journal) = rig.run(&mut history);
|
||||||
|
let done = done.unwrap();
|
||||||
|
assert!(done.is_clean(), "a scrub hit does not make a run unclean");
|
||||||
|
let stored = format!("{:?}", rig.points().iter().map(|point| (&point.payload.messages, &point.payload.text)).collect::<Vec<_>>());
|
||||||
|
for seen in [stored, rig.embedded().join("\n"), journal.join("\n")] {
|
||||||
|
assert!(!seen.contains(&needle), "{kind}: {secret:?} leaked into {seen}");
|
||||||
|
}
|
||||||
|
assert!(rig.bodies().iter().filter(|body| body.contains("<redacted>")).count() >= 2, "{kind}: both bodies show the replacement");
|
||||||
|
let hits: Vec<(&str, ScrubKind)> = done.scrubbed.iter().map(|hit| (hit.event_id.as_str(), hit.kind)).collect();
|
||||||
|
assert!(hits.contains(&("$m", kind)) && hits.contains(&("$ed", kind)), "{kind}: {hits:?}");
|
||||||
|
assert!(done.scrubbed.iter().all(|hit| hit.room_id == "!r:s"));
|
||||||
|
assert!(journal.contains(&format!("scrubbed: !r:s $m {kind}")), "{journal:?}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn the_scrub_meets_edits_redactions_the_body_cut_replays_and_version_bumps() {
|
||||||
|
let token = format!("ghp_{}", "k2L9".repeat(9));
|
||||||
|
let mut history = FakeHistory::default();
|
||||||
|
history.room(
|
||||||
|
"!r:s",
|
||||||
|
vec![
|
||||||
|
said("$clean", 10, "all clean"),
|
||||||
|
said("$dirty", 20, &format!("token {token} inside")),
|
||||||
|
edit("$adds", "@a:s", 30, "$clean", &format!("now with {token}")),
|
||||||
|
edit("$removes", "@a:s", 40, "$dirty", "token removed"),
|
||||||
|
said("$gone", 50, &format!("secret {token}")),
|
||||||
|
redaction("$r", 60, "$gone"),
|
||||||
|
said("$straddle", 70, &format!("{} {token} {}", "x".repeat(CHAT_BODY_CHARS_MAX - 10), "y".repeat(100))),
|
||||||
|
],
|
||||||
|
);
|
||||||
|
let mut rig = Rig::new();
|
||||||
|
let done = rig.ok(&mut history);
|
||||||
|
assert!(rig.bodies().iter().all(|body| !body.contains("k2L9k2L9")));
|
||||||
|
assert!(rig.embedded().iter().all(|text| !text.contains("k2L9k2L9")));
|
||||||
|
let by_id: BTreeMap<String, String> = rig.points().iter().flat_map(|point| point.payload.messages.iter().map(|message| (message.event_id.clone(), message.body.clone()))).collect();
|
||||||
|
assert_eq!(by_id["$clean"], "now with <redacted>");
|
||||||
|
assert_eq!(by_id["$dirty"], "token removed");
|
||||||
|
assert!(!by_id.contains_key("$gone"));
|
||||||
|
assert!(by_id["$straddle"].chars().count() <= CHAT_BODY_CHARS_MAX && !by_id["$straddle"].contains("ghp_"));
|
||||||
|
assert!(done.scrubbed.iter().any(|hit| hit.event_id == "$adds") && done.scrubbed.iter().any(|hit| hit.event_id == "$gone"));
|
||||||
|
let (ids, texts) = (rig.ids(), rig.points().into_iter().map(|point| point.payload.text).collect::<Vec<_>>());
|
||||||
|
// Replay from the start, as after a lost cursor: identical ids and text.
|
||||||
|
let mut cursors = rig.index.cursors();
|
||||||
|
cursors.remove(&cursor_key("!r:s"));
|
||||||
|
rig.index.set_cursors(&cursors).unwrap();
|
||||||
|
rig.ok(&mut history);
|
||||||
|
assert_eq!((rig.ids(), rig.points().into_iter().map(|point| point.payload.text).collect::<Vec<_>>()), (ids, texts));
|
||||||
|
// An older scrub version re-ingests the room and scrubs it again from the history.
|
||||||
|
let mut cursors = rig.index.cursors();
|
||||||
|
cursors.insert(cursor_key("!r:s"), format!("{}.0:$straddle", huginn_mind::redact::SCRUB_VERSION - 1));
|
||||||
|
rig.index.set_cursors(&cursors).unwrap();
|
||||||
|
rig.ok(&mut history);
|
||||||
|
assert!(rig.bodies().iter().all(|body| !body.contains("k2L9k2L9")));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn ordinary_text_is_stored_byte_for_byte() {
|
||||||
|
let mut history = FakeHistory::default();
|
||||||
|
let same = "risk-averse task-force sk- 3f2b8c1d9e7a4b5c6d0e1f2a3b4c5d6e7f8a9b0c !abc:s user@example.com ssh://git@host/r.git the bearer of bad news";
|
||||||
|
history.room("!r:s", vec![said("$a", 1, same)]);
|
||||||
|
let mut rig = Rig::new();
|
||||||
|
let done = rig.ok(&mut history);
|
||||||
|
assert_eq!(rig.bodies(), [same]);
|
||||||
|
assert!(done.scrubbed.is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- steward purge ----
|
||||||
|
|
||||||
|
fn window(from_ms: u64, to_ms: u64) -> PurgeWindow {
|
||||||
|
PurgeWindow { from_ms, to_ms }
|
||||||
|
}
|
||||||
|
|
||||||
|
fn purge_history() -> FakeHistory {
|
||||||
|
let mut history = FakeHistory::default();
|
||||||
|
// t1 = 1000, t2 = 3000: $b at t1 and $c inside are purged, $d at t2 is kept.
|
||||||
|
history.room("!r:s", vec![said("$a", 500, "before"), said("$b", 1000, "PURGED-B"), said("$c", 2000, "PURGED-C"), said("$d", 3000, "after"), said("$e", 4000, "later")]);
|
||||||
|
history.room("!other:s", vec![said("$o1", 1000, "same time other room"), said("$o2", 2000, "same time")]);
|
||||||
|
history
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_purge_window_removes_exactly_its_events_and_re_chunks_the_neighbours() {
|
||||||
|
let mut history = purge_history();
|
||||||
|
let mut rig = Rig::new();
|
||||||
|
rig.ok(&mut history);
|
||||||
|
let other_ids: BTreeSet<String> = rig.points().iter().filter(|point| point.payload.room_id == "!other:s").map(|point| point.id.clone()).collect();
|
||||||
|
assert_eq!(rig.held_ids("!r:s"), ["$a", "$b", "$c", "$d", "$e"]);
|
||||||
|
assert!(request_purge(&mut rig.index, "!r:s", window(1000, 3000)).unwrap());
|
||||||
|
let embedded_before = rig.embedded().len();
|
||||||
|
rig.ok(&mut history);
|
||||||
|
assert_eq!(rig.held_ids("!r:s"), ["$a", "$d", "$e"], "t1 is inside, t2 is not");
|
||||||
|
assert!(rig.embedded()[embedded_before..].iter().all(|text| !text.contains("PURGED")));
|
||||||
|
assert!(rig.points().iter().all(|point| !point.payload.text.contains("PURGED")));
|
||||||
|
assert_eq!(rig.points().iter().filter(|point| point.payload.room_id == "!other:s").map(|point| point.id.clone()).collect::<BTreeSet<_>>(), other_ids);
|
||||||
|
assert_eq!(rig.held_ids("!other:s"), ["$o1", "$o2"]);
|
||||||
|
assert_eq!(tombstone(&rig, "!r:s").as_deref(), Some("1000..3000"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_purged_first_message_changes_the_chunk_id_and_an_emptied_chunk_is_deleted() {
|
||||||
|
let mut history = FakeHistory::default();
|
||||||
|
history.room("!r:s", vec![said("$a", 1000, "x"), said("$b", 2000, "y"), said("$c", 99_000_000, "far")]);
|
||||||
|
let mut rig = Rig::new();
|
||||||
|
rig.ok(&mut history);
|
||||||
|
assert_eq!(rig.ids().len(), 2);
|
||||||
|
request_purge(&mut rig.index, "!r:s", window(900, 1500)).unwrap();
|
||||||
|
rig.ok(&mut history);
|
||||||
|
assert_eq!(rig.ids(), [chunk_id("!r:s", "$b"), chunk_id("!r:s", "$c")].into());
|
||||||
|
request_purge(&mut rig.index, "!r:s", window(99_000_000, 99_000_001)).unwrap();
|
||||||
|
rig.ok(&mut history);
|
||||||
|
assert_eq!(rig.held_ids("!r:s"), ["$b"]);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_purged_window_does_not_come_back() {
|
||||||
|
let mut history = purge_history();
|
||||||
|
let mut rig = Rig::new();
|
||||||
|
rig.ok(&mut history);
|
||||||
|
request_purge(&mut rig.index, "!r:s", window(1000, 3000)).unwrap();
|
||||||
|
rig.ok(&mut history);
|
||||||
|
let want = ["$a", "$d", "$e"];
|
||||||
|
// New events, an in-window edit of an outside message, a redaction.
|
||||||
|
history.push("!r:s", vec![said("$f", 5000, "new"), edit("$ed", "@a:s", 2500, "$a", "EDIT-IN-WINDOW"), redaction("$rd", 6000, "$e")]);
|
||||||
|
rig.ok(&mut history);
|
||||||
|
assert_eq!(rig.held_ids("!r:s"), ["$a", "$d", "$f"]);
|
||||||
|
assert!(rig.bodies().contains(&"before".to_owned()) && !rig.bodies().iter().any(|body| body.contains("EDIT-IN-WINDOW")), "the in-window edit is dropped and the original stays");
|
||||||
|
// A lost cursor re-ingests from the start; a version bump re-ingests too.
|
||||||
|
for cursor in [None, Some("0.0:$f".to_owned())] {
|
||||||
|
let mut cursors = rig.index.cursors();
|
||||||
|
match cursor {
|
||||||
|
None => cursors.remove(&cursor_key("!r:s")),
|
||||||
|
Some(old) => cursors.insert(cursor_key("!r:s"), old),
|
||||||
|
};
|
||||||
|
rig.index.set_cursors(&cursors).unwrap();
|
||||||
|
rig.ok(&mut history);
|
||||||
|
assert!(rig.bodies().iter().all(|body| !body.contains("PURGED")), "{:?}", rig.held_ids("!r:s"));
|
||||||
|
assert_eq!(rig.held_ids("!r:s"), ["$a", "$d", "$f"], "$e stays redacted by the history; {want:?} before the new events");
|
||||||
|
}
|
||||||
|
// The reader leaves the room and joins again.
|
||||||
|
let mut left = FakeHistory::default();
|
||||||
|
left.room("!other:s", vec![said("$o1", 1000, "x")]);
|
||||||
|
rig.ok(&mut left);
|
||||||
|
assert!(rig.held_ids("!r:s").is_empty() && tombstone(&rig, "!r:s").is_some());
|
||||||
|
rig.ok(&mut history);
|
||||||
|
assert!(rig.bodies().iter().all(|body| !body.contains("PURGED")));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_purge_of_an_unindexed_window_writes_only_the_tombstone_and_still_holds() {
|
||||||
|
let mut rig = Rig::new();
|
||||||
|
let writes = rig.index.state.lock().unwrap().cursor_writes.len();
|
||||||
|
assert!(request_purge(&mut rig.index, "!never:s", window(10, 20)).unwrap());
|
||||||
|
assert_eq!(rig.index.state.lock().unwrap().cursor_writes.len(), writes + 1);
|
||||||
|
assert!(rig.points().is_empty() && rig.embedded().is_empty());
|
||||||
|
let mut history = FakeHistory::default();
|
||||||
|
history.room("!never:s", vec![said("$a", 5, "ok"), said("$b", 15, "PURGED"), said("$c", 25, "ok")]);
|
||||||
|
rig.ok(&mut history);
|
||||||
|
assert_eq!(rig.held_ids("!never:s"), ["$a", "$c"]);
|
||||||
|
assert!(rig.embedded().iter().all(|text| !text.contains("PURGED")));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_repeated_or_overlapping_request_changes_nothing_or_merges_and_the_33rd_window_is_refused() {
|
||||||
|
let mut rig = Rig::new();
|
||||||
|
assert!(request_purge(&mut rig.index, "!r:s", window(10, 20)).unwrap());
|
||||||
|
let writes = rig.index.state.lock().unwrap().cursor_writes.len();
|
||||||
|
assert!(!request_purge(&mut rig.index, "!r:s", window(10, 20)).unwrap());
|
||||||
|
assert!(!request_purge(&mut rig.index, "!r:s", window(12, 18)).unwrap());
|
||||||
|
assert_eq!(rig.index.state.lock().unwrap().cursor_writes.len(), writes, "no write for a request the tombstone already holds");
|
||||||
|
assert!(request_purge(&mut rig.index, "!r:s", window(15, 30)).unwrap());
|
||||||
|
assert_eq!(tombstone(&rig, "!r:s").as_deref(), Some("10..30"));
|
||||||
|
for n in 0..(PURGE_WINDOWS_MAX as u64 - 1) {
|
||||||
|
request_purge(&mut rig.index, "!r:s", window(100 + n * 10, 105 + n * 10)).unwrap();
|
||||||
|
}
|
||||||
|
let error = format!("{:#}", request_purge(&mut rig.index, "!r:s", window(10_000, 10_001)).unwrap_err());
|
||||||
|
assert!(error.contains("32"), "{error}");
|
||||||
|
assert!(request_purge(&mut Rig::new().index, "!r:s", window(5, 5)).is_err());
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- the source stays in its lane ----
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn the_ingest_code_is_pure_over_its_ports_and_names_no_credential_shape_no_second_literal_and_no_join_rule() {
|
||||||
|
let sources = [("chat.rs", include_str!("chat.rs")), ("chat_chunk.rs", include_str!("chat_chunk.rs")), ("chat_purge.rs", include_str!("chat_purge.rs")), ("room_scope.rs", include_str!("room_scope.rs"))];
|
||||||
|
let process = ["Command", "::new"].concat();
|
||||||
|
for (name, source) in sources {
|
||||||
|
let code = source.split("#[cfg(test)]").next().unwrap();
|
||||||
|
assert!(!code.contains(&process) && !code.contains("std::process"), "{name}");
|
||||||
|
assert!(!code.to_ascii_lowercase().contains("join_rule"), "{name}");
|
||||||
|
if name != "room_scope.rs" {
|
||||||
|
assert!(!code.to_ascii_lowercase().contains("discord"), "{name}");
|
||||||
|
}
|
||||||
|
for shape in ["sk-", "sk_live_", "ghp_", "github_pat_", "glpat-", "AKIA", "xoxb-", "PRIVATE KEY", "Bearer"] {
|
||||||
|
assert!(!code.contains(shape), "{name} names {shape}");
|
||||||
|
}
|
||||||
|
let defined = usize::from(name == "chat.rs");
|
||||||
|
assert_eq!((code.matches("\"chat:").count(), code.matches("\"chat-purge:").count()), (defined, defined), "{name}: each prefix is defined once");
|
||||||
|
}
|
||||||
|
let chat = include_str!("chat.rs");
|
||||||
|
assert_eq!(chat.matches("scrub(").count(), 2, "scrub is called for a body and for the room name, nowhere else");
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- redacted edits, redacted originals, late edits ----
|
||||||
|
|
||||||
|
const EDIT_SECRET: &str = "EDIT-SECRET";
|
||||||
|
|
||||||
|
fn nothing_secret_stored(rig: &Rig) {
|
||||||
|
let stored = format!("{:?}", rig.points().iter().map(|point| (&point.payload.messages, &point.payload.text)).collect::<Vec<_>>());
|
||||||
|
assert!(!stored.contains(EDIT_SECRET), "{stored}");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_redacted_edit_in_the_same_page_reverts_the_message_to_its_earlier_body() {
|
||||||
|
let mut history = FakeHistory::default();
|
||||||
|
history.room("!r:s", vec![said("$m", 10, "original words"), edit("$e1", "@a:s", 20, "$m", "first edit"), edit("$e2", "@a:s", 30, "$m", EDIT_SECRET), redaction("$r", 40, "$e2")]);
|
||||||
|
let mut rig = Rig::new();
|
||||||
|
rig.ok(&mut history);
|
||||||
|
assert_eq!(rig.bodies(), ["first edit"]);
|
||||||
|
assert!(rig.embedded().iter().all(|text| !text.contains(EDIT_SECRET)));
|
||||||
|
history.room("!q:s", vec![said("$m", 10, "orig"), edit("$e", "@a:s", 20, "$m", EDIT_SECRET), redaction("$r", 30, "$e")]);
|
||||||
|
rig.ok(&mut history);
|
||||||
|
assert!(rig.bodies().contains(&"orig".to_owned()));
|
||||||
|
nothing_secret_stored(&rig);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_redaction_of_an_edit_a_later_run_stored_rebuilds_the_room_and_the_secret_does_not_return() {
|
||||||
|
let mut history = FakeHistory { prune: true, ..FakeHistory::default() };
|
||||||
|
history.room("!r:s", vec![said("$m", 10, "original"), edit("$e", "@a:s", 20, "$m", EDIT_SECRET)]);
|
||||||
|
history.room("!q:s", vec![said("$q", 10, "quiet")]);
|
||||||
|
let mut rig = Rig::new();
|
||||||
|
rig.ok(&mut history);
|
||||||
|
assert!(rig.bodies().iter().any(|body| body == EDIT_SECRET), "the edit applied while it stood");
|
||||||
|
let recreated = rig.index.state.lock().unwrap().recreated;
|
||||||
|
history.push("!r:s", vec![redaction("$r", 30, "$e")]);
|
||||||
|
let done = rig.ok(&mut history);
|
||||||
|
assert!(done.is_clean());
|
||||||
|
nothing_secret_stored(&rig);
|
||||||
|
assert!(rig.bodies().contains(&"original".to_owned()));
|
||||||
|
assert_eq!(rig.held_ids("!q:s"), ["$q"]);
|
||||||
|
assert_eq!(rig.index.state.lock().unwrap().recreated, recreated);
|
||||||
|
assert_eq!(rig.cursor("!r:s").unwrap(), format!("{}:$r", chat_version()));
|
||||||
|
// The rebuild does not loop: the redaction it replays is not a trigger the second time, and nothing changes.
|
||||||
|
let ids = rig.ids();
|
||||||
|
rig.ok(&mut history);
|
||||||
|
assert_eq!(rig.ids(), ids);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_redacted_original_takes_its_live_and_late_edits_with_it_in_a_page_across_runs_and_across_a_rebuild() {
|
||||||
|
let script = |history: &mut FakeHistory| {
|
||||||
|
history.room("!r:s", vec![said("$m", 10, "original"), edit("$e1", "@a:s", 20, "$m", "live edit"), redaction("$re", 25, "$e1"), redaction("$rm", 30, "$m"), edit("$late", "@a:s", 40, "$m", EDIT_SECRET)]);
|
||||||
|
history.prune = true;
|
||||||
|
};
|
||||||
|
// One page.
|
||||||
|
let (mut history, mut rig) = (FakeHistory::default(), Rig::new());
|
||||||
|
script(&mut history);
|
||||||
|
rig.ok(&mut history);
|
||||||
|
nothing_secret_stored(&rig);
|
||||||
|
assert!(rig.points().is_empty());
|
||||||
|
// Across runs: the original and a live edit stand, then it is redacted, then a late edit arrives.
|
||||||
|
let (mut history, mut rig) = (FakeHistory { prune: true, ..FakeHistory::default() }, Rig::new());
|
||||||
|
history.room("!r:s", vec![said("$m", 10, "original"), edit("$e1", "@a:s", 20, "$m", EDIT_SECRET)]);
|
||||||
|
rig.ok(&mut history);
|
||||||
|
history.push("!r:s", vec![redaction("$rm", 30, "$m")]);
|
||||||
|
rig.ok(&mut history);
|
||||||
|
nothing_secret_stored(&rig);
|
||||||
|
history.push("!r:s", vec![edit("$late", "@a:s", 40, "$m", EDIT_SECRET)]);
|
||||||
|
rig.ok(&mut history);
|
||||||
|
nothing_secret_stored(&rig);
|
||||||
|
assert!(rig.points().is_empty());
|
||||||
|
// Across a rebuild: a lost cursor re-reads the whole history.
|
||||||
|
let mut cursors = rig.index.cursors();
|
||||||
|
cursors.remove(&cursor_key("!r:s"));
|
||||||
|
rig.index.set_cursors(&cursors).unwrap();
|
||||||
|
rig.ok(&mut history);
|
||||||
|
nothing_secret_stored(&rig);
|
||||||
|
assert!(rig.points().is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_stranger_cannot_edit_a_message_of_the_same_page() {
|
||||||
|
let mut history = FakeHistory::default();
|
||||||
|
history.room("!r:s", vec![said("$m", 10, "mine"), edit("$e", "@stranger:s", 20, "$m", "hijacked")]);
|
||||||
|
let mut rig = Rig::new();
|
||||||
|
rig.ok(&mut history);
|
||||||
|
assert_eq!(rig.bodies(), ["mine"]);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn the_room_name_is_scrubbed_before_it_is_embedded_or_stored() {
|
||||||
|
let mut history = FakeHistory::default();
|
||||||
|
let token = format!("ghp_{}", "k2L9".repeat(9));
|
||||||
|
history.room("!r:s", vec![said("$m", 10, "hello")]).facts("!r:s", |facts| facts.name = Some(format!("Swarm {token}")));
|
||||||
|
let mut rig = Rig::new();
|
||||||
|
let (done, journal) = rig.run(&mut history);
|
||||||
|
let done = done.unwrap();
|
||||||
|
assert!(rig.embedded().iter().all(|text| !text.contains("k2L9k2L9")));
|
||||||
|
assert!(rig.points().iter().all(|point| !point.payload.text.contains("k2L9k2L9") && point.payload.text.starts_with("Swarm <redacted>")));
|
||||||
|
assert!(done.scrubbed.iter().any(|hit| hit.event_id == "(name)") && journal.iter().all(|line| !line.contains("k2L9k2L9")));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_token_split_by_zero_width_characters_leaves_no_tail_in_a_stored_body() {
|
||||||
|
let mut history = FakeHistory::default();
|
||||||
|
history.room("!r:s", vec![said("$m", 10, &format!("key sk-{}\u{200b}{} done", "q7W3".repeat(4), "q7W3".repeat(4)))]);
|
||||||
|
let mut rig = Rig::new();
|
||||||
|
rig.ok(&mut history);
|
||||||
|
assert_eq!(rig.bodies(), ["key <redacted> done"]);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn retire_goes_on_past_a_failed_delete_and_names_each_failure() {
|
||||||
|
let mut rig = Rig::new();
|
||||||
|
for room in ["!a:s", "!b:s", "!c:s"] {
|
||||||
|
rig.seed(room, "$e", 1);
|
||||||
|
}
|
||||||
|
rig.index.state.lock().unwrap().fail_chat_deletes_for = ["!a:s".to_owned()].into();
|
||||||
|
let done = retire_chat(&mut rig.index, &BTreeSet::new()).unwrap();
|
||||||
|
assert_eq!(done.retired, ["!b:s", "!c:s"]);
|
||||||
|
assert_eq!(done.failed.len(), 1);
|
||||||
|
assert_eq!(done.failed[0].0, "!a:s");
|
||||||
|
assert_eq!(rig.held_ids("!a:s"), ["$e"]);
|
||||||
|
assert!(rig.cursor("!a:s").is_some(), "the cursor stays so the next run finds it again");
|
||||||
|
// The refresh reports it too.
|
||||||
|
let mut history = FakeHistory::default();
|
||||||
|
history.room("!z:s", vec![said("$z", 1, "x")]);
|
||||||
|
let done = rig.ok(&mut history);
|
||||||
|
assert!(done.failed.contains(&"!a:s".to_owned()) && !done.is_clean());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_facts_error_on_a_joined_room_keeps_its_points_and_cursor_and_is_reported() {
|
||||||
|
let mut history = FakeHistory::default();
|
||||||
|
history.room("!r:s", vec![said("$a", 1, "x")]);
|
||||||
|
let mut rig = Rig::new();
|
||||||
|
rig.ok(&mut history);
|
||||||
|
let (cursor, points) = (rig.cursor("!r:s"), rig.points());
|
||||||
|
history.facts_fail.insert("!r:s".into());
|
||||||
|
let (done, journal) = rig.run(&mut history);
|
||||||
|
let done = done.unwrap();
|
||||||
|
assert_eq!((rig.cursor("!r:s"), rig.points()), (cursor, points));
|
||||||
|
assert_eq!(done.failed, ["!r:s"]);
|
||||||
|
assert!(done.rooms.iter().any(|report| report.room_id == "!r:s" && report.failure.is_some()));
|
||||||
|
assert!(journal.iter().any(|line| line.contains("!r:s") && line.contains("failed")));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn the_tail_on_equal_ends_is_the_greatest_chunk_id() {
|
||||||
|
let mut rig = Rig::new();
|
||||||
|
let (a, b) = (chunk_id("!r:s", "$a"), chunk_id("!r:s", "$b"));
|
||||||
|
rig.index.upsert_chat(&[
|
||||||
|
ChatPoint { id: a.clone(), vector: vec![1.0; 4], payload: chunk_payload_of("!r:s", &[("$a", 5)]) },
|
||||||
|
ChatPoint { id: b.clone(), vector: vec![1.0; 4], payload: chunk_payload_of("!r:s", &[("$b", 5)]) },
|
||||||
|
])
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(rig.index.chat_tail("!r:s").unwrap().unwrap().id, a.max(b));
|
||||||
|
}
|
||||||
|
|
@ -180,7 +180,7 @@ impl HttpForge {
|
||||||
capped.push(repo.clone());
|
capped.push(repo.clone());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
events.sort_by(|a, b| b.id.cmp(&a.id));
|
events.sort_by_key(|event| std::cmp::Reverse(event.id));
|
||||||
// The cursor never moves backwards: a read that saw nothing new keeps the one it was given.
|
// The cursor never moves backwards: a read that saw nothing new keeps the one it was given.
|
||||||
head = head.max(after.unwrap_or(0));
|
head = head.max(after.unwrap_or(0));
|
||||||
let (mut open_prs, mut unseen) = (Vec::new(), dark.clone());
|
let (mut open_prs, mut unseen) = (Vec::new(), dark.clone());
|
||||||
|
|
@ -251,7 +251,7 @@ fn redirect_target(base: &str, location: &str) -> Option<OrgRepo> {
|
||||||
let at = segments.windows(3).position(|window| window == ["api", "v1", "repos"])?;
|
let at = segments.windows(3).position(|window| window == ["api", "v1", "repos"])?;
|
||||||
let plain = |segment: &&str| !segment.is_empty() && *segment != "." && *segment != ".." && segment.chars().all(|c| c.is_ascii_alphanumeric() || matches!(c, '.' | '_' | '-'));
|
let plain = |segment: &&str| !segment.is_empty() && *segment != "." && *segment != ".." && segment.chars().all(|c| c.is_ascii_alphanumeric() || matches!(c, '.' | '_' | '-'));
|
||||||
let [owner, name, "activities", "feeds"] = segments[at + 3..] else { return None };
|
let [owner, name, "activities", "feeds"] = segments[at + 3..] else { return None };
|
||||||
if !segments[..at].iter().all(|segment| plain(segment)) || !plain(&owner) || !plain(&name) {
|
if !segments[..at].iter().all(plain) || !plain(&owner) || !plain(&name) {
|
||||||
return None;
|
return None;
|
||||||
}
|
}
|
||||||
let repo = OrgRepo(format!("{owner}/{name}"));
|
let repo = OrgRepo(format!("{owner}/{name}"));
|
||||||
|
|
@ -322,6 +322,7 @@ mod tests {
|
||||||
use serde_json::json;
|
use serde_json::json;
|
||||||
|
|
||||||
use super::*;
|
use super::*;
|
||||||
|
use crate::sources::test_http::{Reply, forge_of};
|
||||||
|
|
||||||
/// Only public, live, original, non-empty repos are the corpus; a repo the
|
/// Only public, live, original, non-empty repos are the corpus; a repo the
|
||||||
/// forge does not say is safe is not indexed.
|
/// forge does not say is safe is not indexed.
|
||||||
|
|
@ -400,60 +401,6 @@ mod tests {
|
||||||
assert!(HttpForge::new(&unstated).catalog().unwrap().shortfall.unwrap().contains("X-Total-Count"));
|
assert!(HttpForge::new(&unstated).catalog().unwrap().shortfall.unwrap().contains("X-Total-Count"));
|
||||||
}
|
}
|
||||||
|
|
||||||
/// What a fake forge answers: a status, the `Location` header when there is one, and a body.
|
|
||||||
struct Reply {
|
|
||||||
status: u16,
|
|
||||||
location: Option<String>,
|
|
||||||
body: String,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl From<String> for Reply {
|
|
||||||
fn from(body: String) -> Self {
|
|
||||||
Self { status: 200, location: None, body }
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Reply {
|
|
||||||
/// What the live forge answers an anonymous reader for a private repo.
|
|
||||||
fn not_found() -> Self {
|
|
||||||
Self { status: 404, location: None, body: json!({ "message": "The target couldn't be found." }).to_string() }
|
|
||||||
}
|
|
||||||
|
|
||||||
/// What the live forge answers for a renamed repo: a 301 to its public address, `/forge` prefix and all.
|
|
||||||
fn moved_to(location: &str) -> Self {
|
|
||||||
Self { status: 301, location: Some(location.to_owned()), body: String::new() }
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Answers each request by its path from `routes`; a path it has no answer for is held open and never answered, as a hung forge
|
|
||||||
/// does. Every path asked is recorded in order.
|
|
||||||
fn forge_of<R: Into<Reply>>(routes: impl Fn(&str) -> Option<R> + Send + 'static) -> (String, std::sync::Arc<std::sync::Mutex<Vec<String>>>) {
|
|
||||||
use std::io::{Read, Write};
|
|
||||||
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
|
|
||||||
let base = format!("http://{}", listener.local_addr().unwrap());
|
|
||||||
let asked = std::sync::Arc::new(std::sync::Mutex::new(Vec::new()));
|
|
||||||
let record = std::sync::Arc::clone(&asked);
|
|
||||||
std::thread::spawn(move || {
|
|
||||||
let mut held = Vec::new();
|
|
||||||
for stream in listener.incoming() {
|
|
||||||
let mut stream = stream.unwrap();
|
|
||||||
let mut buffer = [0_u8; 4096];
|
|
||||||
let read = stream.read(&mut buffer).unwrap();
|
|
||||||
let request = String::from_utf8_lossy(&buffer[..read]).into_owned();
|
|
||||||
let path = request.split_whitespace().nth(1).unwrap_or_default().to_owned();
|
|
||||||
record.lock().unwrap().push(path.clone());
|
|
||||||
match routes(&path).map(Into::into) {
|
|
||||||
Some(Reply { status, location, body }) => {
|
|
||||||
let location = location.map(|location| format!("location: {location}\r\n")).unwrap_or_default();
|
|
||||||
write!(stream, "HTTP/1.1 {status} X\r\ncontent-type: application/json\r\n{location}content-length: {}\r\nconnection: close\r\n\r\n{body}", body.len()).unwrap();
|
|
||||||
}
|
|
||||||
None => held.push(stream),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
});
|
|
||||||
(base, asked)
|
|
||||||
}
|
|
||||||
|
|
||||||
const FEED_SAMPLE: &str = include_str!("../../tests/fixtures/feed.json");
|
const FEED_SAMPLE: &str = include_str!("../../tests/fixtures/feed.json");
|
||||||
|
|
||||||
const NOW: &str = "2026-11-03T12:00:00Z";
|
const NOW: &str = "2026-11-03T12:00:00Z";
|
||||||
|
|
@ -523,7 +470,7 @@ mod tests {
|
||||||
assert!(read.open_prs.is_empty() && read.unseen.is_empty() && read.capped.is_empty() && read.moved.is_empty(), "a cursor asks for neither list");
|
assert!(read.open_prs.is_empty() && read.unseen.is_empty() && read.capped.is_empty() && read.moved.is_empty(), "a cursor asks for neither list");
|
||||||
let wire = serde_json::to_string(&read).unwrap();
|
let wire = serde_json::to_string(&read).unwrap();
|
||||||
assert!(!wire.contains("CANARY") && !wire.contains("obey") && !wire.contains("Message"), "no forge text leaves the read: {wire}");
|
assert!(!wire.contains("CANARY") && !wire.contains("obey") && !wire.contains("Message"), "no forge text leaves the read: {wire}");
|
||||||
let asked = asked.lock().unwrap().clone();
|
let asked = asked.paths();
|
||||||
assert!(!asked.is_empty() && asked.iter().all(|path| names.iter().any(|name| path.starts_with(&format!("/api/v1/repos/{name}/activities/feeds")))), "{asked:?}");
|
assert!(!asked.is_empty() && asked.iter().all(|path| names.iter().any(|name| path.starts_with(&format!("/api/v1/repos/{name}/activities/feeds")))), "{asked:?}");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -555,13 +502,13 @@ mod tests {
|
||||||
let far = HttpForge::new(&base).activity(&wanted, &me, Some(5), now).unwrap();
|
let far = HttpForge::new(&base).activity(&wanted, &me, Some(5), now).unwrap();
|
||||||
assert!(far.capped == repos(&["GameCult/Huginn"]), "stopped at the cap with the cursor not reached");
|
assert!(far.capped == repos(&["GameCult/Huginn"]), "stopped at the cap with the cursor not reached");
|
||||||
assert_eq!((far.events.len(), far.head), (200, 2500));
|
assert_eq!((far.events.len(), far.head), (200, 2500));
|
||||||
assert_eq!(asked.lock().unwrap().len(), 4, "four pages and no more");
|
assert_eq!(asked.paths().len(), 4, "four pages and no more");
|
||||||
|
|
||||||
let (base, asked) = forge_of(feed("nibu"));
|
let (base, asked) = forge_of(feed("nibu"));
|
||||||
let near = HttpForge::new(&base).activity(&wanted, &me, Some(2300), now).unwrap();
|
let near = HttpForge::new(&base).activity(&wanted, &me, Some(2300), now).unwrap();
|
||||||
assert!(near.capped.is_empty());
|
assert!(near.capped.is_empty());
|
||||||
assert_eq!((near.events.len(), near.head), (20, 2500), "ids above the cursor only");
|
assert_eq!((near.events.len(), near.head), (20, 2500), "ids above the cursor only");
|
||||||
assert_eq!(asked.lock().unwrap().len(), 1);
|
assert_eq!(asked.paths().len(), 1);
|
||||||
|
|
||||||
let (base, _) = forge_of(feed("eureka"));
|
let (base, _) = forge_of(feed("eureka"));
|
||||||
let hers = HttpForge::new(&base).activity(&wanted, &me, Some(2300), now).unwrap();
|
let hers = HttpForge::new(&base).activity(&wanted, &me, Some(2300), now).unwrap();
|
||||||
|
|
@ -583,7 +530,7 @@ mod tests {
|
||||||
let read = HttpForge::new(&base).activity(&repos(&["GameCult/Ratatoskr"]), &me(), None, at(NOW)).unwrap();
|
let read = HttpForge::new(&base).activity(&repos(&["GameCult/Ratatoskr"]), &me(), None, at(NOW)).unwrap();
|
||||||
assert_eq!(read.events.iter().map(|event| event.id).collect::<Vec<_>>(), [300]);
|
assert_eq!(read.events.iter().map(|event| event.id).collect::<Vec<_>>(), [300]);
|
||||||
assert!(read.capped.is_empty());
|
assert!(read.capped.is_empty());
|
||||||
assert!(asked.lock().unwrap().iter().all(|path| !path.contains("/orgs/GameCult/activities")));
|
assert!(asked.paths().iter().all(|path| !path.contains("/orgs/GameCult/activities")));
|
||||||
}
|
}
|
||||||
|
|
||||||
/// A first read keeps what is inside 72 hours and nothing older, and its cursor is the newest id it saw, an entry past the window
|
/// A first read keeps what is inside 72 hours and nothing older, and its cursor is the newest id it saw, an entry past the window
|
||||||
|
|
@ -619,13 +566,13 @@ mod tests {
|
||||||
let (base, asked) = forge_of(move |path| if is_feed(path) { repo_feed(&entries, path) } else { Some("[]".into()) });
|
let (base, asked) = forge_of(move |path| if is_feed(path) { repo_feed(&entries, path) } else { Some("[]".into()) });
|
||||||
let read = HttpForge::new(&base).activity(&wanted, &me(), None, at(NOW)).unwrap();
|
let read = HttpForge::new(&base).activity(&wanted, &me(), None, at(NOW)).unwrap();
|
||||||
assert_eq!((read.events.len(), read.capped.len()), (53, 0));
|
assert_eq!((read.events.len(), read.capped.len()), (53, 0));
|
||||||
assert_eq!(asked.lock().unwrap().iter().filter(|path| is_feed(path)).count(), 2, "page 2 was asked, page 3 was not");
|
assert_eq!(asked.paths().iter().filter(|path| is_feed(path)).count(), 2, "page 2 was asked, page 3 was not");
|
||||||
|
|
||||||
let entries: Vec<Value> = (0..400).map(|n| push_at(5000 - n, "GameCult/Huginn", "Nibu", 5)).collect();
|
let entries: Vec<Value> = (0..400).map(|n| push_at(5000 - n, "GameCult/Huginn", "Nibu", 5)).collect();
|
||||||
let (base, asked) = forge_of(move |path| if is_feed(path) { repo_feed(&entries, path) } else { Some("[]".into()) });
|
let (base, asked) = forge_of(move |path| if is_feed(path) { repo_feed(&entries, path) } else { Some("[]".into()) });
|
||||||
let read = HttpForge::new(&base).activity(&wanted, &me(), None, at(NOW)).unwrap();
|
let read = HttpForge::new(&base).activity(&wanted, &me(), None, at(NOW)).unwrap();
|
||||||
assert_eq!((read.events.len(), read.capped), (REPO_FEED_PAGES * PAGE, repos(&["GameCult/Huginn"])), "200 read and the rest disclosed");
|
assert_eq!((read.events.len(), read.capped), (REPO_FEED_PAGES * PAGE, repos(&["GameCult/Huginn"])), "200 read and the rest disclosed");
|
||||||
assert_eq!(asked.lock().unwrap().iter().filter(|path| is_feed(path)).count(), REPO_FEED_PAGES);
|
assert_eq!(asked.paths().iter().filter(|path| is_feed(path)).count(), REPO_FEED_PAGES);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// With no cursor the read also lists the open PRs in her repos and hers anywhere, and the repos the forge's listing does not hold.
|
/// With no cursor the read also lists the open PRs in her repos and hers anywhere, and the repos the forge's listing does not hold.
|
||||||
|
|
@ -636,9 +583,9 @@ mod tests {
|
||||||
let mut first_page: Vec<Value> = (0..PAGE as u32).map(|n| pr(n + 100, "kiko", "GameCult/Elsewhere")).collect();
|
let mut first_page: Vec<Value> = (0..PAGE as u32).map(|n| pr(n + 100, "kiko", "GameCult/Elsewhere")).collect();
|
||||||
first_page[0] = pr(8, "nibu", "GameCult/Ratatoskr");
|
first_page[0] = pr(8, "nibu", "GameCult/Ratatoskr");
|
||||||
first_page[1] = pr(42, "Eureka", "GameCult/Eureka");
|
first_page[1] = pr(42, "Eureka", "GameCult/Eureka");
|
||||||
let search = vec![Value::Array(first_page).to_string(), Value::Array(vec![pr(9, "void", "GameCult/Huginn")]).to_string()];
|
let search = [Value::Array(first_page).to_string(), Value::Array(vec![pr(9, "void", "GameCult/Huginn")]).to_string()];
|
||||||
let names: Vec<Value> = (0..61).map(|n| json!({ "full_name": match n { 0 => "GameCult/ratatoskr".to_owned(), 1 => "GameCult/Huginn".to_owned(), n => format!("GameCult/r{n}") } })).collect();
|
let names: Vec<Value> = (0..61).map(|n| json!({ "full_name": match n { 0 => "GameCult/ratatoskr".to_owned(), 1 => "GameCult/Huginn".to_owned(), n => format!("GameCult/r{n}") } })).collect();
|
||||||
let listing = vec![Value::Array(names[..PAGE].to_vec()).to_string(), Value::Array(names[PAGE..].to_vec()).to_string()];
|
let listing = [Value::Array(names[..PAGE].to_vec()).to_string(), Value::Array(names[PAGE..].to_vec()).to_string()];
|
||||||
let sample = sample();
|
let sample = sample();
|
||||||
let (base, asked) = forge_of(move |path| {
|
let (base, asked) = forge_of(move |path| {
|
||||||
if is_feed(path) && path.contains("/GameCult/Ghost/") {
|
if is_feed(path) && path.contains("/GameCult/Ghost/") {
|
||||||
|
|
@ -660,14 +607,14 @@ mod tests {
|
||||||
assert_eq!(first.unseen, repos(&["GameCult/Ghost"]), "r60 is on page 2 of the listing");
|
assert_eq!(first.unseen, repos(&["GameCult/Ghost"]), "r60 is on page 2 of the listing");
|
||||||
let window = at("2026-10-31T12:00:00Z").timestamp_millis() as u64;
|
let window = at("2026-10-31T12:00:00Z").timestamp_millis() as u64;
|
||||||
assert!(!first.events.is_empty() && first.events.iter().all(|event| event.at >= window), "inside 72 h: {:?}", first.events);
|
assert!(!first.events.is_empty() && first.events.iter().all(|event| event.at >= window), "inside 72 h: {:?}", first.events);
|
||||||
let paths = asked.lock().unwrap().clone();
|
let paths = asked.paths();
|
||||||
let count = |what: &str| paths.iter().filter(|path| path.contains(what)).count();
|
let count = |what: &str| paths.iter().filter(|path| path.contains(what)).count();
|
||||||
assert_eq!((count("/issues/search"), count("/orgs/GameCult/repos")), (2, 2), "a short page ends each: {paths:?}");
|
assert_eq!((count("/issues/search"), count("/orgs/GameCult/repos")), (2, 2), "a short page ends each: {paths:?}");
|
||||||
|
|
||||||
// A cursor asks for neither list.
|
// A cursor asks for neither list.
|
||||||
let before = paths.len();
|
let before = paths.len();
|
||||||
HttpForge::new(&base).activity(&wanted, &me(), Some(1), at(NOW)).unwrap();
|
HttpForge::new(&base).activity(&wanted, &me(), Some(1), at(NOW)).unwrap();
|
||||||
let after = asked.lock().unwrap().clone();
|
let after = asked.paths();
|
||||||
assert!(after[before..].iter().all(|path| is_feed(path)), "only the feeds: {after:?}");
|
assert!(after[before..].iter().all(|path| is_feed(path)), "only the feeds: {after:?}");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -679,7 +626,7 @@ mod tests {
|
||||||
let full = |from: u32| Value::Array((0..PAGE as u32).map(|n| pr(from + n, "GameCult/Elsewhere")).collect()).to_string();
|
let full = |from: u32| Value::Array((0..PAGE as u32).map(|n| pr(from + n, "GameCult/Elsewhere")).collect()).to_string();
|
||||||
let mut third: Vec<Value> = (0..PAGE as u32).map(|n| pr(300 + n, "GameCult/Elsewhere")).collect();
|
let mut third: Vec<Value> = (0..PAGE as u32).map(|n| pr(300 + n, "GameCult/Elsewhere")).collect();
|
||||||
third[PAGE - 1] = pr(7, "GameCult/Huginn");
|
third[PAGE - 1] = pr(7, "GameCult/Huginn");
|
||||||
let search = vec![full(100), full(200), Value::Array(third).to_string(), "[]".to_owned()];
|
let search = [full(100), full(200), Value::Array(third).to_string(), "[]".to_owned()];
|
||||||
let (base, asked) = forge_of(move |path| {
|
let (base, asked) = forge_of(move |path| {
|
||||||
if is_feed(path) {
|
if is_feed(path) {
|
||||||
Some("[]".to_owned())
|
Some("[]".to_owned())
|
||||||
|
|
@ -693,12 +640,12 @@ mod tests {
|
||||||
});
|
});
|
||||||
let read = HttpForge::new(&base).activity(&repos(&["GameCult/Huginn"]), &me(), None, at(NOW)).unwrap();
|
let read = HttpForge::new(&base).activity(&repos(&["GameCult/Huginn"]), &me(), None, at(NOW)).unwrap();
|
||||||
assert_eq!(read.open_prs.iter().map(|pr| pr.number).collect::<Vec<_>>(), [7], "page 3 was read");
|
assert_eq!(read.open_prs.iter().map(|pr| pr.number).collect::<Vec<_>>(), [7], "page 3 was read");
|
||||||
assert_eq!(asked.lock().unwrap().iter().filter(|path| path.contains("/issues/search")).count(), 4, "to the short page");
|
assert_eq!(asked.paths().iter().filter(|path| path.contains("/issues/search")).count(), 4, "to the short page");
|
||||||
}
|
}
|
||||||
|
|
||||||
/// The forge as it answers an anonymous reader: public repos give their feed (`[]` when quiet), a private repo answers 404, a
|
/// The forge as it answers an anonymous reader: public repos give their feed (`[]` when quiet), a private repo answers 404, a
|
||||||
/// renamed repo answers 301 to a path under `/forge` that this loopback does not serve (asking it gives 404).
|
/// renamed repo answers 301 to a path under `/forge` that this loopback does not serve (asking it gives 404).
|
||||||
fn live_forge(entries: Vec<Value>, public: &'static [&'static str], listing: String) -> (String, std::sync::Arc<std::sync::Mutex<Vec<String>>>) {
|
fn live_forge(entries: Vec<Value>, public: &'static [&'static str], listing: String) -> (String, crate::sources::test_http::Asked) {
|
||||||
forge_of(move |path| {
|
forge_of(move |path| {
|
||||||
if path.starts_with("/forge/") {
|
if path.starts_with("/forge/") {
|
||||||
return Some(Reply::not_found());
|
return Some(Reply::not_found());
|
||||||
|
|
@ -733,7 +680,7 @@ mod tests {
|
||||||
assert_eq!(first.unseen, repos(&["GameCult/gamecult-ops", "GameCult/cinny"]));
|
assert_eq!(first.unseen, repos(&["GameCult/gamecult-ops", "GameCult/cinny"]));
|
||||||
assert_eq!(first.moved, [RepoMoved { from: "GameCult/cinny".into(), to: "GameCult/Ratatoskr".into() }]);
|
assert_eq!(first.moved, [RepoMoved { from: "GameCult/cinny".into(), to: "GameCult/Ratatoskr".into() }]);
|
||||||
assert!(first.capped.is_empty());
|
assert!(first.capped.is_empty());
|
||||||
assert!(asked.lock().unwrap().iter().all(|path| !path.starts_with("/forge/")), "a redirect is not followed: {:?}", asked.lock().unwrap());
|
assert!(asked.paths().iter().all(|path| !path.starts_with("/forge/")), "a redirect is not followed: {:?}", asked.paths());
|
||||||
|
|
||||||
let later = forge.activity(&wanted, &me(), Some(250), at(NOW)).unwrap();
|
let later = forge.activity(&wanted, &me(), Some(250), at(NOW)).unwrap();
|
||||||
assert_eq!((later.events.iter().map(|event| event.id).collect::<Vec<_>>(), later.head), (vec![300], 300), "a later read goes on past the dark repos");
|
assert_eq!((later.events.iter().map(|event| event.id).collect::<Vec<_>>(), later.head), (vec![300], 300), "a later read goes on past the dark repos");
|
||||||
|
|
@ -799,9 +746,7 @@ mod tests {
|
||||||
let (base, _) = forge_of(move |path| {
|
let (base, _) = forge_of(move |path| {
|
||||||
if is_feed(path) && path.contains("/GameCult/cinny/") {
|
if is_feed(path) && path.contains("/GameCult/cinny/") {
|
||||||
Some(dark())
|
Some(dark())
|
||||||
} else if is_feed(path) {
|
} else if is_feed(path) || path.contains("/issues/search") {
|
||||||
Some("[]".to_owned().into())
|
|
||||||
} else if path.contains("/issues/search") {
|
|
||||||
Some("[]".to_owned().into())
|
Some("[]".to_owned().into())
|
||||||
} else {
|
} else {
|
||||||
Some(listing.clone().into())
|
Some(listing.clone().into())
|
||||||
|
|
|
||||||
128
crates/huginn-daemon/src/sources/job.rs
Normal file
128
crates/huginn-daemon/src/sources/job.rs
Normal file
|
|
@ -0,0 +1,128 @@
|
||||||
|
//! What one `huginn-sources` run does with the pieces it is handed: the forge
|
||||||
|
//! refresh, then the chat refresh (or, with chat off, the retirement of every
|
||||||
|
//! chat point), one exit status for both, and the steward-purge request. The
|
||||||
|
//! binary supplies the real forge, embedder, vector store and Matrix client and
|
||||||
|
//! owns the flags and the lock; this file decides nothing about scope or
|
||||||
|
//! scrubbing (`room_scope` and `redact` do) and never sees a credential.
|
||||||
|
|
||||||
|
use std::collections::BTreeSet;
|
||||||
|
use std::path::Path;
|
||||||
|
|
||||||
|
use anyhow::{Context, Result, bail, ensure};
|
||||||
|
use chrono::{DateTime, Utc};
|
||||||
|
|
||||||
|
use super::chat::{Entry, RoomHistory, refresh_chat, retire_chat};
|
||||||
|
use super::chat_purge::{PurgeWindow, parse_windows, purge_key, request_purge};
|
||||||
|
use super::room_scope::Estate;
|
||||||
|
use super::{DescribedSources, Forge, SourceIndex, refresh};
|
||||||
|
use crate::index::{Embedder, INDEX_CALL_TIMEOUT};
|
||||||
|
|
||||||
|
/// Chat on: the reader's rooms and the estate ids the scope refuses.
|
||||||
|
pub struct Chat<'a, H> {
|
||||||
|
pub history: &'a mut H,
|
||||||
|
pub estate: &'a Estate,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One run: the forge refresh, then chat (`Some`) or the retirement of every chat point (`None`).
|
||||||
|
/// `say` takes the journal's report lines (stdout), `complain` the failures, refusals and scrub hits (stderr).
|
||||||
|
/// A forge failure never skips the chat run and a chat failure never hides a forge one.
|
||||||
|
/// Returns the exit status: 0 only when both were clean.
|
||||||
|
pub fn run<E: Embedder, S: SourceIndex, F: Forge, H: RoomHistory>(
|
||||||
|
embedder: &mut E,
|
||||||
|
index: &mut S,
|
||||||
|
forge: &mut F,
|
||||||
|
state_root: &Path,
|
||||||
|
chat: Option<Chat<'_, H>>,
|
||||||
|
say: &mut dyn FnMut(String),
|
||||||
|
complain: &mut dyn FnMut(String),
|
||||||
|
) -> u8 {
|
||||||
|
let mut clean = true;
|
||||||
|
match refresh(embedder, index, forge, state_root, &mut |report| say(report.to_string())) {
|
||||||
|
Ok(refreshed) if refreshed.is_clean() => {}
|
||||||
|
Ok(refreshed) => {
|
||||||
|
clean = false;
|
||||||
|
if !refreshed.failed.is_empty() {
|
||||||
|
complain(format!("huginn-sources: {} repos failed: {}", refreshed.failed.len(), refreshed.failed.join(", ")));
|
||||||
|
}
|
||||||
|
if let Some(why) = &refreshed.retirement_withheld {
|
||||||
|
complain(format!("huginn-sources: no repo was retired: {why}"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(error) => {
|
||||||
|
clean = false;
|
||||||
|
complain(format!("huginn-sources: {error:#}"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
match chat {
|
||||||
|
Some(Chat { history, estate }) => {
|
||||||
|
let mut line = |entry: Entry| if entry.is_alarm() { complain(entry.to_string()) } else { say(entry.to_string()) };
|
||||||
|
match refresh_chat(embedder, index, history, estate, &mut line) {
|
||||||
|
Ok(refreshed) => clean &= refreshed.is_clean(),
|
||||||
|
Err(error) => {
|
||||||
|
clean = false;
|
||||||
|
complain(format!("huginn-sources: chat: {error:#}"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
None => match retire_chat(index, &BTreeSet::new()) {
|
||||||
|
Ok(done) => {
|
||||||
|
if !done.retired.is_empty() {
|
||||||
|
say(format!("chat retired: {} rooms", done.retired.len()));
|
||||||
|
}
|
||||||
|
for (room, why) in done.failed {
|
||||||
|
clean = false;
|
||||||
|
complain(format!("huginn-sources: retiring {room} failed: {why}"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(error) => {
|
||||||
|
clean = false;
|
||||||
|
complain(format!("huginn-sources: chat: {error:#}"));
|
||||||
|
}
|
||||||
|
},
|
||||||
|
}
|
||||||
|
u8::from(!clean)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The longest span one request may name.
|
||||||
|
const PURGE_SPAN_DAYS: i64 = 366;
|
||||||
|
const REQUEST_BYTES_MAX: usize = 120;
|
||||||
|
|
||||||
|
/// A steward's purge request as the admin typed it, checked. Each error names its option.
|
||||||
|
#[derive(Debug, PartialEq, Eq)]
|
||||||
|
pub struct Purge {
|
||||||
|
pub room_id: String,
|
||||||
|
pub from: String,
|
||||||
|
pub to: String,
|
||||||
|
pub request: String,
|
||||||
|
pub window: PurgeWindow,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn instant(option: &str, text: &str) -> Result<DateTime<Utc>> {
|
||||||
|
ensure!(text.ends_with('Z'), "--{option} must be an RFC 3339 UTC time ending in Z, got {text:?}");
|
||||||
|
let at = DateTime::parse_from_rfc3339(text).map_err(|_| anyhow::anyhow!("--{option} must be an RFC 3339 UTC time ending in Z, got {text:?}"))?;
|
||||||
|
Ok(at.with_timezone(&Utc))
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Purge {
|
||||||
|
pub fn check(room: &str, from: &str, to: &str, request: &str, now: DateTime<Utc>) -> Result<Self> {
|
||||||
|
ensure!(room.starts_with('!'), "--room must be a room id beginning with !, got {room:?}: an alias needs Matrix, and the journal prints both #alias (!id)");
|
||||||
|
let (start, end) = (instant("from", from)?, instant("to", to)?);
|
||||||
|
ensure!(start < end, "--from must be before --to");
|
||||||
|
ensure!(end <= now, "--to must not be after now");
|
||||||
|
ensure!(end - start <= chrono::Duration::days(PURGE_SPAN_DAYS), "the window must be at most {PURGE_SPAN_DAYS} days");
|
||||||
|
ensure!(!request.is_empty() && request.len() <= REQUEST_BYTES_MAX, "--request must be 1 to {REQUEST_BYTES_MAX} bytes");
|
||||||
|
let ms = |at: DateTime<Utc>| u64::try_from(at.timestamp_millis()).context("a purge time before 1970");
|
||||||
|
Ok(Self { room_id: room.to_owned(), from: from.to_owned(), to: to.to_owned(), request: request.to_owned(), window: PurgeWindow { from_ms: ms(start)?, to_ms: ms(end)? } })
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Writes the tombstone and returns the line the admin sees; the free-text request is quoted and escaped so it cannot forge a journal line. Touches no point and makes no Matrix call;
|
||||||
|
/// the caller holds the job's lock. The next run applies it.
|
||||||
|
pub fn purge<S: SourceIndex>(index: &mut S, purge: &Purge) -> Result<String> {
|
||||||
|
request_purge(index, &purge.room_id, purge.window)?;
|
||||||
|
let DescribedSources::Labelled { cursors, .. } = index.describe_sources(INDEX_CALL_TIMEOUT).context("describing the source collection")? else {
|
||||||
|
bail!("the source collection is absent or not Huginn's");
|
||||||
|
};
|
||||||
|
let windows = cursors.get(&purge_key(&purge.room_id)).map(|held| parse_windows(held)).transpose()?.map_or(0, |held| held.len());
|
||||||
|
Ok(format!("chat-purge requested: {} {}..{} request={:?} windows={windows}", purge.room_id, purge.from, purge.to, purge.request))
|
||||||
|
}
|
||||||
287
crates/huginn-daemon/src/sources/job_tests.rs
Normal file
287
crates/huginn-daemon/src/sources/job_tests.rs
Normal file
|
|
@ -0,0 +1,287 @@
|
||||||
|
//! One run of the job: forge refresh and chat refresh over a fake Synapse reached through the real client,
|
||||||
|
//! a fake vector store and the fake embedder; the exit status; the chat-off retirement; the purge request.
|
||||||
|
|
||||||
|
use std::collections::BTreeMap;
|
||||||
|
use std::sync::{Arc, Mutex};
|
||||||
|
|
||||||
|
use anyhow::{Result, bail};
|
||||||
|
use chrono::{DateTime, Utc};
|
||||||
|
use serde_json::json;
|
||||||
|
|
||||||
|
use super::chat::{CHAT_PURGE_PREFIX, RoomHistory};
|
||||||
|
use super::job::{Chat, Purge, purge, run};
|
||||||
|
use super::matrix::HttpMatrix;
|
||||||
|
use super::matrix_tests::{Room, Rooms, TOKEN, message, state, synapse};
|
||||||
|
use super::room_scope::Estate;
|
||||||
|
use super::tests::{FakeForge, Remote};
|
||||||
|
use super::test_http::forge_of;
|
||||||
|
use super::*;
|
||||||
|
use crate::index::fakes::{FakeEmbedder, FakeSources};
|
||||||
|
|
||||||
|
const DAY0: u64 = 1_794_268_800_000; // 2026-11-10T00:00:00Z
|
||||||
|
|
||||||
|
fn now() -> DateTime<Utc> {
|
||||||
|
DateTime::parse_from_rfc3339("2026-11-17T00:00:00Z").unwrap().with_timezone(&Utc)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn estate() -> Estate {
|
||||||
|
Estate::new(["!e1:s", "!e2:s", "!e3:s"].map(str::to_owned).into()).unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
|
struct Out {
|
||||||
|
said: Vec<String>,
|
||||||
|
complained: Vec<String>,
|
||||||
|
status: u8,
|
||||||
|
}
|
||||||
|
|
||||||
|
struct Rig {
|
||||||
|
embedder: FakeEmbedder,
|
||||||
|
index: FakeSources,
|
||||||
|
forge: FakeForge,
|
||||||
|
_remote: Remote,
|
||||||
|
root: tempfile::TempDir,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Rig {
|
||||||
|
fn new() -> Self {
|
||||||
|
let remote = Remote::new();
|
||||||
|
remote.commit(&[("README.md", b"# Demo\n")], &[]);
|
||||||
|
Self { embedder: FakeEmbedder::new("d1"), index: FakeSources::default(), forge: FakeForge::of(&[("Demo", &remote)]), _remote: remote, root: tempfile::tempdir().unwrap() }
|
||||||
|
}
|
||||||
|
|
||||||
|
fn go<H: RoomHistory>(&mut self, history: Option<&mut H>) -> Out {
|
||||||
|
let (mut said, mut complained) = (Vec::new(), Vec::new());
|
||||||
|
let estate = estate();
|
||||||
|
let chat = history.map(|history| Chat { history, estate: &estate });
|
||||||
|
let status = run(&mut self.embedder, &mut self.index, &mut self.forge, self.root.path(), chat, &mut |line| said.push(line), &mut |line| complained.push(line));
|
||||||
|
Out { said, complained, status }
|
||||||
|
}
|
||||||
|
|
||||||
|
fn chat_bodies(&self) -> Vec<String> {
|
||||||
|
let mut bodies: Vec<String> = self.index.state.lock().unwrap().chat.values().flat_map(|point| point.payload.messages.iter().map(|m| m.body.clone())).collect();
|
||||||
|
bodies.sort();
|
||||||
|
bodies
|
||||||
|
}
|
||||||
|
|
||||||
|
fn chat_rooms(&self) -> std::collections::BTreeSet<String> {
|
||||||
|
self.index.state.lock().unwrap().chat.values().map(|point| point.payload.room_id.clone()).collect()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn rooms(list: Vec<(&str, Vec<serde_json::Value>, Vec<serde_json::Value>)>) -> Rooms {
|
||||||
|
Arc::new(Mutex::new(list.into_iter().map(|(id, state, timeline)| (id.to_owned(), Room { state, timeline })).collect::<BTreeMap<_, _>>()))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn platform() -> String {
|
||||||
|
["disc", "ord"].concat()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn world() -> Rooms {
|
||||||
|
let relay = format!("@_{}_1:s", platform());
|
||||||
|
rooms(vec![
|
||||||
|
("!swarm:s", vec![state("m.room.canonical_alias", json!({ "alias": "#swarm:gamecult.org" })), state("m.room.join_rules", json!({ "join_rule": "invite" }))], vec![
|
||||||
|
message("$m1", "@a:s", DAY0 - 5000, "first words"),
|
||||||
|
message("$m2", "@b:s", DAY0 - 4000, "to be redacted"),
|
||||||
|
message("$m3", "@a:s", DAY0 + 1000, "inside the window"),
|
||||||
|
]),
|
||||||
|
("!bridged:s", vec![state("m.bridge", json!({}))], vec![message("$b1", "@a:s", DAY0, "bridged talk")]),
|
||||||
|
("!relay:s", vec![], vec![message("$d1", &relay, DAY0, "relayed talk")]),
|
||||||
|
("!aliased:s", vec![state("m.room.canonical_alias", json!({ "alias": "#Users:gamecult.org" }))], vec![message("$u1", "@a:s", DAY0, "estate talk")]),
|
||||||
|
])
|
||||||
|
}
|
||||||
|
|
||||||
|
fn all_lines(out: &Out) -> String {
|
||||||
|
out.said.iter().chain(&out.complained).cloned().collect::<Vec<_>>().join("\n")
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_full_run_over_a_fake_synapse() {
|
||||||
|
let world = world();
|
||||||
|
let (base, asked) = synapse(&world);
|
||||||
|
let mut matrix = HttpMatrix::new(&base, TOKEN);
|
||||||
|
let mut rig = Rig::new();
|
||||||
|
|
||||||
|
let first = rig.go(Some(&mut matrix));
|
||||||
|
assert_eq!(first.status, 1, "joined bridged, relay and estate rooms are refused: {:?}", first.complained);
|
||||||
|
assert_eq!(rig.chat_rooms(), ["!swarm:s".to_owned()].into(), "an invite-only room is ingested; nothing of the refused rooms is");
|
||||||
|
assert_eq!(rig.chat_bodies(), ["first words", "inside the window", "to be redacted"]);
|
||||||
|
for refused in ["!bridged:s", "!relay:s", "!aliased:s"] {
|
||||||
|
assert!(first.complained.iter().any(|line| line.starts_with("refused:") && line.contains(refused)), "{refused}: {:?}", first.complained);
|
||||||
|
}
|
||||||
|
assert!(!rig.embedder.texts().iter().any(|text| text.contains("bridged talk") || text.contains("relayed talk") || text.contains("estate talk")));
|
||||||
|
assert!(first.said.iter().any(|line| line.starts_with("Demo ")), "the forge's line is journaled too");
|
||||||
|
assert!(first.said.iter().any(|line| line.starts_with("#swarm:gamecult.org (!swarm:s)") && line.contains("events=3")), "{:?}", first.said);
|
||||||
|
assert!(asked.seen().iter().all(|seen| seen.method == "GET"));
|
||||||
|
|
||||||
|
// The steward asks for the window; then the room edits, redacts and leaks a key.
|
||||||
|
let request = Purge::check("!swarm:s", "2026-11-10T00:00:00Z", "2026-11-11T00:00:00Z", "follow_up:x", now()).unwrap();
|
||||||
|
assert_eq!(purge(&mut rig.index, &request).unwrap(), "chat-purge requested: !swarm:s 2026-11-10T00:00:00Z..2026-11-11T00:00:00Z request=\"follow_up:x\" windows=1");
|
||||||
|
let before = rig.index.cursors();
|
||||||
|
assert!(purge(&mut rig.index, &request).unwrap().ends_with("windows=1"));
|
||||||
|
assert_eq!(rig.index.cursors(), before, "a repeated request writes nothing");
|
||||||
|
let secret = format!("key ghp_{}", "k2L9".repeat(9));
|
||||||
|
{
|
||||||
|
let mut held = world.lock().unwrap();
|
||||||
|
let room = held.get_mut("!swarm:s").unwrap();
|
||||||
|
room.timeline.push(json!({ "type": "m.room.message", "event_id": "$edit", "sender": "@a:s", "origin_server_ts": DAY0 + 90_000_000, "content": {
|
||||||
|
"msgtype": "m.text", "body": "* first words, amended", "m.new_content": { "msgtype": "m.text", "body": "first words, amended" }, "m.relates_to": { "rel_type": "m.replace", "event_id": "$m1" } } }));
|
||||||
|
room.timeline.push(json!({ "type": "m.room.redaction", "event_id": "$red", "sender": "@b:s", "origin_server_ts": DAY0 + 91_000_000, "content": { "redacts": "$m2" } }));
|
||||||
|
room.timeline.push(message("$m4", "@a:s", DAY0 + 92_000_000, &secret));
|
||||||
|
}
|
||||||
|
let second = rig.go(Some(&mut matrix));
|
||||||
|
assert_eq!(rig.chat_bodies().len(), 2, "{:?}", rig.chat_bodies());
|
||||||
|
let bodies = rig.chat_bodies().join("\n");
|
||||||
|
assert!(bodies.contains("first words, amended") && !bodies.contains("to be redacted") && !bodies.contains("inside the window"), "{bodies}");
|
||||||
|
assert!(!bodies.contains("ghp_k2L9") && !rig.embedder.texts().iter().any(|text| text.contains("ghp_k2L9")), "the key was scrubbed before it was embedded");
|
||||||
|
let text = all_lines(&second);
|
||||||
|
assert!(second.said.iter().any(|line| line.contains("purged=")), "{text}");
|
||||||
|
assert!(second.complained.iter().any(|line| line == "scrubbed: !swarm:s $m4 key-shape" || line.starts_with("scrubbed: !swarm:s $m4 ")), "{text}");
|
||||||
|
for body in ["first words", "to be redacted", "inside the window", "relayed talk", "ghp_"] {
|
||||||
|
assert!(!text.contains(body), "the journal carries ids and counts, not bodies: {body}");
|
||||||
|
}
|
||||||
|
let third = rig.go(Some(&mut matrix));
|
||||||
|
assert!(!rig.chat_bodies().join("\n").contains("inside the window"), "a purged window does not come back");
|
||||||
|
assert!(third.complained.iter().all(|line| line.starts_with("refused:")), "{:?}", third.complained);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn no_chat_flags_purge_the_corpus_and_make_no_matrix_call() {
|
||||||
|
let world = world();
|
||||||
|
let (base, _) = synapse(&world);
|
||||||
|
let mut matrix = HttpMatrix::new(&base, TOKEN);
|
||||||
|
let mut rig = Rig::new();
|
||||||
|
rig.go(Some(&mut matrix));
|
||||||
|
let request = Purge::check("!swarm:s", "2026-11-10T00:00:00Z", "2026-11-11T00:00:00Z", "r", now()).unwrap();
|
||||||
|
purge(&mut rig.index, &request).unwrap();
|
||||||
|
assert!(!rig.chat_bodies().is_empty());
|
||||||
|
|
||||||
|
let (hung, asked) = forge_of(|_| None::<String>);
|
||||||
|
drop(hung);
|
||||||
|
let out = rig.go::<HttpMatrix>(None);
|
||||||
|
assert_eq!(out.status, 0, "{:?}", out.complained);
|
||||||
|
assert!(rig.chat_bodies().is_empty() && rig.chat_rooms().is_empty());
|
||||||
|
assert!(rig.index.cursors().keys().any(|key| key.starts_with(CHAT_PURGE_PREFIX)), "tombstones stay");
|
||||||
|
assert!(!rig.index.cursors().keys().any(|key| key.starts_with("chat:")), "cursors go");
|
||||||
|
assert!(asked.seen().is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
|
struct BrokenForge;
|
||||||
|
|
||||||
|
impl Forge for BrokenForge {
|
||||||
|
fn catalog(&mut self) -> Result<Catalog> {
|
||||||
|
bail!("the forge is down")
|
||||||
|
}
|
||||||
|
|
||||||
|
fn clone_url(&self, _name: &str) -> String {
|
||||||
|
String::new()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
struct DeadSynapse;
|
||||||
|
|
||||||
|
impl RoomHistory for DeadSynapse {
|
||||||
|
fn joined_rooms(&mut self) -> Result<Vec<String>> {
|
||||||
|
bail!("synapse is down")
|
||||||
|
}
|
||||||
|
|
||||||
|
fn facts(&mut self, _room_id: &str) -> Result<super::room_scope::RoomFacts> {
|
||||||
|
bail!("synapse is down")
|
||||||
|
}
|
||||||
|
|
||||||
|
fn events_after(&mut self, _room_id: &str, _after: Option<&str>, _limit: u32) -> Result<Option<super::chat::EventPage>> {
|
||||||
|
bail!("synapse is down")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn exit_status_is_clean_only_when_forge_and_chat_both_are() {
|
||||||
|
let clean = rooms(vec![("!swarm:s", vec![], vec![message("$m1", "@a:s", 1, "words")])]);
|
||||||
|
let (base, _) = synapse(&clean);
|
||||||
|
let mut rig = Rig::new();
|
||||||
|
let out = rig.go(Some(&mut HttpMatrix::new(&base, TOKEN)));
|
||||||
|
assert_eq!((out.status, out.complained.len()), (0, 0));
|
||||||
|
|
||||||
|
// Scrub hits alone are not a failure; they are reported.
|
||||||
|
clean.lock().unwrap().get_mut("!swarm:s").unwrap().timeline.push(message("$m2", "@a:s", 2, &format!("ghp_{}", "k2L9".repeat(9))));
|
||||||
|
let out = rig.go(Some(&mut HttpMatrix::new(&base, TOKEN)));
|
||||||
|
assert_eq!(out.status, 0, "{:?}", out.complained);
|
||||||
|
assert!(out.complained.iter().any(|line| line.starts_with("scrubbed: !swarm:s $m2")));
|
||||||
|
|
||||||
|
// A chat failure still reports the forge, and a forge failure still runs chat.
|
||||||
|
let out = rig.go(Some(&mut DeadSynapse));
|
||||||
|
assert_eq!(out.status, 1);
|
||||||
|
assert!(out.complained.iter().any(|line| line.contains("synapse is down")));
|
||||||
|
let (mut said, mut complained) = (Vec::new(), Vec::new());
|
||||||
|
let (estate, mut ok) = (estate(), HttpMatrix::new(&base, TOKEN));
|
||||||
|
let status = run(&mut rig.embedder, &mut rig.index, &mut BrokenForge, rig.root.path(), Some(Chat { history: &mut ok, estate: &estate }), &mut |l| said.push(l), &mut |l| complained.push(l));
|
||||||
|
assert_eq!(status, 1);
|
||||||
|
assert!(complained.iter().any(|line| line.contains("the forge is down")), "{complained:?}");
|
||||||
|
assert!(said.iter().any(|line| line.contains("(!swarm:s)")), "chat ran after the forge failed: {said:?}");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_request_cannot_forge_a_journal_line() {
|
||||||
|
let mut rig = Rig::new();
|
||||||
|
rig.go::<HttpMatrix>(None);
|
||||||
|
let request = Purge::check("!swarm:s", "2026-11-10T00:00:00Z", "2026-11-11T00:00:00Z", "x\nrefused: !forged:s", now()).unwrap();
|
||||||
|
let line = purge(&mut rig.index, &request).unwrap();
|
||||||
|
assert_eq!(line.lines().count(), 1, "{line:?}");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_failed_room_report_goes_to_the_complaint_stream_and_fails_the_run() {
|
||||||
|
let mut rig = Rig::new();
|
||||||
|
struct FactsDown;
|
||||||
|
impl RoomHistory for FactsDown {
|
||||||
|
fn joined_rooms(&mut self) -> Result<Vec<String>> {
|
||||||
|
Ok(vec!["!swarm:s".to_owned()])
|
||||||
|
}
|
||||||
|
fn facts(&mut self, _room_id: &str) -> Result<super::room_scope::RoomFacts> {
|
||||||
|
bail!("facts are down")
|
||||||
|
}
|
||||||
|
fn events_after(&mut self, _room_id: &str, _after: Option<&str>, _limit: u32) -> Result<Option<super::chat::EventPage>> {
|
||||||
|
bail!("unreachable")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let out = rig.go(Some(&mut FactsDown));
|
||||||
|
assert_eq!(out.status, 1);
|
||||||
|
assert!(out.complained.iter().any(|line| line.contains("!swarm:s") && line.contains("facts are down")), "{:?}", out.complained);
|
||||||
|
assert!(!out.said.iter().any(|line| line.contains("facts are down")), "{:?}", out.said);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn chat_purge_makes_no_matrix_request() {
|
||||||
|
let world = world();
|
||||||
|
let (base, asked) = synapse(&world);
|
||||||
|
let mut rig = Rig::new();
|
||||||
|
rig.go(Some(&mut HttpMatrix::new(&base, TOKEN)));
|
||||||
|
let made = asked.seen().len();
|
||||||
|
let request = Purge::check("!swarm:s", "2026-11-10T00:00:00Z", "2026-11-11T00:00:00Z", "r", now()).unwrap();
|
||||||
|
purge(&mut rig.index, &request).unwrap();
|
||||||
|
assert_eq!(asked.seen().len(), made);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_retirement_failure_goes_to_the_complaint_stream() {
|
||||||
|
struct Left;
|
||||||
|
impl RoomHistory for Left {
|
||||||
|
fn joined_rooms(&mut self) -> Result<Vec<String>> {
|
||||||
|
Ok(vec![])
|
||||||
|
}
|
||||||
|
fn facts(&mut self, _room_id: &str) -> Result<super::room_scope::RoomFacts> {
|
||||||
|
bail!("unreachable")
|
||||||
|
}
|
||||||
|
fn events_after(&mut self, _room_id: &str, _after: Option<&str>, _limit: u32) -> Result<Option<super::chat::EventPage>> {
|
||||||
|
bail!("unreachable")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let world = rooms(vec![("!swarm:s", vec![], vec![message("$m1", "@a:s", 1, "words")])]);
|
||||||
|
let (base, _) = synapse(&world);
|
||||||
|
let mut rig = Rig::new();
|
||||||
|
rig.go(Some(&mut HttpMatrix::new(&base, TOKEN)));
|
||||||
|
rig.index.state.lock().unwrap().fail_chat_deletes = true;
|
||||||
|
let out = rig.go(Some(&mut Left));
|
||||||
|
assert_eq!(out.status, 1);
|
||||||
|
assert!(out.complained.iter().any(|line| line.starts_with("huginn-sources: retiring !swarm:s failed")), "{:?}", out.complained);
|
||||||
|
assert!(!out.said.iter().any(|line| line.contains("retiring")), "{:?}", out.said);
|
||||||
|
}
|
||||||
159
crates/huginn-daemon/src/sources/matrix.rs
Normal file
159
crates/huginn-daemon/src/sources/matrix.rs
Normal file
|
|
@ -0,0 +1,159 @@
|
||||||
|
//! Synapse's client API as the reader user sees it: Matrix JSON in, `RoomEvent`
|
||||||
|
//! and `RoomFacts` out. JSON for Matrix is spoken here and nowhere else. This
|
||||||
|
//! client reads with GET only and holds no policy: it filters nothing, decides
|
||||||
|
//! no room's or sender's scope, and names no platform. What is in scope is
|
||||||
|
//! `room_scope`'s; the token is read at one place, the `Authorization` header.
|
||||||
|
|
||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
|
use anyhow::{Context, Result, bail};
|
||||||
|
use serde_json::Value;
|
||||||
|
|
||||||
|
use super::chat::{EventKind, EventPage, RoomEvent, RoomHistory};
|
||||||
|
use super::room_scope::RoomFacts;
|
||||||
|
|
||||||
|
const CALL_TIMEOUT: Duration = Duration::from_secs(60);
|
||||||
|
|
||||||
|
/// A bridge's state event types: either marks a room as relaying another platform.
|
||||||
|
const BRIDGE_TYPES: [&str; 2] = ["m.bridge", "uk.half-shot.bridge"];
|
||||||
|
|
||||||
|
pub struct HttpMatrix {
|
||||||
|
agent: ureq::Agent,
|
||||||
|
base: String,
|
||||||
|
token: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl std::fmt::Debug for HttpMatrix {
|
||||||
|
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||||
|
formatter.debug_struct("HttpMatrix").field("base", &self.base).finish_non_exhaustive()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A path segment with everything but unreserved characters escaped.
|
||||||
|
fn segment(raw: &str) -> String {
|
||||||
|
let mut out = String::with_capacity(raw.len());
|
||||||
|
for byte in raw.bytes() {
|
||||||
|
if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_' | b'.' | b'~') {
|
||||||
|
out.push(char::from(byte));
|
||||||
|
} else {
|
||||||
|
out.push_str(&format!("%{byte:02X}"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
out
|
||||||
|
}
|
||||||
|
|
||||||
|
impl HttpMatrix {
|
||||||
|
pub fn new(base_url: &str, token: &str) -> Self {
|
||||||
|
let agent: ureq::Agent =
|
||||||
|
ureq::Agent::config_builder().timeout_global(Some(CALL_TIMEOUT)).http_status_as_error(false).build().into();
|
||||||
|
Self { agent, base: base_url.trim_end_matches('/').to_owned(), token: token.to_owned() }
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One GET of `path` (already escaped): the status and the JSON body. No message made here carries the token.
|
||||||
|
fn get(&self, path: &str) -> Result<(u16, Value)> {
|
||||||
|
let url = format!("{}{path}", self.base);
|
||||||
|
let mut response = self
|
||||||
|
.agent
|
||||||
|
.get(&url)
|
||||||
|
.header("Authorization", format!("Bearer {}", self.token))
|
||||||
|
.call()
|
||||||
|
.with_context(|| format!("no answer from {url}"))?;
|
||||||
|
let status = response.status().as_u16();
|
||||||
|
let body = response.body_mut().read_to_string().with_context(|| format!("reading the answer from {url}"))?;
|
||||||
|
Ok((status, serde_json::from_str(&body).unwrap_or(Value::Null)))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn ok(&self, path: &str) -> Result<Value> {
|
||||||
|
let (status, body) = self.get(path)?;
|
||||||
|
if status != 200 {
|
||||||
|
bail!("synapse answered {status} for {path}");
|
||||||
|
}
|
||||||
|
Ok(body)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn event_of(raw: &Value) -> Result<RoomEvent> {
|
||||||
|
let text = |key: &str| raw[key].as_str().map(str::to_owned).with_context(|| format!("an event has no {key}"));
|
||||||
|
let content = &raw["content"];
|
||||||
|
let content_keys = content.as_object().map(|map| map.keys().cloned().collect()).unwrap_or_default();
|
||||||
|
Ok(RoomEvent { event_id: text("event_id")?, sender: text("sender")?, ts_ms: raw["origin_server_ts"].as_u64().context("an event has no origin_server_ts")?, content_keys, kind: kind_of(raw) })
|
||||||
|
}
|
||||||
|
|
||||||
|
fn kind_of(raw: &Value) -> EventKind {
|
||||||
|
let content = &raw["content"];
|
||||||
|
if !raw["unsigned"]["redacted_because"].is_null() {
|
||||||
|
return EventKind::Other;
|
||||||
|
}
|
||||||
|
match raw["type"].as_str() {
|
||||||
|
Some("m.room.redaction") => match content["redacts"].as_str().or_else(|| raw["redacts"].as_str()) {
|
||||||
|
Some(target) => EventKind::Redaction { target: target.to_owned() },
|
||||||
|
None => EventKind::Other,
|
||||||
|
},
|
||||||
|
Some("m.room.message") if matches!(content["msgtype"].as_str(), Some("m.text" | "m.notice" | "m.emote")) => {
|
||||||
|
let Some(body) = content["body"].as_str() else { return EventKind::Other };
|
||||||
|
let relation = &content["m.relates_to"];
|
||||||
|
if relation["rel_type"].as_str() == Some("m.replace") {
|
||||||
|
return match (relation["event_id"].as_str(), content["m.new_content"]["body"].as_str()) {
|
||||||
|
(Some(target), Some(new)) => EventKind::Edit { target: target.to_owned(), body: new.to_owned() },
|
||||||
|
_ => EventKind::Other,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
EventKind::Message { body: body.to_owned() }
|
||||||
|
}
|
||||||
|
_ => EventKind::Other,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl RoomHistory for HttpMatrix {
|
||||||
|
fn joined_rooms(&mut self) -> Result<Vec<String>> {
|
||||||
|
let body = self.ok("/_matrix/client/v3/joined_rooms")?;
|
||||||
|
let rooms = body["joined_rooms"].as_array().context("the joined rooms answer is not a list")?;
|
||||||
|
Ok(rooms.iter().filter_map(|room| room.as_str().map(str::to_owned)).collect())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn facts(&mut self, room_id: &str) -> Result<RoomFacts> {
|
||||||
|
let body = self.ok(&format!("/_matrix/client/v3/rooms/{}/state", segment(room_id)))?;
|
||||||
|
let state = body.as_array().context("the room state is not a list")?;
|
||||||
|
let mut facts = RoomFacts { room_id: room_id.to_owned(), alias: None, alt_aliases: Vec::new(), name: None, encrypted: false, direct: false, bridge: None };
|
||||||
|
for event in state {
|
||||||
|
let content = &event["content"];
|
||||||
|
match event["type"].as_str() {
|
||||||
|
Some("m.room.canonical_alias") => {
|
||||||
|
facts.alias = content["alias"].as_str().map(str::to_owned);
|
||||||
|
facts.alt_aliases = content["alt_aliases"].as_array().map(|list| list.iter().filter_map(|alias| alias.as_str().map(str::to_owned)).collect()).unwrap_or_default();
|
||||||
|
}
|
||||||
|
Some("m.room.name") => facts.name = content["name"].as_str().map(str::to_owned),
|
||||||
|
Some("m.room.encryption") => facts.encrypted = true,
|
||||||
|
Some("m.room.member") if content["is_direct"].as_bool() == Some(true) => facts.direct = true,
|
||||||
|
Some(kind) if BRIDGE_TYPES.contains(&kind) => {
|
||||||
|
facts.bridge.get_or_insert_with(|| kind.to_owned());
|
||||||
|
}
|
||||||
|
_ => {}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(facts)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn events_after(&mut self, room_id: &str, after: Option<&str>, limit: u32) -> Result<Option<EventPage>> {
|
||||||
|
let room = segment(room_id);
|
||||||
|
let from = match after {
|
||||||
|
None => String::new(),
|
||||||
|
Some(event) => {
|
||||||
|
let (status, body) = self.get(&format!("/_matrix/client/v3/rooms/{room}/context/{}?limit=0", segment(event)))?;
|
||||||
|
match status {
|
||||||
|
200 => match body["end"].as_str() {
|
||||||
|
Some(end) => format!("&from={}", segment(end)),
|
||||||
|
None => bail!("synapse gave no end token for the context of {event} in {room_id}"),
|
||||||
|
},
|
||||||
|
404 => return Ok(None),
|
||||||
|
other => bail!("synapse answered {other} for the context of {event} in {room_id}"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let body = self.ok(&format!("/_matrix/client/v3/rooms/{room}/messages?dir=f&limit={limit}{from}"))?;
|
||||||
|
let chunk = body["chunk"].as_array().context("the messages answer has no chunk")?;
|
||||||
|
let events = chunk.iter().map(event_of).collect::<Result<Vec<_>>>()?;
|
||||||
|
let more = body["end"].as_str().is_some() && !events.is_empty();
|
||||||
|
Ok(Some(EventPage { events, more }))
|
||||||
|
}
|
||||||
|
}
|
||||||
207
crates/huginn-daemon/src/sources/matrix_tests.rs
Normal file
207
crates/huginn-daemon/src/sources/matrix_tests.rs
Normal file
|
|
@ -0,0 +1,207 @@
|
||||||
|
//! The Matrix client against a fake Synapse that speaks the client API's JSON.
|
||||||
|
|
||||||
|
use std::collections::BTreeMap;
|
||||||
|
use std::sync::{Arc, Mutex};
|
||||||
|
|
||||||
|
use serde_json::{Value, json};
|
||||||
|
|
||||||
|
use super::chat::{EventKind, RoomHistory};
|
||||||
|
use super::matrix::HttpMatrix;
|
||||||
|
use super::test_http::{Asked, Reply, forge_of};
|
||||||
|
|
||||||
|
pub(super) const TOKEN: &str = "syt_reader_9f8e7d6c5b4a";
|
||||||
|
|
||||||
|
/// A room as the fake server holds it: its state events and its timeline.
|
||||||
|
pub(super) struct Room {
|
||||||
|
pub state: Vec<Value>,
|
||||||
|
pub timeline: Vec<Value>,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) type Rooms = Arc<Mutex<BTreeMap<String, Room>>>;
|
||||||
|
|
||||||
|
pub(super) fn decode(raw: &str) -> String {
|
||||||
|
let bytes = raw.as_bytes();
|
||||||
|
let mut out = Vec::new();
|
||||||
|
let mut at = 0;
|
||||||
|
while at < bytes.len() {
|
||||||
|
if bytes[at] == b'%' {
|
||||||
|
out.push(u8::from_str_radix(&raw[at + 1..at + 3], 16).unwrap());
|
||||||
|
at += 3;
|
||||||
|
} else {
|
||||||
|
out.push(bytes[at]);
|
||||||
|
at += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
String::from_utf8(out).unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn message(id: &str, sender: &str, ts: u64, body: &str) -> Value {
|
||||||
|
json!({ "type": "m.room.message", "event_id": id, "sender": sender, "origin_server_ts": ts, "content": { "msgtype": "m.text", "body": body } })
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn state(kind: &str, content: Value) -> Value {
|
||||||
|
json!({ "type": kind, "state_key": "", "content": content })
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A Synapse over `rooms`, joined to the rooms in the map: GET only, bearer required.
|
||||||
|
pub(super) fn synapse(rooms: &Rooms) -> (String, Asked) {
|
||||||
|
let rooms = Arc::clone(rooms);
|
||||||
|
forge_of(move |raw| {
|
||||||
|
let (path, query) = raw.split_once('?').unwrap_or((raw, ""));
|
||||||
|
let path = decode(path);
|
||||||
|
let rooms = rooms.lock().unwrap();
|
||||||
|
let rest = path.strip_prefix("/_matrix/client/v3/")?;
|
||||||
|
if rest == "joined_rooms" {
|
||||||
|
return Some(Reply::from(json!({ "joined_rooms": rooms.keys().collect::<Vec<_>>() }).to_string()));
|
||||||
|
}
|
||||||
|
let rest = rest.strip_prefix("rooms/")?;
|
||||||
|
let (id, rest) = rest.split_once('/')?;
|
||||||
|
let Some(room) = rooms.get(id) else { return Some(Reply::status(404, r#"{"errcode":"M_NOT_FOUND"}"#)) };
|
||||||
|
let query: BTreeMap<String, String> = query.split('&').filter_map(|pair| pair.split_once('=')).map(|(k, v)| (k.to_owned(), decode(v))).collect();
|
||||||
|
if rest == "state" {
|
||||||
|
return Some(Reply::from(Value::Array(room.state.clone()).to_string()));
|
||||||
|
}
|
||||||
|
if let Some(event) = rest.strip_prefix("context/") {
|
||||||
|
return Some(match room.timeline.iter().any(|held| held["event_id"] == event) {
|
||||||
|
true => Reply::from(json!({ "end": format!("t_{event}"), "events_before": [], "events_after": [] }).to_string()),
|
||||||
|
false => Reply::status(404, r#"{"errcode":"M_NOT_FOUND"}"#),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if rest == "messages" {
|
||||||
|
let start = match query.get("from") {
|
||||||
|
Some(from) => 1 + room.timeline.iter().position(|held| format!("t_{}", held["event_id"].as_str().unwrap()) == *from)?,
|
||||||
|
None => 0,
|
||||||
|
};
|
||||||
|
let limit: usize = query["limit"].parse().unwrap();
|
||||||
|
let chunk: Vec<&Value> = room.timeline.iter().skip(start).take(limit).collect();
|
||||||
|
let end = chunk.last().map(|last| format!("t_{}", last["event_id"].as_str().unwrap()));
|
||||||
|
return Some(Reply::from(json!({ "chunk": chunk, "end": end }).to_string()));
|
||||||
|
}
|
||||||
|
None
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn one_room(state: Vec<Value>, timeline: Vec<Value>) -> Rooms {
|
||||||
|
Arc::new(Mutex::new(BTreeMap::from([("!r:s".to_owned(), Room { state, timeline })])))
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn state_maps_to_facts_and_the_join_rule_is_not_read() {
|
||||||
|
let plain = one_room(vec![state("m.room.join_rules", json!({ "join_rule": "invite" }))], vec![]);
|
||||||
|
let (base, _) = synapse(&plain);
|
||||||
|
let facts = HttpMatrix::new(&base, TOKEN).facts("!r:s").unwrap();
|
||||||
|
assert_eq!((facts.alias, facts.name, facts.encrypted, facts.direct, facts.bridge), (None, None, false, false, None));
|
||||||
|
for rule in ["public", "knock", "invite"] {
|
||||||
|
let rooms = one_room(vec![state("m.room.join_rules", json!({ "join_rule": rule }))], vec![]);
|
||||||
|
let (base, _) = synapse(&rooms);
|
||||||
|
let facts = HttpMatrix::new(&base, TOKEN).facts("!r:s").unwrap();
|
||||||
|
assert!(!facts.direct && !facts.encrypted && facts.bridge.is_none(), "{rule}");
|
||||||
|
}
|
||||||
|
let full = one_room(
|
||||||
|
vec![
|
||||||
|
state("m.room.canonical_alias", json!({ "alias": "#a:s", "alt_aliases": ["#b:s", "#c:s"] })),
|
||||||
|
state("m.room.name", json!({ "name": "Swarm" })),
|
||||||
|
state("m.room.encryption", json!({ "algorithm": "m.megolm.v1.aes-sha2" })),
|
||||||
|
json!({ "type": "m.room.member", "state_key": "@x:s", "content": { "membership": "join" } }),
|
||||||
|
json!({ "type": "m.room.member", "state_key": "@y:s", "content": { "membership": "join", "is_direct": true } }),
|
||||||
|
state("uk.half-shot.bridge", json!({})),
|
||||||
|
],
|
||||||
|
vec![],
|
||||||
|
);
|
||||||
|
let (base, _) = synapse(&full);
|
||||||
|
let facts = HttpMatrix::new(&base, TOKEN).facts("!r:s").unwrap();
|
||||||
|
assert_eq!(facts.alias.as_deref(), Some("#a:s"));
|
||||||
|
assert_eq!(facts.alt_aliases, ["#b:s", "#c:s"]);
|
||||||
|
assert_eq!(facts.name.as_deref(), Some("Swarm"));
|
||||||
|
assert!(facts.encrypted && facts.direct);
|
||||||
|
assert_eq!(facts.bridge.as_deref(), Some("uk.half-shot.bridge"));
|
||||||
|
let relay = one_room(vec![state("m.bridge", json!({}))], vec![]);
|
||||||
|
let (base, _) = synapse(&relay);
|
||||||
|
assert_eq!(HttpMatrix::new(&base, TOKEN).facts("!r:s").unwrap().bridge.as_deref(), Some("m.bridge"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn events_map_to_kinds_and_content_keys_are_names_only() {
|
||||||
|
let mut other = message("$img", "@a:s", 3, "pic.png");
|
||||||
|
other["content"]["msgtype"] = json!("m.image");
|
||||||
|
let mut pruned = message("$pruned", "@a:s", 4, "was here");
|
||||||
|
pruned["unsigned"] = json!({ "redacted_because": { "event_id": "$x" } });
|
||||||
|
let mut notice = message("$n", "@a:s", 5, "n");
|
||||||
|
notice["content"]["msgtype"] = json!("m.notice");
|
||||||
|
let mut emote = message("$e", "@a:s", 6, "e");
|
||||||
|
emote["content"]["msgtype"] = json!("m.emote");
|
||||||
|
let edit = json!({ "type": "m.room.message", "event_id": "$edit", "sender": "@a:s", "origin_server_ts": 7, "content": {
|
||||||
|
"msgtype": "m.text", "body": "* new", "m.new_content": { "msgtype": "m.text", "body": "new" }, "m.relates_to": { "rel_type": "m.replace", "event_id": "$m1" } } });
|
||||||
|
let redaction = json!({ "type": "m.room.redaction", "event_id": "$red", "sender": "@a:s", "origin_server_ts": 8, "content": { "redacts": "$m1" } });
|
||||||
|
let old_redaction = json!({ "type": "m.room.redaction", "event_id": "$red2", "sender": "@a:s", "origin_server_ts": 9, "redacts": "$m2", "content": {} });
|
||||||
|
let encrypted = json!({ "type": "m.room.encrypted", "event_id": "$enc", "sender": "@a:s", "origin_server_ts": 10, "content": { "ciphertext": "xx" } });
|
||||||
|
let timeline = vec![message("$m1", "@a:s", 1, "hello"), other, pruned, notice, emote, edit, redaction, old_redaction, encrypted];
|
||||||
|
let rooms = one_room(vec![], timeline);
|
||||||
|
let (base, _) = synapse(&rooms);
|
||||||
|
let page = HttpMatrix::new(&base, TOKEN).events_after("!r:s", None, 50).unwrap().unwrap();
|
||||||
|
let kinds: Vec<EventKind> = page.events.iter().map(|event| event.kind.clone()).collect();
|
||||||
|
assert_eq!(
|
||||||
|
kinds,
|
||||||
|
[
|
||||||
|
EventKind::Message { body: "hello".into() },
|
||||||
|
EventKind::Other,
|
||||||
|
EventKind::Other,
|
||||||
|
EventKind::Message { body: "n".into() },
|
||||||
|
EventKind::Message { body: "e".into() },
|
||||||
|
EventKind::Edit { target: "$m1".into(), body: "new".into() },
|
||||||
|
EventKind::Redaction { target: "$m1".into() },
|
||||||
|
EventKind::Redaction { target: "$m2".into() },
|
||||||
|
EventKind::Other,
|
||||||
|
]
|
||||||
|
);
|
||||||
|
assert_eq!(page.events[0].content_keys, ["body", "msgtype"]);
|
||||||
|
assert_eq!(page.events[5].content_keys, ["body", "m.new_content", "m.relates_to", "msgtype"], "names, never values");
|
||||||
|
assert_eq!((page.events[0].event_id.as_str(), page.events[0].sender.as_str(), page.events[0].ts_ms), ("$m1", "@a:s", 1));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn resume_uses_the_cursor_events_end_token_and_a_missing_event_is_none() {
|
||||||
|
let rooms = one_room(vec![], vec![message("$1", "@a:s", 1, "a"), message("$2", "@a:s", 2, "b"), message("$3", "@a:s", 3, "c")]);
|
||||||
|
let (base, asked) = synapse(&rooms);
|
||||||
|
let mut matrix = HttpMatrix::new(&base, TOKEN);
|
||||||
|
let page = matrix.events_after("!r:s", Some("$1"), 1).unwrap().unwrap();
|
||||||
|
assert_eq!(page.events.iter().map(|event| event.event_id.as_str()).collect::<Vec<_>>(), ["$2"]);
|
||||||
|
assert!(page.more);
|
||||||
|
assert_eq!(asked.paths(), ["/_matrix/client/v3/rooms/%21r%3As/context/%241?limit=0", "/_matrix/client/v3/rooms/%21r%3As/messages?dir=f&limit=1&from=t_%241"]);
|
||||||
|
assert!(matrix.events_after("!r:s", Some("$unknown"), 5).unwrap().is_none());
|
||||||
|
let tail = matrix.events_after("!r:s", Some("$3"), 5).unwrap().unwrap();
|
||||||
|
assert!(tail.events.is_empty() && !tail.more, "an empty chunk ends the run");
|
||||||
|
let first = matrix.events_after("!r:s", None, 2).unwrap().unwrap();
|
||||||
|
assert_eq!(first.events.len(), 2);
|
||||||
|
assert_eq!(asked.paths().last().unwrap(), "/_matrix/client/v3/rooms/%21r%3As/messages?dir=f&limit=2", "no from, no context call");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn only_get_always_the_bearer_and_the_token_is_never_printed() {
|
||||||
|
let rooms = one_room(vec![], vec![message("$1", "@a:s", 1, "a")]);
|
||||||
|
let (base, asked) = synapse(&rooms);
|
||||||
|
let mut matrix = HttpMatrix::new(&base, TOKEN);
|
||||||
|
matrix.joined_rooms().unwrap();
|
||||||
|
matrix.facts("!r:s").unwrap();
|
||||||
|
matrix.events_after("!r:s", Some("$1"), 3).unwrap();
|
||||||
|
let seen = asked.seen();
|
||||||
|
assert!(seen.len() >= 4 && seen.iter().all(|seen| seen.method == "GET" && seen.authorization.as_deref() == Some(&format!("Bearer {TOKEN}"))), "{seen:?}");
|
||||||
|
let (base, _) = forge_of(|_| Some(Reply::status(401, &format!(r#"{{"errcode":"M_UNKNOWN_TOKEN","error":"bad {TOKEN}"}}"#))));
|
||||||
|
let mut matrix = HttpMatrix::new(&base, TOKEN);
|
||||||
|
let error = matrix.joined_rooms().unwrap_err();
|
||||||
|
assert!(format!("{error:#}").contains("401"));
|
||||||
|
assert!(!format!("{error:#}{error:?}").contains(TOKEN));
|
||||||
|
assert!(!format!("{matrix:?}").contains(TOKEN));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn ids_are_percent_encoded_in_paths() {
|
||||||
|
let weird = "$a+b/c=d:s";
|
||||||
|
let rooms: Rooms = Arc::new(Mutex::new(BTreeMap::from([("!ro om:s".to_owned(), Room { state: vec![], timeline: vec![message(weird, "@a:s", 1, "x"), message("$2", "@a:s", 2, "y")] })])));
|
||||||
|
let (base, asked) = synapse(&rooms);
|
||||||
|
let mut matrix = HttpMatrix::new(&base, TOKEN);
|
||||||
|
let page = matrix.events_after("!ro om:s", Some(weird), 5).unwrap().unwrap();
|
||||||
|
assert_eq!(page.events.len(), 1, "the server decoded the room and event ids to the originals");
|
||||||
|
let raw = asked.paths();
|
||||||
|
assert!(raw.iter().all(|path| !path.contains('+') && !path.contains(' ') && !path.contains("!ro")), "{raw:?}");
|
||||||
|
}
|
||||||
|
|
@ -1,26 +1,48 @@
|
||||||
//! The source index: the public GameCult forge repos' code and docs as chunks
|
//! The source index: the public GameCult forge repos' code and docs, and the
|
||||||
//! in one Qdrant collection, `huginn_sources`, for semantic search.
|
//! Minds' chat rooms, as chunks in one Qdrant collection, `huginn_sources`,
|
||||||
|
//! for semantic search. Each point's `corpus` payload (`forge` or `chat`) keeps
|
||||||
|
//! the two apart; a forge search filters `forge` and a chat search `chat`.
|
||||||
//!
|
//!
|
||||||
//! Two truths, one writer. The forge is the truth of which repos exist and what
|
//! Two truths, one job. The forge is the truth of which repos exist and what
|
||||||
//! they hold; the collection is derived from it and can be deleted whenever.
|
//! they hold, Synapse of what the Minds' rooms said; the collection is derived
|
||||||
//! `refresh` is the only writer of the collection, run by the `huginn-sources`
|
//! from them and can be deleted whenever (but for a steward's purge tombstones, see
|
||||||
//! oneshot job: first build, incremental refresh, rebuild after a model or
|
//! `chat_purge`). `refresh` is the forge's writer and the only creator of the
|
||||||
//! chunker change, and retirement are all the one routine (delete by filter,
|
//! collection, run by the `huginn-sources` oneshot job: first build, incremental
|
||||||
//! ingest at a commit, rewrite the cursors). The daemon only reads: `search`
|
//! refresh, rebuild after a model or chunker change, and retirement are all the one
|
||||||
//! is its whole part, and nothing in `serve` or the worker calls a write.
|
//! routine (delete by filter, ingest at a commit, rewrite the cursors). `chat::refresh_chat`
|
||||||
|
//! joins it in the same job and the same cursors object, as the only writer of chat points;
|
||||||
|
//! it requires the collection to exist and never recreates it. The daemon only reads: `search`
|
||||||
|
//! and `search_chat` are its whole part, and nothing in `serve` or the worker calls a write.
|
||||||
//!
|
//!
|
||||||
//! A cursor is the commit whose chunks the collection holds for a repo. It
|
//! A cursor is the commit (or, for a room, the event) whose chunks the collection
|
||||||
//! lives in the collection's own metadata, so it is lost with the collection
|
//! holds. It lives in the collection's own metadata, so it is lost with the collection
|
||||||
//! and the two cannot disagree about what is held. It is advanced only after
|
//! and the two cannot disagree about what is held. It is advanced only after
|
||||||
//! the repo's deletes and upserts are acknowledged: it may lag and never leads.
|
//! the source's deletes and upserts are acknowledged: it may lag and never leads.
|
||||||
//! It is dropped before the repo's first delete, so a run that fails between
|
//! A repo's is dropped before its first delete, so a run that fails between
|
||||||
//! the delete and the upserts leaves no cursor, and the repo is ingested whole.
|
//! the delete and the upserts leaves no cursor, and the repo is ingested whole; a room's
|
||||||
//! The cursors object is written whole by this job alone, once per repo.
|
//! chat chunks are written before anything they replace is deleted. The cursors
|
||||||
|
//! object is written whole by this job alone. A key is a repo name, or a chat key
|
||||||
|
//! (`chat::is_chat_key`), never both.
|
||||||
|
|
||||||
|
pub mod chat;
|
||||||
|
pub mod chat_chunk;
|
||||||
|
pub mod chat_purge;
|
||||||
pub mod chunk;
|
pub mod chunk;
|
||||||
pub mod forge;
|
pub mod forge;
|
||||||
pub mod git;
|
pub mod git;
|
||||||
|
pub mod job;
|
||||||
|
pub mod matrix;
|
||||||
|
pub mod room_scope;
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod test_http;
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod chat_tests;
|
||||||
|
#[cfg(test)]
|
||||||
|
mod job_tests;
|
||||||
|
#[cfg(test)]
|
||||||
|
mod matrix_tests;
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests;
|
mod tests;
|
||||||
|
|
||||||
|
|
@ -30,7 +52,7 @@ use std::path::Path;
|
||||||
use std::time::{Duration, Instant};
|
use std::time::{Duration, Instant};
|
||||||
|
|
||||||
use anyhow::{Context, Result, bail, ensure};
|
use anyhow::{Context, Result, bail, ensure};
|
||||||
use huginn_mind::{SOURCE_TOP_K_MAX, SourceHit, SourceHits, SourceModel, SourceQuery};
|
use huginn_mind::{ChatHit, ChatHits, ChatMessage, ChatQuery, ChatWindow, ChatWindowQuery, MindRefusal, SOURCE_TOP_K_MAX, SourceHit, SourceHits, SourceModel, SourceQuery};
|
||||||
|
|
||||||
use crate::index::{BATCH, BULK_EMBED_TIMEOUT, Budget, Embedder, INDEX_CALL_TIMEOUT, MANAGED_BY, ModelIdentity, point_id};
|
use crate::index::{BATCH, BULK_EMBED_TIMEOUT, Budget, Embedder, INDEX_CALL_TIMEOUT, MANAGED_BY, ModelIdentity, point_id};
|
||||||
use chunk::{CHUNKER_VERSION, embedded_text};
|
use chunk::{CHUNKER_VERSION, embedded_text};
|
||||||
|
|
@ -43,8 +65,19 @@ pub const COLLECTION: &str = "huginn_sources";
|
||||||
/// agent reads first to learn what the repo is. The only owner of the list.
|
/// agent reads first to learn what the repo is. The only owner of the list.
|
||||||
pub const FRONT_PAGE_FILES: [&str; 6] = ["README.md", "README", "readme.md", "README.markdown", "CLAUDE.md", "AGENTS.md"];
|
pub const FRONT_PAGE_FILES: [&str; 6] = ["README.md", "README", "readme.md", "README.markdown", "CLAUDE.md", "AGENTS.md"];
|
||||||
|
|
||||||
/// What the collection says its corpus is.
|
/// What the collection says its corpus is: the set of corpora it holds, each
|
||||||
pub const CORPUS: &str = "forge:GameCult";
|
/// point's `corpus` payload naming which (`forge` or `chat`). A collection
|
||||||
|
/// labelled with any other set is rebuilt.
|
||||||
|
pub const CORPUS: &str = "forge:GameCult+chat:rooms";
|
||||||
|
|
||||||
|
/// The `corpus` payload of a forge point, and of a chat point.
|
||||||
|
pub const CORPUS_FORGE: &str = "forge";
|
||||||
|
pub const CORPUS_CHAT: &str = "chat";
|
||||||
|
|
||||||
|
/// The instruction Qwen3 embedding models take in front of a question about
|
||||||
|
/// past conversation.
|
||||||
|
pub const CHAT_QUERY_INSTRUCTION: &str =
|
||||||
|
"Given a question about past conversation in chat rooms, retrieve the chat passages that answer it";
|
||||||
|
|
||||||
/// The instruction Qwen3 embedding models take in front of a query about code.
|
/// The instruction Qwen3 embedding models take in front of a query about code.
|
||||||
pub const SOURCE_QUERY_INSTRUCTION: &str =
|
pub const SOURCE_QUERY_INSTRUCTION: &str =
|
||||||
|
|
@ -111,6 +144,80 @@ pub struct SourcePoint {
|
||||||
pub payload: SourcePayload,
|
pub payload: SourcePayload,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// What travels beside a chat chunk's vector: consecutive messages of one
|
||||||
|
/// room, never spanning rooms. The single owner of the chat point's shape; the
|
||||||
|
/// ingest job is its only writer (through `upsert_chat`) and nothing else of the
|
||||||
|
/// payload is read. `messages` are the display truth and hold, in order, at
|
||||||
|
/// least one message. The chunk's span, its first and last event ids and its
|
||||||
|
/// `event_ids` are derived from them by the codec, so they cannot disagree.
|
||||||
|
/// `text` is what was embedded: the room's name, then `time sender: body` per
|
||||||
|
/// message.
|
||||||
|
#[derive(Clone, Debug, PartialEq)]
|
||||||
|
pub struct ChatPayload {
|
||||||
|
pub room_id: String,
|
||||||
|
/// Absent when the room has no canonical alias.
|
||||||
|
pub room_alias: Option<String>,
|
||||||
|
pub messages: Vec<ChatMessage>,
|
||||||
|
pub text: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, PartialEq)]
|
||||||
|
pub struct ChatPoint {
|
||||||
|
pub id: String,
|
||||||
|
pub vector: Vec<f32>,
|
||||||
|
pub payload: ChatPayload,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One chat chunk a vector search returned and how near it scored.
|
||||||
|
#[derive(Clone, Debug, PartialEq)]
|
||||||
|
pub struct ChatRow {
|
||||||
|
pub score: f32,
|
||||||
|
pub payload: ChatPayload,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Which chat chunks a read keeps; the selectors compose as AND. `rooms` are
|
||||||
|
/// any-of room ids (`!`) or aliases (`#`); the span keeps chunks whose own
|
||||||
|
/// span overlaps [`after_ms`, `before_ms`]. `event_id` keeps the chunk holding
|
||||||
|
/// that message. `starts_from_ms` and `starts_to_ms` bound a chunk's own start
|
||||||
|
/// (inclusive), which is what a neighbour read walks by.
|
||||||
|
#[derive(Clone, Debug, Default, PartialEq, Eq)]
|
||||||
|
pub struct ChatFilter {
|
||||||
|
pub rooms: Vec<String>,
|
||||||
|
pub after_ms: Option<u64>,
|
||||||
|
pub before_ms: Option<u64>,
|
||||||
|
pub event_id: Option<String>,
|
||||||
|
pub starts_from_ms: Option<u64>,
|
||||||
|
pub starts_to_ms: Option<u64>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Which end of the room's history a chat scroll begins at.
|
||||||
|
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||||
|
pub enum ChatOrder {
|
||||||
|
OldestFirst,
|
||||||
|
NewestFirst,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ChatFilter {
|
||||||
|
/// Whether a chunk of `room_id`/`room_alias` spanning [`start`, `end`]
|
||||||
|
/// passes. The fake index and the tests read the filter through this; the
|
||||||
|
/// Qdrant index states the same thing as conditions.
|
||||||
|
pub fn keeps(&self, room_id: &str, room_alias: Option<&str>, start: u64, end: u64) -> bool {
|
||||||
|
(self.rooms.is_empty() || self.rooms.iter().any(|room| room == room_id || Some(room.as_str()) == room_alias))
|
||||||
|
&& self.after_ms.is_none_or(|after| end >= after)
|
||||||
|
&& self.before_ms.is_none_or(|before| start <= before)
|
||||||
|
&& self.starts_from_ms.is_none_or(|from| start >= from)
|
||||||
|
&& self.starts_to_ms.is_none_or(|to| start <= to)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A chat chunk as the index holds it, with its point id: what the ingest job
|
||||||
|
/// reads back to rewrite a chunk.
|
||||||
|
#[derive(Clone, Debug, PartialEq)]
|
||||||
|
pub struct StoredChat {
|
||||||
|
pub id: String,
|
||||||
|
pub payload: ChatPayload,
|
||||||
|
}
|
||||||
|
|
||||||
/// One chunk a vector search returned and how near it scored.
|
/// One chunk a vector search returned and how near it scored.
|
||||||
#[derive(Clone, Debug, PartialEq)]
|
#[derive(Clone, Debug, PartialEq)]
|
||||||
pub struct SourceRow {
|
pub struct SourceRow {
|
||||||
|
|
@ -124,7 +231,9 @@ pub struct SourceRow {
|
||||||
pub trait SourceIndex {
|
pub trait SourceIndex {
|
||||||
fn describe_sources(&mut self, within: Duration) -> Result<DescribedSources>;
|
fn describe_sources(&mut self, within: Duration) -> Result<DescribedSources>;
|
||||||
/// Drops the collection if it exists and creates it empty with this
|
/// Drops the collection if it exists and creates it empty with this
|
||||||
/// metadata, no cursors, and keyword indexes on `repo` and `path`.
|
/// metadata, no cursors, keyword indexes on `repo`, `path`, `corpus`,
|
||||||
|
/// `room_id`, `room_alias` and `event_ids`, and integer indexes on
|
||||||
|
/// `ts_start_ms` and `ts_end_ms`.
|
||||||
fn recreate_sources(&mut self, meta: &SourcesMeta) -> Result<()>;
|
fn recreate_sources(&mut self, meta: &SourcesMeta) -> Result<()>;
|
||||||
/// Replaces the cursors object whole.
|
/// Replaces the cursors object whole.
|
||||||
fn set_cursors(&mut self, cursors: &BTreeMap<String, String>) -> Result<()>;
|
fn set_cursors(&mut self, cursors: &BTreeMap<String, String>) -> Result<()>;
|
||||||
|
|
@ -137,6 +246,25 @@ pub trait SourceIndex {
|
||||||
/// The `limit` nearest chunks, best first, in `repo` alone when given, and
|
/// The `limit` nearest chunks, best first, in `repo` alone when given, and
|
||||||
/// among the `FRONT_PAGE_FILES` alone when `front`.
|
/// among the `FRONT_PAGE_FILES` alone when `front`.
|
||||||
fn search_sources(&mut self, vector: &[f32], repo: Option<&str>, front: bool, limit: u32, within: Duration) -> Result<Vec<SourceRow>>;
|
fn search_sources(&mut self, vector: &[f32], repo: Option<&str>, front: bool, limit: u32, within: Duration) -> Result<Vec<SourceRow>>;
|
||||||
|
/// Writes chat chunks, acknowledged before it returns. The ingest job's
|
||||||
|
/// alone: the daemon never calls it. A chunk with no messages is refused.
|
||||||
|
fn upsert_chat(&mut self, points: &[ChatPoint]) -> Result<()>;
|
||||||
|
/// The `limit` chat chunks nearest `vector`, best first, among those
|
||||||
|
/// `filter` keeps. Reads only `corpus=chat` points; the daemon's.
|
||||||
|
fn chat_nearest(&mut self, vector: &[f32], filter: &ChatFilter, limit: u32, within: Duration) -> Result<Vec<ChatRow>>;
|
||||||
|
/// Up to `limit` chat chunks `filter` keeps, by span start in `order`.
|
||||||
|
/// Reads only `corpus=chat` points; the daemon's.
|
||||||
|
fn chat_scroll(&mut self, filter: &ChatFilter, order: ChatOrder, limit: u32, within: Duration) -> Result<Vec<ChatPayload>>;
|
||||||
|
/// The room's chunk with the greatest `ts_end_ms`, if it has any. The ingest job's alone.
|
||||||
|
fn chat_tail(&mut self, room_id: &str) -> Result<Option<StoredChat>>;
|
||||||
|
/// The room's chunks holding any of `event_ids`. The ingest job's alone.
|
||||||
|
fn chat_chunks_holding(&mut self, room_id: &str, event_ids: &[String]) -> Result<Vec<StoredChat>>;
|
||||||
|
/// The room's chunks whose span reaches into [`from_ms`, `to_ms`): `ts_end_ms >= from_ms`
|
||||||
|
/// and `ts_start_ms < to_ms`. The ingest job's alone.
|
||||||
|
fn chat_chunks_overlapping(&mut self, room_id: &str, from_ms: u64, to_ms: u64) -> Result<Vec<StoredChat>>;
|
||||||
|
/// Deletes a room's chat chunks: those with the point ids `ids`, or all of them when `None`.
|
||||||
|
/// Acknowledged before it returns. Never touches a forge point. The ingest job's alone.
|
||||||
|
fn delete_chat(&mut self, room_id: &str, ids: Option<&[String]>) -> Result<()>;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// A repo the forge lists as indexable.
|
/// A repo the forge lists as indexable.
|
||||||
|
|
@ -305,7 +433,7 @@ fn retire<S: SourceIndex>(
|
||||||
failed: &mut Vec<String>,
|
failed: &mut Vec<String>,
|
||||||
) {
|
) {
|
||||||
let listed = |name: &str| catalog.iter().any(|repo| repo.name == name);
|
let listed = |name: &str| catalog.iter().any(|repo| repo.name == name);
|
||||||
let mut gone: BTreeSet<String> = cursors.keys().filter(|name| !listed(name)).cloned().collect();
|
let mut gone: BTreeSet<String> = cursors.keys().filter(|name| !chat::is_chat_key(name) && !listed(name)).cloned().collect();
|
||||||
if let Ok(entries) = std::fs::read_dir(mirrors) {
|
if let Ok(entries) = std::fs::read_dir(mirrors) {
|
||||||
for entry in entries.flatten() {
|
for entry in entries.flatten() {
|
||||||
if let Some(name) = entry.file_name().to_string_lossy().strip_suffix(".git")
|
if let Some(name) = entry.file_name().to_string_lossy().strip_suffix(".git")
|
||||||
|
|
@ -476,6 +604,10 @@ pub fn fuse<T>(lists: Vec<Vec<T>>, key: impl Fn(&T) -> String, k: u32, top_k: us
|
||||||
/// of one, trimmed, and each distinct text once, in the order asked.
|
/// of one, trimmed, and each distinct text once, in the order asked.
|
||||||
fn query_texts(query: &SourceQuery) -> Vec<&str> {
|
fn query_texts(query: &SourceQuery) -> Vec<&str> {
|
||||||
let asked: Vec<&str> = if query.texts.is_empty() { vec![query.text.0.as_str()] } else { query.texts.iter().map(|text| text.0.as_str()).collect() };
|
let asked: Vec<&str> = if query.texts.is_empty() { vec![query.text.0.as_str()] } else { query.texts.iter().map(|text| text.0.as_str()).collect() };
|
||||||
|
distinct_texts(asked)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn distinct_texts(asked: Vec<&str>) -> Vec<&str> {
|
||||||
let mut distinct: Vec<&str> = Vec::with_capacity(asked.len());
|
let mut distinct: Vec<&str> = Vec::with_capacity(asked.len());
|
||||||
for text in asked {
|
for text in asked {
|
||||||
if !distinct.iter().any(|seen| seen.trim() == text.trim()) {
|
if !distinct.iter().any(|seen| seen.trim() == text.trim()) {
|
||||||
|
|
@ -485,16 +617,14 @@ fn query_texts(query: &SourceQuery) -> Vec<&str> {
|
||||||
distinct
|
distinct
|
||||||
}
|
}
|
||||||
|
|
||||||
/// A source search, which a daemon answers from the collection as it finds it:
|
/// The label of the collection a daemon is about to read, if it was built
|
||||||
/// it embeds the query with its own embedder, and refuses if the collection
|
/// under the model this daemon embeds with. Absent, unlabelled and other-model
|
||||||
/// was built under another model, because vectors of two models do not compare.
|
/// collections are loud refusals: vectors of two models do not compare, and an
|
||||||
pub fn search<E: Embedder, S: SourceIndex>(embedder: &mut E, index: &mut S, query: &SourceQuery, within: Duration) -> Result<SourceHits> {
|
/// empty page is not an answer. A collection built for another corpus set (an
|
||||||
let budget = Budget::new(within);
|
/// old forge-only one) is refused the same way until the rebuild lands, since
|
||||||
let built = match index.describe_sources(budget.left()?).context("describing the source collection")? {
|
/// it would answer a chat search with an empty page.
|
||||||
DescribedSources::Absent => bail!("source index not built"),
|
fn readable_under_this_model<E: Embedder, S: SourceIndex>(embedder: &mut E, index: &mut S, budget: &Budget) -> Result<(SourcesMeta, ModelIdentity)> {
|
||||||
DescribedSources::Unlabelled => bail!("source index not built: {COLLECTION} carries no Huginn metadata"),
|
let built = built_label(index, budget)?;
|
||||||
DescribedSources::Labelled { meta, .. } => meta,
|
|
||||||
};
|
|
||||||
let identity = embedder.model_identity(budget.left()?).context("reading the embedding model's identity")?;
|
let identity = embedder.model_identity(budget.left()?).context("reading the embedding model's identity")?;
|
||||||
ensure!(
|
ensure!(
|
||||||
built.model == identity.name && built.model_digest == identity.digest && built.dimensions == identity.dimensions,
|
built.model == identity.name && built.model_digest == identity.digest && built.dimensions == identity.dimensions,
|
||||||
|
|
@ -506,6 +636,30 @@ pub fn search<E: Embedder, S: SourceIndex>(embedder: &mut E, index: &mut S, quer
|
||||||
identity.digest,
|
identity.digest,
|
||||||
identity.dimensions
|
identity.dimensions
|
||||||
);
|
);
|
||||||
|
Ok((built, identity))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn built_label<S: SourceIndex>(index: &mut S, budget: &Budget) -> Result<SourcesMeta> {
|
||||||
|
match index.describe_sources(budget.left()?).context("describing the source collection")? {
|
||||||
|
DescribedSources::Absent => bail!("source index not built"),
|
||||||
|
DescribedSources::Unlabelled => bail!("source index not built: {COLLECTION} carries no Huginn metadata"),
|
||||||
|
DescribedSources::Labelled { meta, .. } => {
|
||||||
|
ensure!(
|
||||||
|
meta.corpus == CORPUS,
|
||||||
|
"the source index is stale: it was built for corpus {} and this daemon reads {CORPUS}; the rebuild is pending",
|
||||||
|
meta.corpus
|
||||||
|
);
|
||||||
|
Ok(meta)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A source search, which a daemon answers from the collection as it finds it:
|
||||||
|
/// it embeds the query with its own embedder, and refuses if the collection
|
||||||
|
/// was built under another model, because vectors of two models do not compare.
|
||||||
|
pub fn search<E: Embedder, S: SourceIndex>(embedder: &mut E, index: &mut S, query: &SourceQuery, within: Duration) -> Result<SourceHits> {
|
||||||
|
let budget = Budget::new(within);
|
||||||
|
let (built, identity) = readable_under_this_model(embedder, index, &budget)?;
|
||||||
let topics = query_texts(query);
|
let topics = query_texts(query);
|
||||||
let prompts: Vec<String> = topics.iter().map(|topic| format!("Instruct: {SOURCE_QUERY_INSTRUCTION}\nQuery: {topic}")).collect();
|
let prompts: Vec<String> = topics.iter().map(|topic| format!("Instruct: {SOURCE_QUERY_INSTRUCTION}\nQuery: {topic}")).collect();
|
||||||
let vectors = embedder.embed(&prompts, budget.left()?).context("embedding the query")?;
|
let vectors = embedder.embed(&prompts, budget.left()?).context("embedding the query")?;
|
||||||
|
|
@ -539,3 +693,161 @@ pub fn search<E: Embedder, S: SourceIndex>(embedder: &mut E, index: &mut S, quer
|
||||||
.collect(),
|
.collect(),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// A chat search. With texts, the chunks nearest them among those the room and
|
||||||
|
/// span selectors keep, several texts fused by `fuse`; with none, a listing of
|
||||||
|
/// what the selectors keep, oldest first, `top_k` of it and `truncated` saying
|
||||||
|
/// more exist (no embedding, so the model's identity is not consulted, though
|
||||||
|
/// an absent collection is still refused). A hit outside the span is not
|
||||||
|
/// returned and a boundary hit loses its messages outside [after, before], its
|
||||||
|
/// span and first and last ids recomputed from what is left. Never writes.
|
||||||
|
pub fn search_chat<E: Embedder, S: SourceIndex>(embedder: &mut E, index: &mut S, query: &ChatQuery, within: Duration) -> Result<ChatHits> {
|
||||||
|
let budget = Budget::new(within);
|
||||||
|
let filter = ChatFilter { rooms: query.rooms.iter().map(|room| room.0.clone()).collect(), after_ms: query.after_ms, before_ms: query.before_ms, ..ChatFilter::default() };
|
||||||
|
let top_k = query.top_k as usize;
|
||||||
|
let topics = distinct_texts(query.texts.iter().map(|text| text.0.as_str()).collect());
|
||||||
|
let (built, rows, truncated) = if topics.is_empty() {
|
||||||
|
let built = built_label(index, &budget)?;
|
||||||
|
let mut held = index.chat_scroll(&filter, ChatOrder::OldestFirst, query.top_k + 1, budget.left()?)?;
|
||||||
|
let truncated = held.len() > top_k;
|
||||||
|
held.truncate(top_k);
|
||||||
|
(built, held.into_iter().map(|payload| (None, payload)).collect::<Vec<_>>(), truncated)
|
||||||
|
} else {
|
||||||
|
let (built, identity) = readable_under_this_model(embedder, index, &budget)?;
|
||||||
|
let prompts: Vec<String> = topics.iter().map(|topic| format!("Instruct: {CHAT_QUERY_INSTRUCTION}\nQuery: {topic}")).collect();
|
||||||
|
let vectors = embedder.embed(&prompts, budget.left()?).context("embedding the query")?;
|
||||||
|
ensure!(vectors.len() == prompts.len(), "the embedder returned {} vectors for {} query texts", vectors.len(), prompts.len());
|
||||||
|
let depth = if vectors.len() == 1 { query.top_k } else { FUSE_DEPTH };
|
||||||
|
let mut lists = Vec::with_capacity(vectors.len());
|
||||||
|
for vector in &vectors {
|
||||||
|
ensure!(vector.len() == identity.dimensions as usize, "the embedder returned a query vector of {} where the collection holds {}", vector.len(), identity.dimensions);
|
||||||
|
lists.push(index.chat_nearest(vector, &filter, depth, budget.left()?)?);
|
||||||
|
}
|
||||||
|
let rows: Vec<ChatRow> = if lists.len() == 1 {
|
||||||
|
lists.remove(0)
|
||||||
|
} else {
|
||||||
|
let key = |row: &ChatRow| format!("{}\0{}", row.payload.room_id, row.payload.messages.first().map_or("", |message| message.event_id.as_str()));
|
||||||
|
fuse(lists, key, FUSE_K, top_k).into_iter().map(|(score, row)| ChatRow { score, ..row }).collect()
|
||||||
|
};
|
||||||
|
(built, rows.into_iter().map(|row| (Some(row.score), row.payload)).collect(), false)
|
||||||
|
};
|
||||||
|
Ok(ChatHits {
|
||||||
|
indexed_under: SourceModel { name: built.model, digest: built.model_digest },
|
||||||
|
truncated,
|
||||||
|
hits: rows.into_iter().filter_map(|(score, payload)| chat_hit(score, payload, &filter)).collect(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Chunks a neighbour read asks for at a time, and the most such reads each way.
|
||||||
|
const WINDOW_CHUNKS_PER_SCROLL: u32 = 5;
|
||||||
|
const WINDOW_SCROLLS_MAX: usize = 6;
|
||||||
|
|
||||||
|
/// The message a chat window is about and its neighbours, read from the chat
|
||||||
|
/// index alone: the chunk holding `event_id`, then chunks of that room before
|
||||||
|
/// and after it until the asked count of messages is held, the room ends or
|
||||||
|
/// `WINDOW_SCROLLS_MAX` reads are spent. An id the index does not hold is a
|
||||||
|
/// typed refusal naming `event_id`, never an empty window; an id in two chunks
|
||||||
|
/// is an index fault. Needs no embedder. Never writes.
|
||||||
|
pub fn window<S: SourceIndex>(index: &mut S, query: &ChatWindowQuery, within: Duration) -> Result<Result<ChatWindow, MindRefusal>> {
|
||||||
|
let budget = Budget::new(within);
|
||||||
|
built_label(index, &budget)?;
|
||||||
|
let id = query.event_id.0.as_str();
|
||||||
|
let anchor_filter = ChatFilter { event_id: Some(id.to_owned()), ..ChatFilter::default() };
|
||||||
|
let mut found = index.chat_scroll(&anchor_filter, ChatOrder::OldestFirst, 2, budget.left()?)?;
|
||||||
|
ensure!(found.len() < 2, "the chat index holds event {id} in two chunks: an ingest fault, not a window");
|
||||||
|
let Some(anchor) = found.pop() else {
|
||||||
|
return Ok(Err(MindRefusal::SelectionInvalid {
|
||||||
|
field: "event_id".into(),
|
||||||
|
value: Some(id.to_owned()),
|
||||||
|
message: "the chat index does not hold this event: it may not be ingested yet, it may have been removed (redacted, or purged by a steward request), or it may be in a room the index does not read (a DM, #users, #members, #patrons, any room the reader has not joined)".into(),
|
||||||
|
}));
|
||||||
|
};
|
||||||
|
let start_of = |chunk: &ChatPayload| chunk.messages.iter().map(|message| message.ts_ms).min().unwrap_or(0);
|
||||||
|
let first_id = |chunk: &ChatPayload| chunk.messages.first().map(|message| message.event_id.clone()).unwrap_or_default();
|
||||||
|
let room = ChatFilter { rooms: vec![anchor.room_id.clone()], ..ChatFilter::default() };
|
||||||
|
let at = anchor.messages.iter().position(|message| message.event_id == id).context("the chat index returned a chunk that does not hold the event it was asked for")?;
|
||||||
|
// Chunks held so far as (first event id, start): the anchor's, then each walk's.
|
||||||
|
let mut held = vec![(first_id(&anchor), start_of(&anchor))];
|
||||||
|
let mut walk = |forward: bool, want: usize, mine: usize| -> Result<(Vec<ChatPayload>, bool)> {
|
||||||
|
let (mut chunks, mut count, mut edge, mut exhausted) = (Vec::new(), mine, start_of(&anchor), false);
|
||||||
|
for _ in 0..WINDOW_SCROLLS_MAX {
|
||||||
|
if count > want {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
let (filter, order) = if forward {
|
||||||
|
(ChatFilter { starts_from_ms: Some(edge), ..room.clone() }, ChatOrder::OldestFirst)
|
||||||
|
} else {
|
||||||
|
(ChatFilter { starts_to_ms: Some(edge), ..room.clone() }, ChatOrder::NewestFirst)
|
||||||
|
};
|
||||||
|
// Held chunks starting at the edge come back first with any others
|
||||||
|
// that start there; asking for that many extra means a run of
|
||||||
|
// chunks at one millisecond cannot fill a read with chunks held.
|
||||||
|
let limit = WINDOW_CHUNKS_PER_SCROLL + held.iter().filter(|(_, start)| *start == edge).count() as u32;
|
||||||
|
let read = index.chat_scroll(&filter, order, limit, budget.left()?)?;
|
||||||
|
exhausted = (read.len() as u32) < limit;
|
||||||
|
for chunk in read {
|
||||||
|
let key = (first_id(&chunk), start_of(&chunk));
|
||||||
|
if held.contains(&key) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
edge = if forward { key.1.max(edge) } else { key.1.min(edge) };
|
||||||
|
held.push(key);
|
||||||
|
count += chunk.messages.len();
|
||||||
|
chunks.push(chunk);
|
||||||
|
}
|
||||||
|
if exhausted {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Stopping short of the asked count without reaching the room's end
|
||||||
|
// means messages remain that this read did not reach.
|
||||||
|
Ok((chunks, count <= want && !exhausted))
|
||||||
|
};
|
||||||
|
let (earlier, unreached_before) = walk(false, query.before as usize, at)?;
|
||||||
|
let (later, unreached_after) = walk(true, query.after as usize, anchor.messages.len() - at - 1)?;
|
||||||
|
// Chronological by (ts, chunk start, chunk first event id, position), so
|
||||||
|
// the order does not depend on which message anchored the read.
|
||||||
|
let mut keyed: Vec<(u64, u64, String, usize, ChatMessage)> = Vec::new();
|
||||||
|
for chunk in earlier.iter().chain(std::iter::once(&anchor)).chain(later.iter()) {
|
||||||
|
let (start, first) = (start_of(chunk), first_id(chunk));
|
||||||
|
keyed.extend(chunk.messages.iter().enumerate().map(|(position, message)| (message.ts_ms, start, first.clone(), position, message.clone())));
|
||||||
|
}
|
||||||
|
keyed.sort_by(|a, b| (a.0, a.1, &a.2, a.3).cmp(&(b.0, b.1, &b.2, b.3)));
|
||||||
|
let (mut seen, mut messages) = (std::collections::BTreeSet::new(), Vec::new());
|
||||||
|
for (.., message) in keyed {
|
||||||
|
if seen.insert(message.event_id.clone()) {
|
||||||
|
messages.push(message);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let at = messages.iter().position(|message| message.event_id == id).context("the anchor fell out of its own window")?;
|
||||||
|
let (lo, hi) = (at.saturating_sub(query.before as usize), (at + 1 + query.after as usize).min(messages.len()));
|
||||||
|
Ok(Ok(ChatWindow {
|
||||||
|
room_id: anchor.room_id,
|
||||||
|
room_alias: anchor.room_alias,
|
||||||
|
anchor_event_id: id.to_owned(),
|
||||||
|
more_before: unreached_before || lo > 0,
|
||||||
|
more_after: unreached_after || hi < messages.len(),
|
||||||
|
messages: messages[lo..hi].to_vec(),
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A held chunk as a hit: only the messages inside the asked span, the span and
|
||||||
|
/// ids those messages make, and nothing at all when none is inside.
|
||||||
|
fn chat_hit(score: Option<f32>, payload: ChatPayload, filter: &ChatFilter) -> Option<ChatHit> {
|
||||||
|
let messages: Vec<ChatMessage> = payload
|
||||||
|
.messages
|
||||||
|
.into_iter()
|
||||||
|
.filter(|message| filter.after_ms.is_none_or(|after| message.ts_ms >= after) && filter.before_ms.is_none_or(|before| message.ts_ms <= before))
|
||||||
|
.collect();
|
||||||
|
let (first, last) = (messages.first()?, messages.last()?);
|
||||||
|
Some(ChatHit {
|
||||||
|
room_id: payload.room_id,
|
||||||
|
room_alias: payload.room_alias,
|
||||||
|
first_event_id: first.event_id.clone(),
|
||||||
|
last_event_id: last.event_id.clone(),
|
||||||
|
ts_start_ms: messages.iter().map(|message| message.ts_ms).min()?,
|
||||||
|
ts_end_ms: messages.iter().map(|message| message.ts_ms).max()?,
|
||||||
|
score,
|
||||||
|
messages,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
|
||||||
184
crates/huginn-daemon/src/sources/room_scope.rs
Normal file
184
crates/huginn-daemon/src/sources/room_scope.rs
Normal file
|
|
@ -0,0 +1,184 @@
|
||||||
|
//! Which joined rooms and which senders are in the chat corpus. The only code
|
||||||
|
//! that decides it, and the only file outside tests that names the bridged
|
||||||
|
//! platform whose content is refused.
|
||||||
|
//!
|
||||||
|
//! Scope (ruling `gamecult-chat:ruling:chat-index-minds-rooms`): the rooms the
|
||||||
|
//! reader user has joined, minus this exclusion. A room's join rule is read by
|
||||||
|
//! nobody (`RoomFacts` has no field for it): a joined invite-only room qualifies
|
||||||
|
//! like any other. The reader roster is the first lock; this file is the second.
|
||||||
|
|
||||||
|
use std::collections::BTreeSet;
|
||||||
|
|
||||||
|
use super::chat::RoomEvent;
|
||||||
|
|
||||||
|
/// The estate rooms by canonical alias, ASCII case-insensitive. Code owns the
|
||||||
|
/// aliases and the rule; the unit supplies the room ids (`Estate`), and either alone excludes.
|
||||||
|
pub const ESTATE_ALIASES: [&str; 3] = ["#users:gamecult.org", "#members:gamecult.org", "#patrons:gamecult.org"];
|
||||||
|
|
||||||
|
/// What the server says about a room, as far as admission needs it. The Matrix
|
||||||
|
/// client fills it from the room's state and decides nothing. It has no `Default`: a
|
||||||
|
/// room whose state could not be read is a port error, never a plain room.
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||||
|
pub struct RoomFacts {
|
||||||
|
pub room_id: String,
|
||||||
|
pub alias: Option<String>,
|
||||||
|
pub alt_aliases: Vec<String>,
|
||||||
|
pub name: Option<String>,
|
||||||
|
/// An `m.room.encryption` state event exists.
|
||||||
|
pub encrypted: bool,
|
||||||
|
/// Any `m.room.member` in the state has `is_direct` true.
|
||||||
|
pub direct: bool,
|
||||||
|
/// The state type (`m.bridge` or `uk.half-shot.bridge`) of a bridge event, when the room has one.
|
||||||
|
pub bridge: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The estate room ids the caller knows (they exist only on the live server): exactly one per
|
||||||
|
/// estate alias, so a job that was handed none, or too few, cannot be built.
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||||
|
pub struct Estate {
|
||||||
|
ids: BTreeSet<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Estate {
|
||||||
|
pub fn new(ids: BTreeSet<String>) -> anyhow::Result<Self> {
|
||||||
|
anyhow::ensure!(
|
||||||
|
ids.len() == ESTATE_ALIASES.len() && ids.iter().all(|id| id.starts_with('!')),
|
||||||
|
"the estate is exactly {} room ids ({}), got {}",
|
||||||
|
ESTATE_ALIASES.len(),
|
||||||
|
ESTATE_ALIASES.join(", "),
|
||||||
|
ids.len()
|
||||||
|
);
|
||||||
|
Ok(Self { ids })
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Whether `room_id` is a named estate room. Decided before the room's state is read.
|
||||||
|
pub fn holds_id(&self, room_id: &str) -> bool {
|
||||||
|
self.ids.contains(room_id)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Why a joined room, or an event in it, is refused.
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||||
|
pub enum Refusal {
|
||||||
|
Estate,
|
||||||
|
Direct,
|
||||||
|
Encrypted,
|
||||||
|
Bridged(String),
|
||||||
|
DiscordSender,
|
||||||
|
DiscordContent,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl std::fmt::Display for Refusal {
|
||||||
|
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||||
|
match self {
|
||||||
|
Self::Estate => formatter.write_str("estate room"),
|
||||||
|
Self::Direct => formatter.write_str("direct room"),
|
||||||
|
Self::Encrypted => formatter.write_str("encrypted room"),
|
||||||
|
Self::Bridged(state_type) => write!(formatter, "bridged room ({state_type})"),
|
||||||
|
Self::DiscordSender => formatter.write_str("discord sender"),
|
||||||
|
Self::DiscordContent => formatter.write_str("discord content"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Whether a joined room may be read at all.
|
||||||
|
pub fn admit_room(facts: &RoomFacts, estate: &Estate) -> Result<(), Refusal> {
|
||||||
|
let estate_alias = |alias: &str| ESTATE_ALIASES.iter().any(|estate| estate.eq_ignore_ascii_case(alias));
|
||||||
|
if estate.holds_id(&facts.room_id) || facts.alias.iter().chain(&facts.alt_aliases).any(|alias| estate_alias(alias)) {
|
||||||
|
return Err(Refusal::Estate);
|
||||||
|
}
|
||||||
|
if facts.direct {
|
||||||
|
return Err(Refusal::Direct);
|
||||||
|
}
|
||||||
|
if facts.encrypted {
|
||||||
|
return Err(Refusal::Encrypted);
|
||||||
|
}
|
||||||
|
if let Some(state_type) = &facts.bridge {
|
||||||
|
return Err(Refusal::Bridged(state_type.clone()));
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Whether an event refuses its whole room: a sender whose localpart, lowercased, starts with the
|
||||||
|
/// platform's name after any leading underscores, has it after an underscore (`mautrix_discord`,
|
||||||
|
/// `_bridge_discordbot`), or has it followed by `bot` or an underscore; or content with a top-level key
|
||||||
|
/// naming it. Deterministic identifier rules, not language reading; `discourse` is not it.
|
||||||
|
pub fn refuse_event(event: &RoomEvent) -> Option<Refusal> {
|
||||||
|
let localpart = event.sender.trim_start_matches('@').split(':').next().unwrap_or("").to_ascii_lowercase();
|
||||||
|
let name = ["disc", "ord"].concat();
|
||||||
|
if localpart.trim_start_matches('_').starts_with(&name) || localpart.contains(&format!("_{name}")) || localpart.contains(&format!("{name}bot")) || localpart.contains(&format!("{name}_")) {
|
||||||
|
return Some(Refusal::DiscordSender);
|
||||||
|
}
|
||||||
|
event.content_keys.iter().any(|key| key.to_ascii_lowercase().contains("discord")).then_some(Refusal::DiscordContent)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::super::chat::EventKind;
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
fn facts(id: &str) -> RoomFacts {
|
||||||
|
RoomFacts { room_id: id.into(), alias: None, alt_aliases: vec![], name: None, encrypted: false, direct: false, bridge: None }
|
||||||
|
}
|
||||||
|
|
||||||
|
fn estate(ids: &[&str]) -> Estate {
|
||||||
|
Estate::new(ids.iter().chain(["!pad1:s", "!pad2:s", "!pad3:s"].iter()).take(3).map(|id| (*id).to_owned()).collect()).unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn event(sender: &str, keys: &[&str]) -> RoomEvent {
|
||||||
|
RoomEvent { event_id: "$e".into(), sender: sender.into(), ts_ms: 1, content_keys: keys.iter().map(|key| (*key).into()).collect(), kind: EventKind::Other }
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_direct_encrypted_or_bridged_room_is_refused_and_a_plain_room_is_admitted() {
|
||||||
|
let estate = estate(&[]);
|
||||||
|
assert_eq!(admit_room(&RoomFacts { direct: true, ..facts("!a:s") }, &estate), Err(Refusal::Direct));
|
||||||
|
assert_eq!(admit_room(&RoomFacts { encrypted: true, ..facts("!a:s") }, &estate), Err(Refusal::Encrypted));
|
||||||
|
for state_type in ["m.bridge", "uk.half-shot.bridge"] {
|
||||||
|
assert_eq!(admit_room(&RoomFacts { bridge: Some(state_type.into()), ..facts("!a:s") }, &estate), Err(Refusal::Bridged(state_type.into())));
|
||||||
|
}
|
||||||
|
assert_eq!(admit_room(&RoomFacts { alias: Some("#swarm:gamecult.org".into()), name: Some("Swarm".into()), ..facts("!a:s") }, &estate), Ok(()));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn the_estate_is_refused_by_id_and_by_alias_in_any_case_and_a_lookalike_is_not() {
|
||||||
|
let named = estate(&["!ids:s"]);
|
||||||
|
assert!(named.holds_id("!ids:s") && !named.holds_id("!other:s"));
|
||||||
|
assert_eq!(admit_room(&facts("!ids:s"), &named), Err(Refusal::Estate));
|
||||||
|
for alias in ESTATE_ALIASES.iter().map(|alias| alias.to_string()).chain(["#USERS:Gamecult.Org".to_owned()]) {
|
||||||
|
assert_eq!(admit_room(&RoomFacts { alias: Some(alias.clone()), ..facts("!x:s") }, &estate(&[])), Err(Refusal::Estate), "{alias}");
|
||||||
|
assert_eq!(admit_room(&RoomFacts { alt_aliases: vec!["#a:s".into(), alias.clone()], ..facts("!x:s") }, &estate(&[])), Err(Refusal::Estate), "alt {alias}");
|
||||||
|
}
|
||||||
|
for lookalike in ["#users-guide:gamecult.org", "#patrons-club:gamecult.org", "#users:gamecult.org.evil", "#users:other.org"] {
|
||||||
|
assert_eq!(admit_room(&RoomFacts { alias: Some(lookalike.into()), ..facts("!x:s") }, &estate(&[])), Ok(()), "{lookalike}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_bridged_platform_sender_or_content_key_refuses_and_a_lookalike_does_not() {
|
||||||
|
for sender in ["@discord_123:s", "@_discord_9:s", "@__DiscordBot:s", "@discordbot:s", "@mautrix_discord:s", "@_bridge_discordbot:s", "@a_discord:s"] {
|
||||||
|
assert_eq!(refuse_event(&event(sender, &[])), Some(Refusal::DiscordSender), "{sender}");
|
||||||
|
}
|
||||||
|
for key in ["fi.mau.discord", "uk.half-shot.discord", "X.DISCORD.y"] {
|
||||||
|
assert_eq!(refuse_event(&event("@a:s", &["body", key])), Some(Refusal::DiscordContent), "{key}");
|
||||||
|
}
|
||||||
|
for sender in ["@discourse:s", "@disco:s", "@eureka:s", "@x:discord.com"] {
|
||||||
|
assert_eq!(refuse_event(&event(sender, &["discourse", "body"])), None, "{sender}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn the_estate_needs_exactly_its_room_ids() {
|
||||||
|
for ids in [vec![], vec!["!a:s"], vec!["!a:s", "!b:s", "!c:s", "!d:s"], vec!["!a:s", "!b:s", "notaroom"]] {
|
||||||
|
assert!(Estate::new(ids.iter().map(|id| (*id).to_owned()).collect()).is_err(), "{ids:?}");
|
||||||
|
}
|
||||||
|
assert!(Estate::new(["!a:s", "!b:s", "!c:s"].map(str::to_owned).into()).is_ok());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_room_facts_carries_no_join_rule() {
|
||||||
|
let source = include_str!("room_scope.rs");
|
||||||
|
let code = &source[..source.find("#[cfg(test)]").unwrap()];
|
||||||
|
assert!(!code.contains(&["join", "_rule"].concat()));
|
||||||
|
}
|
||||||
|
}
|
||||||
92
crates/huginn-daemon/src/sources/test_http.rs
Normal file
92
crates/huginn-daemon/src/sources/test_http.rs
Normal file
|
|
@ -0,0 +1,92 @@
|
||||||
|
//! The one fake HTTP server the source tests talk to (the forge's and Matrix's):
|
||||||
|
//! it answers by path, holds open a path it has no answer for as a hung server
|
||||||
|
//! does, and records each request's method, path and `Authorization` header.
|
||||||
|
|
||||||
|
use std::io::{Read, Write};
|
||||||
|
use std::sync::{Arc, Mutex};
|
||||||
|
|
||||||
|
use serde_json::json;
|
||||||
|
|
||||||
|
/// What a fake server answers: a status, the `Location` header when there is one, and a body.
|
||||||
|
pub struct Reply {
|
||||||
|
pub status: u16,
|
||||||
|
pub location: Option<String>,
|
||||||
|
pub body: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<String> for Reply {
|
||||||
|
fn from(body: String) -> Self {
|
||||||
|
Self { status: 200, location: None, body }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Reply {
|
||||||
|
/// What the live forge answers an anonymous reader for a private repo.
|
||||||
|
pub fn not_found() -> Self {
|
||||||
|
Self { status: 404, location: None, body: json!({ "message": "The target couldn't be found." }).to_string() }
|
||||||
|
}
|
||||||
|
|
||||||
|
/// What the live forge answers for a renamed repo: a 301 to its public address, `/forge` prefix and all.
|
||||||
|
pub fn moved_to(location: &str) -> Self {
|
||||||
|
Self { status: 301, location: Some(location.to_owned()), body: String::new() }
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn status(status: u16, body: &str) -> Self {
|
||||||
|
Self { status, location: None, body: body.to_owned() }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One request as the server saw it.
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||||
|
pub struct Seen {
|
||||||
|
pub method: String,
|
||||||
|
pub path: String,
|
||||||
|
pub authorization: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Every request a fake server has seen, in order.
|
||||||
|
#[derive(Clone, Default)]
|
||||||
|
pub struct Asked(Arc<Mutex<Vec<Seen>>>);
|
||||||
|
|
||||||
|
impl Asked {
|
||||||
|
pub fn seen(&self) -> Vec<Seen> {
|
||||||
|
self.0.lock().unwrap().clone()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn paths(&self) -> Vec<String> {
|
||||||
|
self.0.lock().unwrap().iter().map(|seen| seen.path.clone()).collect()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Answers each request by its path from `routes`; a path it has no answer for is held open and never answered.
|
||||||
|
pub fn forge_of<R: Into<Reply>>(routes: impl Fn(&str) -> Option<R> + Send + 'static) -> (String, Asked) {
|
||||||
|
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
|
||||||
|
let base = format!("http://{}", listener.local_addr().unwrap());
|
||||||
|
let asked = Asked::default();
|
||||||
|
let record = asked.clone();
|
||||||
|
std::thread::spawn(move || {
|
||||||
|
let mut held = Vec::new();
|
||||||
|
for stream in listener.incoming() {
|
||||||
|
let mut stream = stream.unwrap();
|
||||||
|
let mut buffer = [0_u8; 4096];
|
||||||
|
let read = stream.read(&mut buffer).unwrap();
|
||||||
|
let request = String::from_utf8_lossy(&buffer[..read]).into_owned();
|
||||||
|
let mut words = request.split_whitespace();
|
||||||
|
let method = words.next().unwrap_or_default().to_owned();
|
||||||
|
let path = words.next().unwrap_or_default().to_owned();
|
||||||
|
let authorization = request
|
||||||
|
.lines()
|
||||||
|
.find_map(|line| line.split_once(':').filter(|(name, _)| name.eq_ignore_ascii_case("authorization")))
|
||||||
|
.map(|(_, value)| value.trim().to_owned());
|
||||||
|
record.0.lock().unwrap().push(Seen { method, path: path.clone(), authorization });
|
||||||
|
match routes(&path).map(Into::into) {
|
||||||
|
Some(Reply { status, location, body }) => {
|
||||||
|
let location = location.map(|location| format!("location: {location}\r\n")).unwrap_or_default();
|
||||||
|
write!(stream, "HTTP/1.1 {status} X\r\ncontent-type: application/json\r\n{location}content-length: {}\r\nconnection: close\r\n\r\n{body}", body.len()).unwrap();
|
||||||
|
}
|
||||||
|
None => held.push(stream),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
(base, asked)
|
||||||
|
}
|
||||||
|
|
@ -7,7 +7,7 @@ use std::process::Command;
|
||||||
|
|
||||||
use anyhow::Result;
|
use anyhow::Result;
|
||||||
use huginn_mind::eureka_pipeline::{Line, Short};
|
use huginn_mind::eureka_pipeline::{Line, Short};
|
||||||
use huginn_mind::SourceQuery;
|
use huginn_mind::{ChatHits, ChatQuery, ChatWindow, ChatWindowQuery, MindRefusal, SourceQuery};
|
||||||
|
|
||||||
use super::*;
|
use super::*;
|
||||||
use crate::index::fakes::{FakeEmbedder, FakeSources};
|
use crate::index::fakes::{FakeEmbedder, FakeSources};
|
||||||
|
|
@ -25,23 +25,23 @@ fn git_in(dir: &Path, args: &[&str]) -> String {
|
||||||
}
|
}
|
||||||
|
|
||||||
/// A working repo on disk, committed to by the test.
|
/// A working repo on disk, committed to by the test.
|
||||||
struct Remote {
|
pub(super) struct Remote {
|
||||||
dir: tempfile::TempDir,
|
dir: tempfile::TempDir,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Remote {
|
impl Remote {
|
||||||
fn new() -> Self {
|
pub(super) fn new() -> Self {
|
||||||
let dir = tempfile::tempdir().unwrap();
|
let dir = tempfile::tempdir().unwrap();
|
||||||
git_in(dir.path(), &["init", "--quiet", "-b", "main"]);
|
git_in(dir.path(), &["init", "--quiet", "-b", "main"]);
|
||||||
Self { dir }
|
Self { dir }
|
||||||
}
|
}
|
||||||
|
|
||||||
fn path(&self) -> PathBuf {
|
pub(super) fn path(&self) -> PathBuf {
|
||||||
self.dir.path().to_path_buf()
|
self.dir.path().to_path_buf()
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Writes files, removes paths, commits everything and returns the commit.
|
/// Writes files, removes paths, commits everything and returns the commit.
|
||||||
fn commit(&self, write: &[(&str, &[u8])], remove: &[&str]) -> String {
|
pub(super) fn commit(&self, write: &[(&str, &[u8])], remove: &[&str]) -> String {
|
||||||
for (path, bytes) in write {
|
for (path, bytes) in write {
|
||||||
let full = self.dir.path().join(path);
|
let full = self.dir.path().join(path);
|
||||||
std::fs::create_dir_all(full.parent().unwrap()).unwrap();
|
std::fs::create_dir_all(full.parent().unwrap()).unwrap();
|
||||||
|
|
@ -56,13 +56,13 @@ impl Remote {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
struct FakeForge {
|
pub(super) struct FakeForge {
|
||||||
repos: Vec<(SourceRepo, PathBuf)>,
|
repos: Vec<(SourceRepo, PathBuf)>,
|
||||||
shortfall: Option<String>,
|
shortfall: Option<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl FakeForge {
|
impl FakeForge {
|
||||||
fn of(repos: &[(&str, &Remote)]) -> Self {
|
pub(super) fn of(repos: &[(&str, &Remote)]) -> Self {
|
||||||
Self {
|
Self {
|
||||||
repos: repos
|
repos: repos
|
||||||
.iter()
|
.iter()
|
||||||
|
|
@ -755,3 +755,424 @@ fn a_long_brief_of_eight_topics_reaches_the_chunk_only_the_last_is_near() {
|
||||||
assert!(paths(&found).contains(&"chunk-7.rs"), "{:?}", paths(&found));
|
assert!(paths(&found).contains(&"chunk-7.rs"), "{:?}", paths(&found));
|
||||||
assert_eq!(paths(&found).len(), 8);
|
assert_eq!(paths(&found).len(), 8);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ---- chat search ----
|
||||||
|
|
||||||
|
fn msg(event_id: &str, ts_ms: u64) -> ChatMessage {
|
||||||
|
ChatMessage { event_id: event_id.into(), sender: "@a:gamecult.org".into(), ts_ms, body: format!("body of {event_id}") }
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A chunk in `room` (with `alias`) of messages `(id, ts)`, near `axis`.
|
||||||
|
fn chat_point(room: &str, alias: Option<&str>, axis: usize, dimensions: u32, messages: &[(&str, u64)]) -> ChatPoint {
|
||||||
|
let messages: Vec<ChatMessage> = messages.iter().map(|(id, ts)| msg(id, *ts)).collect();
|
||||||
|
ChatPoint {
|
||||||
|
id: point_id(&format!("{room}{}", messages[0].event_id)),
|
||||||
|
vector: (0..dimensions as usize).map(|at| if at == axis { 1.0 } else { 0.0 }).collect(),
|
||||||
|
payload: ChatPayload { room_id: room.into(), room_alias: alias.map(str::to_owned), messages, text: format!("chunk near topic-{axis}") },
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn chat_query(texts: &[&str], rooms: &[&str], after: Option<u64>, before: Option<u64>, top_k: u32) -> ChatQuery {
|
||||||
|
ChatQuery {
|
||||||
|
texts: texts.iter().map(|text| Line((*text).into())).collect(),
|
||||||
|
rooms: rooms.iter().map(|room| Short((*room).into())).collect(),
|
||||||
|
after_ms: after,
|
||||||
|
before_ms: before,
|
||||||
|
top_k,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A labelled collection that also holds the chat chunks given. Two rooms:
|
||||||
|
/// `!a` (alias `#swarm`), `!b` (no alias).
|
||||||
|
fn chat_rig(points: Vec<ChatPoint>) -> (Rig, TopicEmbedder) {
|
||||||
|
let (mut rig, embedder) = topic_rig(4);
|
||||||
|
rig.index.state.lock().unwrap().points.clear();
|
||||||
|
rig.index.upsert_chat(&points).unwrap();
|
||||||
|
(rig, embedder)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn ids(hits: &ChatHits) -> Vec<&str> {
|
||||||
|
hits.hits.iter().map(|hit| hit.first_event_id.as_str()).collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
const WITHIN: Duration = Duration::from_secs(5);
|
||||||
|
|
||||||
|
fn three_rooms() -> Vec<ChatPoint> {
|
||||||
|
vec![
|
||||||
|
chat_point("!a:x", Some("#swarm:x"), 0, 4, &[("$a1", 100), ("$a2", 200)]),
|
||||||
|
chat_point("!a:x", Some("#swarm:x"), 1, 4, &[("$a3", 300), ("$a4", 400)]),
|
||||||
|
chat_point("!b:x", None, 0, 4, &[("$b1", 150)]),
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_room_filter_by_id_and_by_alias_returns_only_that_room() {
|
||||||
|
let (mut rig, mut embedder) = chat_rig(three_rooms());
|
||||||
|
let by_alias = search_chat(&mut embedder, &mut rig.index, &chat_query(&[], &["#swarm:x"], None, None, 10), WITHIN).unwrap();
|
||||||
|
assert_eq!(ids(&by_alias), ["$a1", "$a3"]);
|
||||||
|
let by_id = search_chat(&mut embedder, &mut rig.index, &chat_query(&[], &["!b:x"], None, None, 10), WITHIN).unwrap();
|
||||||
|
assert_eq!(ids(&by_id), ["$b1"], "a room with no alias is reachable by id");
|
||||||
|
assert_eq!(by_id.hits[0].room_alias, None);
|
||||||
|
let both = search_chat(&mut embedder, &mut rig.index, &chat_query(&[], &["#swarm:x", "!b:x"], None, None, 10), WITHIN).unwrap();
|
||||||
|
assert_eq!(ids(&both), ["$a1", "$b1", "$a3"], "any-of, oldest first");
|
||||||
|
let nobody = search_chat(&mut embedder, &mut rig.index, &chat_query(&[], &["#nobody:x"], None, None, 10), WITHIN).unwrap();
|
||||||
|
assert!(nobody.hits.is_empty() && !nobody.truncated, "an unknown alias is zero hits, not an error");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_time_span_returns_the_overlapping_chunks_and_trims_boundary_messages() {
|
||||||
|
let (mut rig, mut embedder) = chat_rig(three_rooms());
|
||||||
|
let hits = search_chat(&mut embedder, &mut rig.index, &chat_query(&[], &["#swarm:x"], Some(150), Some(350), 10), WITHIN).unwrap();
|
||||||
|
assert_eq!(ids(&hits), ["$a2", "$a3"], "the boundary chunks keep only in-span messages and first ids are recomputed");
|
||||||
|
let (first, second) = (&hits.hits[0], &hits.hits[1]);
|
||||||
|
assert_eq!((first.ts_start_ms, first.ts_end_ms, first.last_event_id.as_str(), first.messages.len()), (200, 200, "$a2", 1));
|
||||||
|
assert_eq!((second.ts_start_ms, second.ts_end_ms, second.last_event_id.as_str(), second.messages.len()), (300, 300, "$a3", 1));
|
||||||
|
let none = search_chat(&mut embedder, &mut rig.index, &chat_query(&[], &["#swarm:x"], Some(1000), None, 10), WITHIN).unwrap();
|
||||||
|
assert!(none.hits.is_empty(), "a chunk wholly outside is absent");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn list_mode_is_oldest_first_truncated_when_more_exist_and_needs_no_embed() {
|
||||||
|
let (mut rig, mut embedder) = chat_rig(three_rooms());
|
||||||
|
let page = search_chat(&mut embedder, &mut rig.index, &chat_query(&[], &[], None, None, 2), WITHIN).unwrap();
|
||||||
|
assert_eq!(ids(&page), ["$a1", "$b1"]);
|
||||||
|
assert!(page.truncated);
|
||||||
|
assert!(embedder.0.state.lock().unwrap().embedded.is_empty(), "no texts, no embed call");
|
||||||
|
// Within one room chunks never overlap, so the last hit's end + 1 continues without a repeat.
|
||||||
|
let first = search_chat(&mut embedder, &mut rig.index, &chat_query(&[], &["!a:x"], None, None, 1), WITHIN).unwrap();
|
||||||
|
assert_eq!((ids(&first), first.truncated), (vec!["$a1"], true));
|
||||||
|
let rest = search_chat(&mut embedder, &mut rig.index, &chat_query(&[], &["!a:x"], Some(first.hits[0].ts_end_ms + 1), None, 1), WITHIN).unwrap();
|
||||||
|
assert_eq!((ids(&rest), rest.truncated), (vec!["$a3"], false));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn texts_rank_by_meaning_and_compose_with_room_and_span() {
|
||||||
|
let (mut rig, mut embedder) = chat_rig(three_rooms());
|
||||||
|
let near = search_chat(&mut embedder, &mut rig.index, &chat_query(&["topic-0"], &[], None, None, 5), WITHIN).unwrap();
|
||||||
|
assert_eq!(near.hits.len(), 3);
|
||||||
|
assert!(near.hits[0].score.unwrap() > near.hits[2].score.unwrap());
|
||||||
|
assert_eq!(near.hits[2].first_event_id, "$a3", "the far chunk is last");
|
||||||
|
let narrowed = search_chat(&mut embedder, &mut rig.index, &chat_query(&["topic-0"], &["!a:x"], Some(250), None, 5), WITHIN).unwrap();
|
||||||
|
assert_eq!(ids(&narrowed), ["$a3"], "a near chunk in another room or outside the span never appears");
|
||||||
|
let fused = search_chat(&mut embedder, &mut rig.index, &chat_query(&["topic-0", "topic-1"], &["!a:x"], None, None, 5), WITHIN).unwrap();
|
||||||
|
assert_eq!(fused.hits.len(), 2);
|
||||||
|
assert!(fused.hits[0].score.unwrap() > 0.0 && fused.hits[0].score != Some(1.0), "two texts carry the fused reciprocal-rank score");
|
||||||
|
assert_eq!(embedder.0.state.lock().unwrap().embedded.last().unwrap().len(), 2, "one embed call for two texts");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn every_hit_cites_room_event_ids_and_timestamps() {
|
||||||
|
let (mut rig, mut embedder) = chat_rig(three_rooms());
|
||||||
|
for texts in [&[][..], &["topic-0"][..]] {
|
||||||
|
let hits = search_chat(&mut embedder, &mut rig.index, &chat_query(texts, &[], None, None, 10), WITHIN).unwrap();
|
||||||
|
assert!(!hits.hits.is_empty());
|
||||||
|
for hit in &hits.hits {
|
||||||
|
assert!(hit.room_id.starts_with('!') && !hit.first_event_id.is_empty() && !hit.last_event_id.is_empty());
|
||||||
|
assert!(hit.ts_start_ms <= hit.ts_end_ms && !hit.messages.is_empty());
|
||||||
|
assert_eq!((hit.messages[0].event_id.as_str(), hit.messages.last().unwrap().event_id.as_str()), (hit.first_event_id.as_str(), hit.last_event_id.as_str()));
|
||||||
|
for message in &hit.messages {
|
||||||
|
assert!(message.event_id.starts_with('$') && message.sender.starts_with('@') && message.ts_ms > 0);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_forge_search_never_returns_a_chat_point_and_a_chat_search_never_a_forge_point() {
|
||||||
|
let (mut rig, mut embedder) = topic_rig(4);
|
||||||
|
rig.index.upsert_chat(&three_rooms()).unwrap();
|
||||||
|
let forge = search(&mut embedder, &mut rig.index, &queries(&["topic-0"], 10), WITHIN).unwrap();
|
||||||
|
assert_eq!(forge.hits.len(), 4, "only the four forge chunks");
|
||||||
|
let chat = search_chat(&mut embedder, &mut rig.index, &chat_query(&["topic-0"], &[], None, None, 10), WITHIN).unwrap();
|
||||||
|
assert_eq!(chat.hits.len(), 3, "only the three chat chunks");
|
||||||
|
let listed = search_chat(&mut embedder, &mut rig.index, &chat_query(&[], &[], None, None, 10), WITHIN).unwrap();
|
||||||
|
assert_eq!(listed.hits.len(), 3);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn an_absent_unlabelled_or_other_model_collection_is_a_loud_refusal_for_chat() {
|
||||||
|
let mut rig = Rig::new();
|
||||||
|
let mut embedder = TopicEmbedder(FakeEmbedder::new("d1"));
|
||||||
|
for texts in [&[][..], &["topic-0"][..]] {
|
||||||
|
let error = search_chat(&mut embedder, &mut rig.index, &chat_query(texts, &[], None, None, 3), WITHIN).unwrap_err();
|
||||||
|
assert_eq!(format!("{error:#}"), "source index not built");
|
||||||
|
}
|
||||||
|
rig.index.plant(DescribedSources::Unlabelled);
|
||||||
|
assert!(search_chat(&mut embedder, &mut rig.index, &chat_query(&[], &[], None, None, 3), WITHIN).is_err());
|
||||||
|
let (mut rig, _) = chat_rig(three_rooms());
|
||||||
|
let mut foreign = TopicEmbedder(FakeEmbedder::new("d2"));
|
||||||
|
let error = format!("{:#}", search_chat(&mut foreign, &mut rig.index, &chat_query(&["topic-0"], &[], None, None, 3), WITHIN).unwrap_err());
|
||||||
|
assert!(error.contains("built under model d1") && error.contains("embeds under model d2"), "{error}");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// An old forge-only collection (labelled with the old corpus) is rebuilt once
|
||||||
|
/// under the new corpus, and a second refresh leaves it.
|
||||||
|
#[test]
|
||||||
|
fn an_old_forge_only_collection_rebuilds_once() {
|
||||||
|
let remote = Remote::new();
|
||||||
|
remote.commit(&[("a.rs", b"one\n")], &[]);
|
||||||
|
let mut rig = Rig::new();
|
||||||
|
let mut forge = FakeForge::of(&[("Demo", &remote)]);
|
||||||
|
rig.converged(&mut forge);
|
||||||
|
let mut old = match rig.index.state.lock().unwrap().described.clone() {
|
||||||
|
Some(DescribedSources::Labelled { meta, cursors }) => (meta, cursors),
|
||||||
|
other => panic!("{other:?}"),
|
||||||
|
};
|
||||||
|
assert_eq!(old.0.corpus, "forge:GameCult+chat:rooms");
|
||||||
|
old.0.corpus = "forge:GameCult".into();
|
||||||
|
rig.index.plant(DescribedSources::Labelled { meta: old.0, cursors: old.1 });
|
||||||
|
let before = rig.index.state.lock().unwrap().recreated;
|
||||||
|
rig.converged(&mut forge);
|
||||||
|
rig.converged(&mut forge);
|
||||||
|
assert_eq!(rig.index.state.lock().unwrap().recreated, before + 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn the_chat_read_path_holds_no_discord_matrix_or_second_fuse() {
|
||||||
|
let source = include_str!("mod.rs");
|
||||||
|
let read = &source[source.find("pub fn search_chat").unwrap()..];
|
||||||
|
for banned in ["discord", "puppet", "bridge", "matrix", "synapse", "access_token"] {
|
||||||
|
assert!(!read.to_ascii_lowercase().contains(banned), "{banned} in the chat read path");
|
||||||
|
}
|
||||||
|
assert_eq!(source.matches("pub fn fuse").count(), 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// An old forge-only collection answers both searches with the stale-index
|
||||||
|
/// error until the rebuild lands, not with an empty page.
|
||||||
|
#[test]
|
||||||
|
fn an_old_corpus_collection_is_refused_until_it_is_rebuilt() {
|
||||||
|
let (mut rig, mut embedder) = chat_rig(three_rooms());
|
||||||
|
let Some(DescribedSources::Labelled { mut meta, cursors }) = rig.index.state.lock().unwrap().described.clone() else { panic!() };
|
||||||
|
meta.corpus = "forge:GameCult".into();
|
||||||
|
rig.index.plant(DescribedSources::Labelled { meta, cursors });
|
||||||
|
let error = format!("{:#}", search(&mut embedder, &mut rig.index, &query("one", 3, None), WITHIN).unwrap_err());
|
||||||
|
assert!(error.contains("stale") && error.contains("forge:GameCult") && error.contains("rebuild is pending"), "{error}");
|
||||||
|
for texts in [&[][..], &["topic-0"][..]] {
|
||||||
|
let error = format!("{:#}", search_chat(&mut embedder, &mut rig.index, &chat_query(texts, &[], None, None, 3), WITHIN).unwrap_err());
|
||||||
|
assert!(error.contains("stale") && error.contains("rebuild is pending"), "{error}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A message exactly on `after` or `before` is in the span, and so is a chunk
|
||||||
|
/// whose end is exactly `after` or whose start is exactly `before`.
|
||||||
|
#[test]
|
||||||
|
fn the_span_bounds_are_inclusive_for_chunks_and_for_messages() {
|
||||||
|
let (mut rig, mut embedder) = chat_rig(three_rooms());
|
||||||
|
let mut hits = |after, before| {
|
||||||
|
let found = search_chat(&mut embedder, &mut rig.index, &chat_query(&[], &["#swarm:x"], after, before, 10), WITHIN).unwrap();
|
||||||
|
found.hits.iter().map(|hit| (hit.first_event_id.clone(), hit.messages.len())).collect::<Vec<_>>()
|
||||||
|
};
|
||||||
|
let owned = |pairs: &[(&str, usize)]| pairs.iter().map(|(id, n)| ((*id).to_owned(), *n)).collect::<Vec<_>>();
|
||||||
|
// Chunk $a1..$a2 ends at 200: after=200 keeps it, trimmed to the message on the bound.
|
||||||
|
assert_eq!(hits(Some(200), None), owned(&[("$a2", 1), ("$a3", 2)]));
|
||||||
|
// Chunk $a3..$a4 starts at 300: before=300 keeps it, trimmed to the message on the bound.
|
||||||
|
assert_eq!(hits(None, Some(300)), owned(&[("$a1", 2), ("$a3", 1)]));
|
||||||
|
// Both bounds on one message.
|
||||||
|
assert_eq!(hits(Some(200), Some(200)), owned(&[("$a2", 1)]));
|
||||||
|
|
||||||
|
let filter = ChatFilter { rooms: vec![], after_ms: Some(200), before_ms: Some(300), ..ChatFilter::default() };
|
||||||
|
assert!(filter.keeps("!a:x", None, 100, 200) && filter.keeps("!a:x", None, 300, 400));
|
||||||
|
assert!(!filter.keeps("!a:x", None, 100, 199) && !filter.keeps("!a:x", None, 301, 400));
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- chat window ----
|
||||||
|
|
||||||
|
fn wq(event_id: &str, before: u32, after: u32) -> ChatWindowQuery {
|
||||||
|
ChatWindowQuery { event_id: Short(event_id.into()), before, after }
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Room `!a`: five two-message chunks, $a0..$a9 at 100..1000. Room `!b`:
|
||||||
|
/// chunks whose times fall between them ($b0 at 450, $b1 at 550).
|
||||||
|
fn window_rig() -> Rig {
|
||||||
|
let mut points: Vec<ChatPoint> = (0..5)
|
||||||
|
.map(|chunk| {
|
||||||
|
let (first, second) = (format!("$a{}", chunk * 2), format!("$a{}", chunk * 2 + 1));
|
||||||
|
let at = chunk as u64 * 200;
|
||||||
|
chat_point("!a:x", Some("#swarm:x"), 0, 4, &[(&first, at + 100), (&second, at + 200)])
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
points.push(chat_point("!b:x", None, 0, 4, &[("$b0", 450), ("$b1", 550)]));
|
||||||
|
chat_rig(points).0
|
||||||
|
}
|
||||||
|
|
||||||
|
fn window_ids(window: &ChatWindow) -> Vec<&str> {
|
||||||
|
window.messages.iter().map(|message| message.event_id.as_str()).collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn refusal_field(answer: Result<Result<ChatWindow, MindRefusal>>) -> (String, String) {
|
||||||
|
match answer.unwrap() {
|
||||||
|
Err(MindRefusal::SelectionInvalid { field, message, .. }) => (field, message),
|
||||||
|
other => panic!("{other:?}"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn an_unknown_event_id_is_refused_naming_the_field_and_the_three_reasons() {
|
||||||
|
let mut rig = window_rig();
|
||||||
|
let (field, message) = refusal_field(window(&mut rig.index, &wq("$nope", 3, 3), WITHIN));
|
||||||
|
assert_eq!(field, "event_id");
|
||||||
|
for reason in ["not be ingested yet", "removed (redacted, or purged", "a room the index does not read"] {
|
||||||
|
assert!(message.contains(reason), "{message}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_redacted_event_is_refused_and_absent_from_its_neighbours_window() {
|
||||||
|
// The refresh re-chunked the window without $a5: the index holds only $a4.
|
||||||
|
let mut rig = chat_rig(vec![chat_point("!a:x", None, 0, 4, &[("$a3", 300), ("$a4", 400)]), chat_point("!a:x", None, 0, 4, &[("$a6", 600)])]).0;
|
||||||
|
assert_eq!(refusal_field(window(&mut rig.index, &wq("$a5", 3, 3), WITHIN)).0, "event_id");
|
||||||
|
let around = window(&mut rig.index, &wq("$a4", 3, 3), WITHIN).unwrap().unwrap();
|
||||||
|
assert_eq!(window_ids(&around), ["$a3", "$a4", "$a6"]);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn an_event_in_two_chunks_is_an_index_fault_not_a_window() {
|
||||||
|
let mut rig = chat_rig(vec![chat_point("!a:x", None, 0, 4, &[("$a1", 100), ("$a2", 200)]), chat_point("!a:x", None, 0, 4, &[("$a2", 200), ("$a3", 300)])]).0;
|
||||||
|
let error = window(&mut rig.index, &wq("$a2", 1, 1), WITHIN).unwrap_err().to_string();
|
||||||
|
assert!(error.contains("$a2") && error.contains("two chunks"), "{error}");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_window_stays_in_the_anchors_room_though_another_room_is_time_adjacent() {
|
||||||
|
let mut rig = window_rig();
|
||||||
|
let around = window(&mut rig.index, &wq("$a4", 4, 4), WITHIN).unwrap().unwrap();
|
||||||
|
assert_eq!(window_ids(&around), ["$a0", "$a1", "$a2", "$a3", "$a4", "$a5", "$a6", "$a7", "$a8"], "no $b between them");
|
||||||
|
assert_eq!((around.room_id.as_str(), around.room_alias.as_deref()), ("!a:x", Some("#swarm:x")));
|
||||||
|
let other = window(&mut rig.index, &wq("$b1", 5, 5), WITHIN).unwrap().unwrap();
|
||||||
|
assert_eq!(window_ids(&other), ["$b0", "$b1"], "the other room's window holds none of !a");
|
||||||
|
assert!(!other.more_before && !other.more_after && other.room_alias.is_none());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_window_across_chunks_has_exactly_before_and_after_in_order_with_the_anchor_once() {
|
||||||
|
let mut rig = window_rig();
|
||||||
|
let around = window(&mut rig.index, &wq("$a4", 3, 3), WITHIN).unwrap().unwrap();
|
||||||
|
assert_eq!(window_ids(&around), ["$a1", "$a2", "$a3", "$a4", "$a5", "$a6", "$a7"]);
|
||||||
|
assert_eq!(around.anchor_event_id, "$a4");
|
||||||
|
assert!(around.more_before && around.more_after);
|
||||||
|
assert!(around.messages.iter().all(|message| !message.event_id.is_empty() && !message.sender.is_empty() && message.ts_ms > 0));
|
||||||
|
let exact = window(&mut rig.index, &wq("$a4", 4, 5), WITHIN).unwrap().unwrap();
|
||||||
|
assert_eq!(window_ids(&exact).len(), 10);
|
||||||
|
assert!(!exact.more_before && !exact.more_after, "nothing remains beyond exactly the room's ends");
|
||||||
|
let alone = window(&mut rig.index, &wq("$a4", 0, 0), WITHIN).unwrap().unwrap();
|
||||||
|
assert_eq!(window_ids(&alone), ["$a4"]);
|
||||||
|
assert!(alone.more_before && alone.more_after);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_window_at_a_rooms_edge_returns_fewer_and_says_nothing_more_lies_beyond() {
|
||||||
|
let mut rig = window_rig();
|
||||||
|
let start = window(&mut rig.index, &wq("$a0", 5, 1), WITHIN).unwrap().unwrap();
|
||||||
|
assert_eq!(window_ids(&start), ["$a0", "$a1"]);
|
||||||
|
assert!(!start.more_before && start.more_after);
|
||||||
|
let end = window(&mut rig.index, &wq("$a9", 1, 5), WITHIN).unwrap().unwrap();
|
||||||
|
assert_eq!(window_ids(&end), ["$a8", "$a9"]);
|
||||||
|
assert!(end.more_before && !end.more_after);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_window_stops_after_six_reads_of_five_chunks_and_says_more_remain() {
|
||||||
|
let points: Vec<ChatPoint> = (0..80).map(|n| chat_point("!a:x", None, 0, 4, &[(&format!("$m{n:02}"), 1000 + n as u64)])).collect();
|
||||||
|
let mut rig = chat_rig(points).0;
|
||||||
|
let around = window(&mut rig.index, &wq("$m70", 50, 0), WITHIN).unwrap().unwrap();
|
||||||
|
assert_eq!(around.messages.len(), 31, "six reads of five new chunks: thirty earlier messages and the anchor");
|
||||||
|
assert_eq!(around.messages[0].event_id, "$m40");
|
||||||
|
assert_eq!(around.messages.last().unwrap().event_id, "$m70");
|
||||||
|
assert!(around.more_before && around.more_after, "the cap is not the room's start, and $m71 follows the anchor");
|
||||||
|
// Asking for exactly what the six reads reach leaves the same honest flag.
|
||||||
|
let exact = window(&mut rig.index, &wq("$m70", 30, 0), WITHIN).unwrap().unwrap();
|
||||||
|
assert_eq!((exact.messages.len(), exact.more_before), (31, true), "thirty held is not yet 'more than asked': the sixth read ended the walk, not the room");
|
||||||
|
let fewer = window(&mut rig.index, &wq("$m70", 29, 0), WITHIN).unwrap().unwrap();
|
||||||
|
assert_eq!((fewer.messages.len(), fewer.more_before), (30, true), "the cut leaves $m40 beyond");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_window_does_not_change_with_the_message_that_anchors_it_when_chunks_share_a_start() {
|
||||||
|
let chunks = [("$t1", "$t2", 150), ("$u1", "$u2", 160), ("$v1", "$v2", 170)];
|
||||||
|
let points: Vec<ChatPoint> = chunks.iter().map(|(first, second, at)| chat_point("!a:x", None, 0, 4, &[(first, 100), (second, *at)])).collect();
|
||||||
|
let mut rig = chat_rig(points).0;
|
||||||
|
let want = ["$t1", "$u1", "$v1", "$t2", "$u2", "$v2"];
|
||||||
|
for anchor in want {
|
||||||
|
let around = window(&mut rig.index, &wq(anchor, 10, 10), WITHIN).unwrap().unwrap();
|
||||||
|
assert_eq!(window_ids(&around), want, "anchored at {anchor}");
|
||||||
|
assert!(!around.more_before && !around.more_after);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_run_of_chunks_at_one_millisecond_is_all_reached() {
|
||||||
|
let points: Vec<ChatPoint> = (0..8).map(|n| chat_point("!a:x", None, 0, 4, &[(&format!("$c{n}"), 100)])).collect();
|
||||||
|
let mut rig = chat_rig(points).0;
|
||||||
|
for anchor in ["$c0", "$c3", "$c7"] {
|
||||||
|
let around = window(&mut rig.index, &wq(anchor, 20, 20), WITHIN).unwrap().unwrap();
|
||||||
|
assert_eq!(window_ids(&around), ["$c0", "$c1", "$c2", "$c3", "$c4", "$c5", "$c6", "$c7"], "anchored at {anchor}");
|
||||||
|
assert!(!around.more_before && !around.more_after, "the room ends in both directions");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_chunk_starting_with_the_anchors_chunk_is_reached_going_forward_alone() {
|
||||||
|
// The anchor's own chunk already holds a message before it, so no earlier read happens.
|
||||||
|
let mut rig = chat_rig(vec![chat_point("!a:x", None, 0, 4, &[("$x0", 100), ("$x1", 120)]), chat_point("!a:x", None, 0, 4, &[("$y0", 100), ("$y1", 130)])]).0;
|
||||||
|
let around = window(&mut rig.index, &wq("$x1", 0, 5), WITHIN).unwrap().unwrap();
|
||||||
|
assert_eq!(window_ids(&around), ["$x1", "$y1"]);
|
||||||
|
assert!(around.more_before && !around.more_after, "$x0 and $y0 precede; nothing follows $y1");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_walk_that_holds_exactly_the_asked_count_still_reads_on_to_learn_the_rooms_end() {
|
||||||
|
let points: Vec<ChatPoint> = (0..10).map(|n| chat_point("!a:x", None, 0, 4, &[(&format!("$m{n}"), 1000 + n as u64)])).collect();
|
||||||
|
let mut rig = chat_rig(points).0;
|
||||||
|
let around = window(&mut rig.index, &wq("$m5", 5, 0), WITHIN).unwrap().unwrap();
|
||||||
|
assert_eq!(window_ids(&around), ["$m0", "$m1", "$m2", "$m3", "$m4", "$m5"]);
|
||||||
|
assert!(!around.more_before, "the room starts at $m0");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn an_event_repeated_in_a_neighbour_chunk_appears_once() {
|
||||||
|
let mut rig = chat_rig(vec![chat_point("!a:x", None, 0, 4, &[("$a1", 100), ("$a2", 200)]), chat_point("!a:x", None, 0, 4, &[("$a2", 200), ("$a3", 300)])]).0;
|
||||||
|
let around = window(&mut rig.index, &wq("$a1", 0, 5), WITHIN).unwrap().unwrap();
|
||||||
|
assert_eq!(window_ids(&around), ["$a1", "$a2", "$a3"]);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_forge_point_is_not_an_event_the_window_can_anchor_on() {
|
||||||
|
let (mut rig, _) = topic_rig(4);
|
||||||
|
let mut state = rig.index.state.lock().unwrap();
|
||||||
|
let mut point = state.points.values().next().unwrap().clone();
|
||||||
|
point.id = "forge-with-an-event-id".into();
|
||||||
|
point.payload.text = "$f1 looks like an event id".into();
|
||||||
|
state.points.insert(point.id.clone(), point);
|
||||||
|
drop(state);
|
||||||
|
let (field, _) = refusal_field(window(&mut rig.index, &wq("$f1", 1, 1), WITHIN));
|
||||||
|
assert_eq!(field, "event_id", "only corpus=chat points are read, as for an unknown id");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_window_asks_the_index_only_and_needs_no_embedder() {
|
||||||
|
let source = include_str!("mod.rs");
|
||||||
|
let signature = &source[source.find("pub fn window").unwrap()..];
|
||||||
|
let signature = &signature[..signature.find('{').unwrap()];
|
||||||
|
assert!(!signature.contains("Embedder"), "{signature}");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A chat cursor and a purge tombstone share the forge's cursors object; the
|
||||||
|
/// forge retire never reads either as a repo name, and still retires a repo.
|
||||||
|
#[test]
|
||||||
|
fn chat_cursor_and_purge_tombstone_keys_survive_the_forge_retire() {
|
||||||
|
let (keep, gone) = (Remote::new(), Remote::new());
|
||||||
|
keep.commit(&[("a.rs", b"a\n")], &[]);
|
||||||
|
gone.commit(&[("b.rs", b"b\n")], &[]);
|
||||||
|
let mut rig = Rig::new();
|
||||||
|
rig.converged(&mut FakeForge::of(&[("Keep", &keep), ("Gone", &gone)]));
|
||||||
|
let mut cursors = rig.index.cursors();
|
||||||
|
cursors.insert("chat:!r:s".into(), "1:$e".into());
|
||||||
|
cursors.insert("chat-purge:!r:s".into(), "1..2".into());
|
||||||
|
rig.index.set_cursors(&cursors).unwrap();
|
||||||
|
rig.index.upsert_chat(&[chat_point("!r:s", None, 0, 4, &[("$e", 5)])]).unwrap();
|
||||||
|
rig.converged(&mut FakeForge::of(&[("Keep", &keep)]));
|
||||||
|
let after = rig.index.cursors();
|
||||||
|
assert_eq!(after.keys().map(String::as_str).collect::<Vec<_>>(), ["Keep", "chat-purge:!r:s", "chat:!r:s"], "Gone retired; both chat keys kept");
|
||||||
|
assert!(rig.index.state.lock().unwrap().deletes.iter().all(|(repo, _)| !repo.starts_with("chat")));
|
||||||
|
assert_eq!(rig.index.state.lock().unwrap().chat.len(), 1);
|
||||||
|
}
|
||||||
|
|
|
||||||
|
|
@ -191,12 +191,18 @@ impl Drop for Daemon {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// A test's name for someone as the Bifrost account id the door requires: a deterministic lowercase D-format UUID.
|
||||||
|
fn acct(name: &str) -> huginn_mind::eureka_pipeline::AccountId {
|
||||||
|
let hash = name.bytes().fold(0xcbf2_9ce4_8422_2325_u64, |hash, byte| (hash ^ u64::from(byte)).wrapping_mul(0x0100_0000_01b3));
|
||||||
|
huginn_mind::eureka_pipeline::AccountId(format!("{:08x}-{:04x}-4000-8000-{:012x}", hash >> 32, (hash >> 16) & 0xffff, hash & 0xffff_ffff_ffff))
|
||||||
|
}
|
||||||
|
|
||||||
fn answer(who: &str, hash: &str, request_id: &str) -> DeskRequest {
|
fn answer(who: &str, hash: &str, request_id: &str) -> DeskRequest {
|
||||||
DeskRequest::Answer {
|
DeskRequest::Answer {
|
||||||
mind: INSTANCE.into(),
|
mind: INSTANCE.into(),
|
||||||
subject: short(who),
|
subject: acct(who),
|
||||||
authority: DeskAuthority::Operator,
|
authority: DeskAuthority::Operator,
|
||||||
deciders: vec![short(who)],
|
deciders: vec![acct(who)],
|
||||||
request: ForkAnswerRequest {
|
request: ForkAnswerRequest {
|
||||||
schema_version: RequestSchema::V0,
|
schema_version: RequestSchema::V0,
|
||||||
fork_id: FORK.into(),
|
fork_id: FORK.into(),
|
||||||
|
|
@ -213,8 +219,8 @@ fn answer(who: &str, hash: &str, request_id: &str) -> DeskRequest {
|
||||||
fn forks_request(who: &str, deciders: &[&str]) -> DeskRequest {
|
fn forks_request(who: &str, deciders: &[&str]) -> DeskRequest {
|
||||||
DeskRequest::Forks {
|
DeskRequest::Forks {
|
||||||
mind: INSTANCE.into(),
|
mind: INSTANCE.into(),
|
||||||
subject: short(who),
|
subject: acct(who),
|
||||||
deciders: deciders.iter().map(|d| short(d)).collect(),
|
deciders: deciders.iter().map(|d| acct(d)).collect(),
|
||||||
status: None,
|
status: None,
|
||||||
cursor: None,
|
cursor: None,
|
||||||
limit: huginn_mind::desk_wire::ForkLimit::new(50).unwrap(),
|
limit: huginn_mind::desk_wire::ForkLimit::new(50).unwrap(),
|
||||||
|
|
@ -277,21 +283,21 @@ fn the_binary_serves_both_doors_and_survives_restart() {
|
||||||
let catalog = format!("{:?}", exchange(&desk, &framed(&catalog_request())).expect("a catalog"));
|
let catalog = format!("{:?}", exchange(&desk, &framed(&catalog_request())).expect("a catalog"));
|
||||||
assert!(catalog.contains("huginn.desk_request.v1") && catalog.contains("huginn.desk_response.v1"), "{catalog}");
|
assert!(catalog.contains("huginn.desk_request.v1") && catalog.contains("huginn.desk_response.v1"), "{catalog}");
|
||||||
assert!(!catalog.contains("huginn.mind_request.v1"), "{catalog}");
|
assert!(!catalog.contains("huginn.mind_request.v1"), "{catalog}");
|
||||||
// The reads answer, store nothing, and carry the whole record to a subject the deciders do not name.
|
// The reads answer and store nothing; a decider reading its own list is told the ids.
|
||||||
let (list, listed_at) = match ask(&desk, "m-r1", &forks_request("x", &["x"])) {
|
let (list, listed_at) = match ask(&desk, "m-r1", &forks_request("x", &["x"])) {
|
||||||
DeskResponse::Forks { list, as_of } => (list, as_of),
|
DeskResponse::Forks { list, as_of } => (list, as_of),
|
||||||
other => panic!("{other:?}"),
|
other => panic!("{other:?}"),
|
||||||
};
|
};
|
||||||
assert_eq!((list.open_count, list.recorded_count, list.forks.len(), list.stamp.clone()), (1, 0, 1, format!("s{listed_at}")));
|
assert_eq!((list.open_count, list.recorded_count, list.forks.len(), list.stamp.clone()), (1, 0, 1, format!("s{listed_at}")));
|
||||||
assert_eq!(list.forks[0].card.options_hash, hash);
|
assert_eq!(list.forks[0].card.options_hash, hash);
|
||||||
let DeskResponse::Fork { entry, as_of } = ask(&desk, "m-r2", &DeskRequest::Fork { mind: INSTANCE.into(), subject: short("x"), deciders: vec![short("x")], fork_id: FORK.into() })
|
let DeskResponse::Fork { entry, as_of } = ask(&desk, "m-r2", &DeskRequest::Fork { mind: INSTANCE.into(), subject: acct("x"), deciders: vec![acct("x")], fork_id: FORK.into() })
|
||||||
else {
|
else {
|
||||||
panic!("a read of the seeded fork")
|
panic!("a read of the seeded fork")
|
||||||
};
|
};
|
||||||
assert_eq!((entry, as_of), (list.forks[0].clone(), listed_at));
|
assert_eq!((entry, as_of), (list.forks[0].clone(), listed_at));
|
||||||
assert_eq!(daemon.documents(), seeded);
|
assert_eq!(daemon.documents(), seeded);
|
||||||
// `authority` is no field of a read: the payload is refused whole and nothing is answered or stored.
|
// `authority` is no field of a read: the payload is refused whole and nothing is answered or stored.
|
||||||
for (operation, request) in [("forks", forks_request("x", &["x"])), ("fork", DeskRequest::Fork { mind: INSTANCE.into(), subject: short("x"), deciders: vec![short("x")], fork_id: FORK.into() })] {
|
for (operation, request) in [("forks", forks_request("x", &["x"])), ("fork", DeskRequest::Fork { mind: INSTANCE.into(), subject: acct("x"), deciders: vec![acct("x")], fork_id: FORK.into() })] {
|
||||||
let mut value = serde_json::to_value(&request).unwrap();
|
let mut value = serde_json::to_value(&request).unwrap();
|
||||||
let variant = if operation == "forks" { "Forks" } else { "Fork" };
|
let variant = if operation == "forks" { "Forks" } else { "Fork" };
|
||||||
value[variant]["authority"] = serde_json::json!("Operator");
|
value[variant]["authority"] = serde_json::json!("Operator");
|
||||||
|
|
@ -319,15 +325,20 @@ fn the_binary_serves_both_doors_and_survives_restart() {
|
||||||
assert_eq!((stale.code, counted_answer_id.as_deref()), (Some(ForkAnswerCode::StaleFork), Some("n1")));
|
assert_eq!((stale.code, counted_answer_id.as_deref()), (Some(ForkAnswerCode::StaleFork), Some("n1")));
|
||||||
let stored = daemon.documents();
|
let stored = daemon.documents();
|
||||||
assert_eq!(stored, seeded + 2);
|
assert_eq!(stored, seeded + 2);
|
||||||
// A subject the call's deciders do not name reads both answers, the first with its choice and the stale one with its code.
|
// A subject the call's deciders do not name is a non-decider: the fork is unlisted, so it reads as a missing one.
|
||||||
|
let outsider = ask(&desk, "m-r3", &DeskRequest::Fork { mind: INSTANCE.into(), subject: acct("outsider"), deciders: vec![acct("y")], fork_id: FORK.into() });
|
||||||
|
assert!(matches!(outsider, DeskResponse::NotStored(ref refusal) if refusal.code == huginn_mind::desk_wire::ForkRefusalCode::ForkUnknown), "{outsider:?}");
|
||||||
|
// A decider reads both answers, the first with its choice and the stale one with its code, and is told who answered.
|
||||||
let DeskResponse::Fork { entry, .. } =
|
let DeskResponse::Fork { entry, .. } =
|
||||||
ask(&desk, "m-r3", &DeskRequest::Fork { mind: INSTANCE.into(), subject: short("outsider"), deciders: vec![short("y")], fork_id: FORK.into() })
|
ask(&desk, "m-r4", &DeskRequest::Fork { mind: INSTANCE.into(), subject: acct("y"), deciders: vec![acct("y")], fork_id: FORK.into() })
|
||||||
else {
|
else {
|
||||||
panic!("a read")
|
panic!("a read")
|
||||||
};
|
};
|
||||||
let seen: Vec<_> = entry.record.answers.iter().map(|answer| (answer.answer_id.as_str(), answer.answerer.as_str(), answer.code)).collect();
|
let seen: Vec<_> = entry.record.answers.iter().map(|answer| (answer.answer_id.as_str(), answer.answerer.as_deref(), answer.code)).collect();
|
||||||
assert_eq!(seen, [("n1", "x", None), ("n2", "x", Some(ForkAnswerCode::StaleFork))]);
|
let x = acct("x").0;
|
||||||
assert_eq!((entry.record.counted_answer_id.as_deref(), entry.card.deciders.as_slice()), (Some("n1"), ["y".to_string()].as_slice()));
|
assert_eq!(seen, [("n1", Some(x.as_str()), None), ("n2", Some(x.as_str()), Some(ForkAnswerCode::StaleFork))]);
|
||||||
|
assert_eq!((entry.record.counted_answer_id.as_deref(), entry.record.answerer_count), (Some("n1"), 1));
|
||||||
|
assert_eq!((entry.card.deciders.as_slice(), entry.card.decider_count), (&[acct("y").0][..], 1));
|
||||||
assert_eq!(daemon.documents(), stored);
|
assert_eq!(daemon.documents(), stored);
|
||||||
|
|
||||||
// A stop takes the socket with it; the stored answers are the retry's memory.
|
// A stop takes the socket with it; the stored answers are the retry's memory.
|
||||||
|
|
@ -579,7 +590,7 @@ fn forks_page(who: &str, cursor: Option<&str>) -> DeskRequest {
|
||||||
}
|
}
|
||||||
|
|
||||||
fn fork_read(who: &str, fork_id: &str) -> DeskRequest {
|
fn fork_read(who: &str, fork_id: &str) -> DeskRequest {
|
||||||
DeskRequest::Fork { mind: INSTANCE.into(), subject: short(who), deciders: vec![short(who)], fork_id: fork_id.into() }
|
DeskRequest::Fork { mind: INSTANCE.into(), subject: acct(who), deciders: vec![acct(who)], fork_id: fork_id.into() }
|
||||||
}
|
}
|
||||||
|
|
||||||
/// The wire spelling of a not-stored refusal's code, and its text.
|
/// The wire spelling of a not-stored refusal's code, and its text.
|
||||||
|
|
|
||||||
|
|
@ -62,12 +62,18 @@ pub(crate) enum Door {
|
||||||
Desk,
|
Desk,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Every staged `desk_answer`, whether the caller sent it or a derivation
|
/// The door split. Every staged `desk_answer`, whether the caller sent it or a derivation produced it,
|
||||||
/// produced it, is refused unless it came by the desk door.
|
/// is refused unless it came by the desk door; and a staged `desk_listing` is refused if it came by the
|
||||||
fn refuse_desk_answers_outside_desk(door: Door, docs: &Docs<'_>) -> Result<(), MindRefusal> {
|
/// desk door: listing is the Mind's own act, on the Admit door, with a receipt saying who.
|
||||||
match docs.batch.iter().find(|staged| staged.kind == PipelineKind::DeskAnswer) {
|
fn refuse_documents_at_the_wrong_door(door: Door, docs: &Docs<'_>) -> Result<(), MindRefusal> {
|
||||||
Some(staged) if door != Door::Desk => Err(MindRefusal::DeskAnswerOutsideDesk { id: staged.key.clone() }),
|
match docs.batch.iter().find(|staged| match staged.kind {
|
||||||
_ => Ok(()),
|
PipelineKind::DeskAnswer => door != Door::Desk,
|
||||||
|
PipelineKind::DeskListing => door == Door::Desk,
|
||||||
|
_ => false,
|
||||||
|
}) {
|
||||||
|
Some(staged) if staged.kind == PipelineKind::DeskListing => Err(MindRefusal::DeskRequestInvalid { field: "desk_listing".into() }),
|
||||||
|
Some(staged) => Err(MindRefusal::DeskAnswerOutsideDesk { id: staged.key.clone() }),
|
||||||
|
None => Ok(()),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -160,7 +166,7 @@ impl<S: MindStore> Mind<S> {
|
||||||
for envelope in envelopes {
|
for envelope in envelopes {
|
||||||
docs.push(stage(envelope, &mind)?)?;
|
docs.push(stage(envelope, &mind)?)?;
|
||||||
}
|
}
|
||||||
refuse_desk_answers_outside_desk(door, &docs)?;
|
refuse_documents_at_the_wrong_door(door, &docs)?;
|
||||||
// A6
|
// A6
|
||||||
let carries_instance = docs.batch.iter().filter(|staged| staged.kind == PipelineKind::Instance).count() == 1;
|
let carries_instance = docs.batch.iter().filter(|staged| staged.kind == PipelineKind::Instance).count() == 1;
|
||||||
if self.is_empty() && !carries_instance {
|
if self.is_empty() && !carries_instance {
|
||||||
|
|
@ -184,7 +190,7 @@ impl<S: MindStore> Mind<S> {
|
||||||
docs.push(stage(envelope, &mind)?)?;
|
docs.push(stage(envelope, &mind)?)?;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
refuse_desk_answers_outside_desk(door, &docs)?;
|
refuse_documents_at_the_wrong_door(door, &docs)?;
|
||||||
for staged in &docs.batch[originals..] {
|
for staged in &docs.batch[originals..] {
|
||||||
resolve(&docs, staged, &mind, external, &mut strong)?;
|
resolve(&docs, staged, &mind, external, &mut strong)?;
|
||||||
}
|
}
|
||||||
|
|
@ -627,6 +633,9 @@ fn check(docs: &Docs<'_>, staged: &Staged, mind: &Slug, import: Option<&ImportSo
|
||||||
}
|
}
|
||||||
// Decided in `answer_desk` before the document exists; no other door can stage one.
|
// Decided in `answer_desk` before the document exists; no other door can stage one.
|
||||||
D::DeskAnswer(_) => Ok(()),
|
D::DeskAnswer(_) => Ok(()),
|
||||||
|
// The question is a citation (`listed`), so a missing one is refused with every other missing referent, and
|
||||||
|
// the key derivation holds it to the listing's own campaign.
|
||||||
|
D::DeskListing(_) => Ok(()),
|
||||||
D::Ruling(ruling) => {
|
D::Ruling(ruling) => {
|
||||||
if ruling.operator_quote.is_some() && ruling.authority != RulingAuthority::Operator {
|
if ruling.operator_quote.is_some() && ruling.authority != RulingAuthority::Operator {
|
||||||
return Err(MindRefusal::QuoteWithoutOperator { ruling: key.into() });
|
return Err(MindRefusal::QuoteWithoutOperator { ruling: key.into() });
|
||||||
|
|
@ -662,7 +671,7 @@ fn check(docs: &Docs<'_>, staged: &Staged, mind: &Slug, import: Option<&ImportSo
|
||||||
return Err(MindRefusal::RepoNotInCampaign { repo: spec.repo.0.clone() });
|
return Err(MindRefusal::RepoNotInCampaign { repo: spec.repo.0.clone() });
|
||||||
}
|
}
|
||||||
for ruling in &spec.rulings {
|
for ruling in &spec.rulings {
|
||||||
if !(historic && root_and_local(&ruling.0).0 != root) && !docs.in_force(K::Ruling, &ruling.0) {
|
if !(docs.in_force(K::Ruling, &ruling.0) || historic && root_and_local(&ruling.0).0 != root) {
|
||||||
return Err(MindRefusal::CitesResolvedDocument { kind: K::Ruling, id: ruling.0.clone() });
|
return Err(MindRefusal::CitesResolvedDocument { kind: K::Ruling, id: ruling.0.clone() });
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -905,6 +914,7 @@ fn matrix(subject: PipelineKind, outcome: &ResolutionOutcome) -> bool {
|
||||||
(K::Target, O::Superseded { by }) => all(by, K::Target),
|
(K::Target, O::Superseded { by }) => all(by, K::Target),
|
||||||
(K::Question, O::Answered { by }) => by.kind == K::Ruling,
|
(K::Question, O::Answered { by }) => by.kind == K::Ruling,
|
||||||
(K::Question, O::Withdrawn { .. }) => true,
|
(K::Question, O::Withdrawn { .. }) => true,
|
||||||
|
(K::DeskListing, O::Withdrawn { .. }) => true,
|
||||||
(K::Ruling, O::Superseded { by }) => all(by, K::Ruling),
|
(K::Ruling, O::Superseded { by }) => all(by, K::Ruling),
|
||||||
(K::CutSpec, O::Superseded { by }) => all(by, K::CutSpec),
|
(K::CutSpec, O::Superseded { by }) => all(by, K::CutSpec),
|
||||||
(K::CutSpec, O::Withdrawn { .. }) => true,
|
(K::CutSpec, O::Withdrawn { .. }) => true,
|
||||||
|
|
|
||||||
|
|
@ -19,6 +19,7 @@
|
||||||
|
|
||||||
use chrono::{DateTime, Duration, SecondsFormat, SubsecRound, Utc};
|
use chrono::{DateTime, Duration, SecondsFormat, SubsecRound, Utc};
|
||||||
use eureka_pipeline::{
|
use eureka_pipeline::{
|
||||||
|
AccountId,
|
||||||
DeskAuthority, DeskOutcome, DeskRefusal, PipelineDeskAnswer, PipelineDocument, PipelineKind, PipelineQuestion, PipelineRefusal,
|
DeskAuthority, DeskOutcome, DeskRefusal, PipelineDeskAnswer, PipelineDocument, PipelineKind, PipelineQuestion, PipelineRefusal,
|
||||||
PipelineResolution, RequestId, ResolutionOutcome, Sha256Hex, Short, Slug, Words,
|
PipelineResolution, RequestId, ResolutionOutcome, Sha256Hex, Short, Slug, Words,
|
||||||
};
|
};
|
||||||
|
|
@ -145,7 +146,7 @@ fn counts_under(answer: &PipelineDeskAnswer, current: &Sha256Hex) -> bool {
|
||||||
/// `answer_desk` runs before it reads the fork and the door runs before its showability gate.
|
/// `answer_desk` runs before it reads the fork and the door runs before its showability gate.
|
||||||
pub(crate) fn bound_answer<'a>(docs: &'a Docs<'_>, answerer: &Short, request_id: &RequestId) -> Option<(&'a str, &'a PipelineDeskAnswer)> {
|
pub(crate) fn bound_answer<'a>(docs: &'a Docs<'_>, answerer: &Short, request_id: &RequestId) -> Option<(&'a str, &'a PipelineDeskAnswer)> {
|
||||||
docs.of_kind(PipelineKind::DeskAnswer).find_map(|(key, document)| match document {
|
docs.of_kind(PipelineKind::DeskAnswer).find_map(|(key, document)| match document {
|
||||||
PipelineDocument::DeskAnswer(held) if held.answerer == *answerer && held.request_id == *request_id => Some((key, held)),
|
PipelineDocument::DeskAnswer(held) if held.answerer.0 == answerer.0 && held.request_id == *request_id => Some((key, held)),
|
||||||
_ => None,
|
_ => None,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
@ -314,7 +315,7 @@ impl<S: MindStore> Mind<S> {
|
||||||
claim_in_future,
|
claim_in_future,
|
||||||
request_id: request.request_id.clone(),
|
request_id: request.request_id.clone(),
|
||||||
body_sha256: body_sha256.clone(),
|
body_sha256: body_sha256.clone(),
|
||||||
answerer: stamp.answerer.clone(),
|
answerer: AccountId(stamp.answerer.0.clone()),
|
||||||
authority: stamp.authority,
|
authority: stamp.authority,
|
||||||
outcome,
|
outcome,
|
||||||
};
|
};
|
||||||
|
|
@ -425,7 +426,7 @@ mod tests {
|
||||||
const LIST: [&str; 6] = ["x", "y", "z", "a", "op", "dec"];
|
const LIST: [&str; 6] = ["x", "y", "z", "a", "op", "dec"];
|
||||||
|
|
||||||
fn stamp_as(who: &str, authority: DeskAuthority, list: &[&str]) -> DeskStamp {
|
fn stamp_as(who: &str, authority: DeskAuthority, list: &[&str]) -> DeskStamp {
|
||||||
DeskStamp { answerer: s(who), authority, deciders: list.iter().map(|id| s(id)).collect() }
|
DeskStamp { answerer: account(who), authority, deciders: list.iter().map(|id| account(id)).collect() }
|
||||||
}
|
}
|
||||||
|
|
||||||
fn decider(who: &str) -> DeskStamp {
|
fn decider(who: &str) -> DeskStamp {
|
||||||
|
|
@ -531,7 +532,7 @@ mod tests {
|
||||||
let mut mind = desk();
|
let mut mind = desk();
|
||||||
stored(ask(&mut mind, &decider("z"), pick(FORK, "Q1", "A", "2026-11-10T12:00:00Z")));
|
stored(ask(&mut mind, &decider("z"), pick(FORK, "Q1", "A", "2026-11-10T12:00:00Z")));
|
||||||
stored(ask(&mut mind, &decider("a"), pick(FORK, "Q1", "A", "2026-11-10T12:00:00Z")));
|
stored(ask(&mut mind, &decider("a"), pick(FORK, "Q1", "A", "2026-11-10T12:00:00Z")));
|
||||||
assert_eq!(held(&mind, &counted(&mind, FORK).unwrap()).answerer, s("z"));
|
assert_eq!(held(&mind, &counted(&mind, FORK).unwrap()).answerer, acct("z"));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|
@ -780,7 +781,7 @@ mod tests {
|
||||||
let record = held(&mind, &answer);
|
let record = held(&mind, &answer);
|
||||||
assert_eq!(record.outcome, outcome);
|
assert_eq!(record.outcome, outcome);
|
||||||
assert_eq!(outcome, DeskOutcome::Refused { code: DeskRefusal::NotADecider, retracts: None });
|
assert_eq!(outcome, DeskOutcome::Refused { code: DeskRefusal::NotADecider, retracts: None });
|
||||||
assert_eq!((record.answerer, record.authority, record.answered_at), (s("guest"), DeskAuthority::Steward, s(at)));
|
assert_eq!((record.answerer, record.authority, record.answered_at), (acct("guest"), DeskAuthority::Steward, s(at)));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|
@ -790,7 +791,7 @@ mod tests {
|
||||||
for (who, authority) in [("op", DeskAuthority::Operator), ("dec", DeskAuthority::Steward), ("x", DeskAuthority::Member)] {
|
for (who, authority) in [("op", DeskAuthority::Operator), ("dec", DeskAuthority::Steward), ("x", DeskAuthority::Member)] {
|
||||||
let (answer, _) = stored(ask(&mut mind, &stamp_as(who, authority, &LIST), pick(FORK, "Q1", "A", at)));
|
let (answer, _) = stored(ask(&mut mind, &stamp_as(who, authority, &LIST), pick(FORK, "Q1", "A", at)));
|
||||||
let record = held(&mind, &answer);
|
let record = held(&mind, &answer);
|
||||||
assert_eq!((record.answerer, record.authority), (s(who), authority));
|
assert_eq!((record.answerer, record.authority), (acct(who), authority));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -936,7 +937,7 @@ mod tests {
|
||||||
assert_eq!(counted(&mind, FORK), Some(answer));
|
assert_eq!(counted(&mind, FORK), Some(answer));
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Libby's vectors (CultLib 6231ecad, `contracts/cultnet/governance/fork-card-vectors.json`, the `optionsHash` array):
|
/// Libby's vectors (CultLib d3025d3c, `contracts/cultnet/governance/fork-card-vectors.json`, the `optionsHash` array):
|
||||||
/// the seven of the binding file, A to G, and the eight she adds. Each is recomputed from its shown members.
|
/// the seven of the binding file, A to G, and the eight she adds. Each is recomputed from its shown members.
|
||||||
fn vectors() -> Vec<serde_json::Value> {
|
fn vectors() -> Vec<serde_json::Value> {
|
||||||
let file: serde_json::Value = serde_json::from_str(include_str!("../vectors/fork-card-vectors.json")).unwrap();
|
let file: serde_json::Value = serde_json::from_str(include_str!("../vectors/fork-card-vectors.json")).unwrap();
|
||||||
|
|
@ -1039,7 +1040,7 @@ mod tests {
|
||||||
let DeskAnswerOutcome::Stored { receipt_id, .. } = outcome else { panic!("{outcome:?}") };
|
let DeskAnswerOutcome::Stored { receipt_id, .. } = outcome else { panic!("{outcome:?}") };
|
||||||
let receipt = mind.receipts().unwrap().into_iter().find(|receipt| receipt.receipt_id == receipt_id).unwrap();
|
let receipt = mind.receipts().unwrap().into_iter().find(|receipt| receipt.receipt_id == receipt_id).unwrap();
|
||||||
assert_eq!(receipt.provenance.faculty, faculty, "{authority:?}");
|
assert_eq!(receipt.provenance.faculty, faculty, "{authority:?}");
|
||||||
assert_eq!(receipt.provenance.agent, s("x"));
|
assert_eq!(receipt.provenance.agent, account("x"));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -1206,10 +1207,10 @@ mod tests {
|
||||||
let image = first.image().unwrap();
|
let image = first.image().unwrap();
|
||||||
let found = Docs::at(image, image.head).find(K::DeskAnswer, &key("Q1", 1)).cloned();
|
let found = Docs::at(image, image.head).find(K::DeskAnswer, &key("Q1", 1)).cloned();
|
||||||
let Some(PipelineDocument::DeskAnswer(mut held)) = found else { panic!("stored") };
|
let Some(PipelineDocument::DeskAnswer(mut held)) = found else { panic!("stored") };
|
||||||
held.answerer = s(&long);
|
held.answerer = eureka_pipeline::AccountId(long.clone());
|
||||||
store.plant(PipelineDocument::DeskAnswer(held).prepare(&cache).unwrap());
|
store.plant(PipelineDocument::DeskAnswer(held).prepare(&cache).unwrap());
|
||||||
let mut mind = opened(store, INSTANCE);
|
let mut mind = opened(store, INSTANCE);
|
||||||
let stamp = stamp_as(&long, DeskAuthority::Member, &LIST);
|
let stamp = DeskStamp { answerer: s(&long), ..stamp_as("x", DeskAuthority::Member, &LIST) };
|
||||||
let before = head(&mind);
|
let before = head(&mind);
|
||||||
assert_eq!(replayed(ask(&mut mind, &stamp, request)), (key("Q1", 1), DeskOutcome::Counted));
|
assert_eq!(replayed(ask(&mut mind, &stamp, request)), (key("Q1", 1), DeskOutcome::Counted));
|
||||||
assert_eq!(head(&mind), before);
|
assert_eq!(head(&mind), before);
|
||||||
|
|
@ -1262,8 +1263,8 @@ mod tests {
|
||||||
assert_eq!(stored(ask(&mut mind, &decider("x"), sent.clone())), (key("Q1", 1), DeskOutcome::Counted));
|
assert_eq!(stored(ask(&mut mind, &decider("x"), sent.clone())), (key("Q1", 1), DeskOutcome::Counted));
|
||||||
let (y_answer, y_outcome) = stored(ask(&mut mind, &decider("y"), sent.clone()));
|
let (y_answer, y_outcome) = stored(ask(&mut mind, &decider("y"), sent.clone()));
|
||||||
assert_eq!((y_answer.as_str(), &y_outcome), (key("Q1", 2).as_str(), &DeskOutcome::Revision { of: s(&key("Q1", 1)) }));
|
assert_eq!((y_answer.as_str(), &y_outcome), (key("Q1", 2).as_str(), &DeskOutcome::Revision { of: s(&key("Q1", 1)) }));
|
||||||
assert_eq!(held(&mind, &y_answer).answerer, s("y"));
|
assert_eq!(held(&mind, &y_answer).answerer, acct("y"));
|
||||||
assert_eq!(held(&mind, &key("Q1", 1)).answerer, s("x"));
|
assert_eq!(held(&mind, &key("Q1", 1)).answerer, acct("x"));
|
||||||
// x's later retry still returns n1; y's returns n2.
|
// x's later retry still returns n1; y's returns n2.
|
||||||
assert_eq!(replayed(ask(&mut mind, &decider("x"), sent.clone())).0, key("Q1", 1));
|
assert_eq!(replayed(ask(&mut mind, &decider("x"), sent.clone())).0, key("Q1", 1));
|
||||||
assert_eq!(replayed(ask(&mut mind, &decider("y"), sent)).0, key("Q1", 2));
|
assert_eq!(replayed(ask(&mut mind, &decider("y"), sent)).0, key("Q1", 2));
|
||||||
|
|
@ -1467,7 +1468,7 @@ mod tests {
|
||||||
let before = head(&mind);
|
let before = head(&mind);
|
||||||
let id = request.request_id.clone();
|
let id = request.request_id.clone();
|
||||||
let (answer, outcome) = stored(ask(&mut mind, &decider("y"), request.clone()));
|
let (answer, outcome) = stored(ask(&mut mind, &decider("y"), request.clone()));
|
||||||
assert_eq!(outcome, DeskOutcome::Refused { code: DeskRefusal::BadRetraction, retracts: sent.2.map(|target| s(target)) }, "{what}");
|
assert_eq!(outcome, DeskOutcome::Refused { code: DeskRefusal::BadRetraction, retracts: sent.2.map(s) }, "{what}");
|
||||||
assert_eq!(head(&mind), before + 1, "{what} is stored");
|
assert_eq!(head(&mind), before + 1, "{what} is stored");
|
||||||
// The record keeps what was sent: the choice, the note and the target it named.
|
// The record keeps what was sent: the choice, the note and the target it named.
|
||||||
let record = held(&mind, &answer);
|
let record = held(&mind, &answer);
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,5 @@
|
||||||
//! The fork desk's answer wire, Huginn's half: Libby's answer-side documents
|
//! The fork desk's answer wire, Huginn's half: Libby's answer-side documents
|
||||||
//! (CultLib `contracts/cultnet/governance/`, the contract file and its vectors at e57fbaee)
|
//! (CultLib `contracts/cultnet/governance/`, the contract file and its vectors at d3025d3c)
|
||||||
//! as Rust types, the one conversion between her `n<seq>` answer ids and the
|
//! as Rust types, the one conversion between her `n<seq>` answer ids and the
|
||||||
//! record keys, the projection of a stored answer, the refusal mapping and the
|
//! record keys, the projection of a stored answer, the refusal mapping and the
|
||||||
//! showability predicate, and `Mind::desk_answer`, the one call that runs them in order.
|
//! showability predicate, and `Mind::desk_answer`, the one call that runs them in order.
|
||||||
|
|
@ -20,7 +20,7 @@ pub use reads::{CardSchema, CardStatus, EntrySchema, ForkCard, ForkCursor, ForkE
|
||||||
|
|
||||||
use chrono::{DateTime, Utc};
|
use chrono::{DateTime, Utc};
|
||||||
use eureka_pipeline::{
|
use eureka_pipeline::{
|
||||||
DeskAuthority, DeskOutcome, DeskRefusal, PipelineDeskAnswer, PipelineDocument, PipelineKind, PipelineQuestion, RequestId, Sha256Hex, Short, Slug, Words,
|
AccountId, DeskAuthority, DeskOutcome, DeskRefusal, PipelineDeskAnswer, PipelineDocument, PipelineKind, PipelineQuestion, RequestId, Sha256Hex, Short, Slug, Words,
|
||||||
};
|
};
|
||||||
use schemars::JsonSchema;
|
use schemars::JsonSchema;
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
@ -147,7 +147,10 @@ pub struct ForkDeskAnswer {
|
||||||
pub answered_at: String,
|
pub answered_at: String,
|
||||||
pub received_at: String,
|
pub received_at: String,
|
||||||
pub claim_in_future: bool,
|
pub claim_in_future: bool,
|
||||||
pub answerer: String,
|
/// The answering account, or nil in a non-decider's view (the key is kept). A stored answer always has one.
|
||||||
|
#[serde(deserialize_with = "required")]
|
||||||
|
#[schemars(required)]
|
||||||
|
pub answerer: Option<String>,
|
||||||
pub authority: DeskAuthority,
|
pub authority: DeskAuthority,
|
||||||
pub request_id: String,
|
pub request_id: String,
|
||||||
}
|
}
|
||||||
|
|
@ -180,12 +183,12 @@ pub struct ForkRefusal {
|
||||||
/// `Refused` is any other Mind refusal; an `Unavailable` is Bifrost's to turn into its 503, and
|
/// `Refused` is any other Mind refusal; an `Unavailable` is Bifrost's to turn into its 503, and
|
||||||
/// Huginn writes no document for it.
|
/// Huginn writes no document for it.
|
||||||
///
|
///
|
||||||
/// The trust boundary of the reads (ruling `desk-read-visibility-whole-record`): `Forks` and `Fork`
|
/// The trust boundary of the reads (ruling `desk-read-visibility-withheld-ids`): `Forks` and `Fork`
|
||||||
/// carry the WHOLE record, every answer's note included, to whatever the peer-checked socket
|
/// carry every answer's note, to whatever the peer-checked socket carries; Bifrost strips notes
|
||||||
/// carries. Huginn strips nothing and gates nothing; Bifrost applies visibility at its door (a
|
/// at its door (a stripped read serves `note: null`, CultLib `fork-card.md`). Huginn itself
|
||||||
/// stripped read serves `note: null`, CultLib `fork-card.md`). Huginn must therefore be reachable
|
/// withholds the account ids from a reader who is not a decider (`deciders` empty, `answerer` nil,
|
||||||
/// only by Bifrost's peer-checked socket: the peer-uid check IS the whole authentication, and a
|
/// counts in their place). Huginn must be reachable only by Bifrost's peer-checked socket: the
|
||||||
/// second reader of that socket is a note leak.
|
/// peer-uid check IS the whole authentication, and a second reader of that socket is a note leak.
|
||||||
// One value per call, built and encoded once; a box would only add a deref to every reader.
|
// One value per call, built and encoded once; a box would only add a deref to every reader.
|
||||||
#[allow(clippy::large_enum_variant)]
|
#[allow(clippy::large_enum_variant)]
|
||||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||||
|
|
@ -229,18 +232,33 @@ pub const DESK_PAGE_BUDGET: usize = DESK_MAX_RESPONSE_BYTES - DESK_ENVELOPE_MARG
|
||||||
/// Most deciders one call carries (CultLib `card.deciders` `maxItems`).
|
/// Most deciders one call carries (CultLib `card.deciders` `maxItems`).
|
||||||
pub const DESK_DECIDERS_MAX: usize = 64;
|
pub const DESK_DECIDERS_MAX: usize = 64;
|
||||||
|
|
||||||
/// A decider list, read whole or not at all: one to `DESK_DECIDERS_MAX` subjects, none twice. A
|
/// An account id read from the wire: a lowercase D-format UUID, or the decoder's refusal, which names the
|
||||||
/// bad list is the decoder's refusal, which names the field and never quotes a subject.
|
/// field and never quotes the value.
|
||||||
fn deciders<'de, D: serde::Deserializer<'de>>(deserializer: D) -> Result<Vec<Short>, D::Error> {
|
fn account_id<'de, D: serde::Deserializer<'de>>(deserializer: D) -> Result<AccountId, D::Error> {
|
||||||
|
let id = AccountId(String::deserialize(deserializer)?);
|
||||||
|
id.validate_account().map_err(|_| serde::de::Error::custom("not an account id: a lowercase D-format UUID"))?;
|
||||||
|
Ok(id)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A decider list, read whole or not at all: one to `DESK_DECIDERS_MAX` account ids, none twice. A
|
||||||
|
/// bad list is the decoder's refusal, which names the field and never quotes an id.
|
||||||
|
fn deciders<'de, D: serde::Deserializer<'de>>(deserializer: D) -> Result<Vec<AccountId>, D::Error> {
|
||||||
struct List;
|
struct List;
|
||||||
|
/// One list entry, through the same grammar as `subject`.
|
||||||
|
struct Account(AccountId);
|
||||||
|
impl<'de> Deserialize<'de> for Account {
|
||||||
|
fn deserialize<D: serde::Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
|
||||||
|
account_id(deserializer).map(Self)
|
||||||
|
}
|
||||||
|
}
|
||||||
impl<'de> serde::de::Visitor<'de> for List {
|
impl<'de> serde::de::Visitor<'de> for List {
|
||||||
type Value = Vec<Short>;
|
type Value = Vec<AccountId>;
|
||||||
fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
|
fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
|
||||||
formatter.write_str("a list of distinct deciders")
|
formatter.write_str("a list of distinct deciders")
|
||||||
}
|
}
|
||||||
fn visit_seq<A: serde::de::SeqAccess<'de>>(self, mut seq: A) -> Result<Vec<Short>, A::Error> {
|
fn visit_seq<A: serde::de::SeqAccess<'de>>(self, mut seq: A) -> Result<Vec<AccountId>, A::Error> {
|
||||||
let mut list = Vec::new();
|
let mut list = Vec::new();
|
||||||
while let Some(subject) = seq.next_element::<Short>()? {
|
while let Some(Account(subject)) = seq.next_element::<Account>()? {
|
||||||
if list.len() == DESK_DECIDERS_MAX {
|
if list.len() == DESK_DECIDERS_MAX {
|
||||||
return Err(serde::de::Error::custom("too many deciders"));
|
return Err(serde::de::Error::custom("too many deciders"));
|
||||||
}
|
}
|
||||||
|
|
@ -261,7 +279,7 @@ fn deciders<'de, D: serde::Deserializer<'de>>(deserializer: D) -> Result<Vec<Sho
|
||||||
fn deciders_schema(generator: &mut schemars::SchemaGenerator) -> schemars::Schema {
|
fn deciders_schema(generator: &mut schemars::SchemaGenerator) -> schemars::Schema {
|
||||||
schemars::json_schema!({
|
schemars::json_schema!({
|
||||||
"type": "array",
|
"type": "array",
|
||||||
"items": generator.subschema_for::<Short>(),
|
"items": generator.subschema_for::<AccountId>(),
|
||||||
"minItems": 1,
|
"minItems": 1,
|
||||||
"maxItems": DESK_DECIDERS_MAX,
|
"maxItems": DESK_DECIDERS_MAX,
|
||||||
"uniqueItems": true
|
"uniqueItems": true
|
||||||
|
|
@ -275,31 +293,33 @@ fn deciders_schema(generator: &mut schemars::SchemaGenerator) -> schemars::Schem
|
||||||
/// the set refuses the whole payload.
|
/// the set refuses the whole payload.
|
||||||
///
|
///
|
||||||
/// The reads, `Forks` and `Fork`, carry NO `authority` (CultLib `list_request` and
|
/// The reads, `Forks` and `Fork`, carry NO `authority` (CultLib `list_request` and
|
||||||
/// `read_request` are closed and have none; one that names it is refused whole). Their `subject`
|
/// `read_request` are closed and have none; one that names it is refused whole). Their `deciders`
|
||||||
/// and `deciders` gate nothing: `deciders` only fills the card's `deciders` and the showability
|
/// feed the showability gate, and `subject` decides the view: a subject in `deciders` is a decider,
|
||||||
/// gate, `subject` is read by no read. `Forks` takes the contract's `list_request` fields flat
|
/// sees every showable fork and is told the ids; any other reads only the forks the Mind lists
|
||||||
/// beside them (`status`, `cursor`, `limit`), each key written, `null` where absent. Huginn
|
/// (a `desk_listing` in force) and is told counts in the ids' place. `Forks` takes the contract's `list_request` fields flat
|
||||||
/// answers every read with the whole record and Bifrost applies visibility; see `DeskResponse`
|
/// beside them (`status`, `cursor`, `limit`), each key written, `null` where absent. Bifrost
|
||||||
/// for the trust boundary. These docs and the doc of `Mind::desk_forks`/`Mind::desk_fork` are the
|
/// strips notes; see `DeskResponse` for the trust boundary. These docs and the doc of `Mind::desk_forks`/`Mind::desk_fork` are the
|
||||||
/// reads spec.
|
/// reads spec.
|
||||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||||
#[serde(deny_unknown_fields)]
|
#[serde(deny_unknown_fields)]
|
||||||
pub enum DeskRequest {
|
pub enum DeskRequest {
|
||||||
Answer {
|
Answer {
|
||||||
mind: Slug,
|
mind: Slug,
|
||||||
subject: Short,
|
#[serde(deserialize_with = "account_id")]
|
||||||
|
subject: AccountId,
|
||||||
authority: DeskAuthority,
|
authority: DeskAuthority,
|
||||||
#[serde(deserialize_with = "deciders")]
|
#[serde(deserialize_with = "deciders")]
|
||||||
#[schemars(schema_with = "deciders_schema")]
|
#[schemars(schema_with = "deciders_schema")]
|
||||||
deciders: Vec<Short>,
|
deciders: Vec<AccountId>,
|
||||||
request: ForkAnswerRequest,
|
request: ForkAnswerRequest,
|
||||||
},
|
},
|
||||||
Forks {
|
Forks {
|
||||||
mind: Slug,
|
mind: Slug,
|
||||||
subject: Short,
|
#[serde(deserialize_with = "account_id")]
|
||||||
|
subject: AccountId,
|
||||||
#[serde(deserialize_with = "deciders")]
|
#[serde(deserialize_with = "deciders")]
|
||||||
#[schemars(schema_with = "deciders_schema")]
|
#[schemars(schema_with = "deciders_schema")]
|
||||||
deciders: Vec<Short>,
|
deciders: Vec<AccountId>,
|
||||||
#[serde(deserialize_with = "required")]
|
#[serde(deserialize_with = "required")]
|
||||||
#[schemars(required)]
|
#[schemars(required)]
|
||||||
status: Option<ListStatus>,
|
status: Option<ListStatus>,
|
||||||
|
|
@ -310,10 +330,11 @@ pub enum DeskRequest {
|
||||||
},
|
},
|
||||||
Fork {
|
Fork {
|
||||||
mind: Slug,
|
mind: Slug,
|
||||||
subject: Short,
|
#[serde(deserialize_with = "account_id")]
|
||||||
|
subject: AccountId,
|
||||||
#[serde(deserialize_with = "deciders")]
|
#[serde(deserialize_with = "deciders")]
|
||||||
#[schemars(schema_with = "deciders_schema")]
|
#[schemars(schema_with = "deciders_schema")]
|
||||||
deciders: Vec<Short>,
|
deciders: Vec<AccountId>,
|
||||||
fork_id: String,
|
fork_id: String,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
@ -335,7 +356,7 @@ impl DeskRequest {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn deciders(&self) -> &[Short] {
|
pub fn deciders(&self) -> &[AccountId] {
|
||||||
match self {
|
match self {
|
||||||
Self::Answer { deciders, .. } | Self::Forks { deciders, .. } | Self::Fork { deciders, .. } => deciders,
|
Self::Answer { deciders, .. } | Self::Forks { deciders, .. } | Self::Fork { deciders, .. } => deciders,
|
||||||
}
|
}
|
||||||
|
|
@ -459,7 +480,7 @@ fn project(key: &str, stored: &PipelineDeskAnswer) -> Result<ForkDeskAnswer, Min
|
||||||
answered_at: stored.answered_at.0.clone(),
|
answered_at: stored.answered_at.0.clone(),
|
||||||
received_at: stored.received_at.0.clone(),
|
received_at: stored.received_at.0.clone(),
|
||||||
claim_in_future: stored.claim_in_future,
|
claim_in_future: stored.claim_in_future,
|
||||||
answerer: stored.answerer.0.clone(),
|
answerer: Some(stored.answerer.0.clone()),
|
||||||
authority: stored.authority,
|
authority: stored.authority,
|
||||||
request_id: stored.request_id.0.clone(),
|
request_id: stored.request_id.0.clone(),
|
||||||
})
|
})
|
||||||
|
|
@ -528,10 +549,13 @@ impl<S: MindStore> Mind<S> {
|
||||||
};
|
};
|
||||||
let image = self.image()?;
|
let image = self.image()?;
|
||||||
let docs = Docs::at(image, image.head);
|
let docs = Docs::at(image, image.head);
|
||||||
// A request already stored replays whatever the list is now: the lookup is `answer_desk`'s,
|
// A fork this reader may not see is unshowable here too, by the read side's own `visible`: the answer path
|
||||||
// asked here only so the gate does not shadow it.
|
// must not tell a hidden fork from a missing one, nor store a refusal for it. A request already stored replays
|
||||||
let unshowable = bound_answer(&docs, &subject, &sent.request_id).is_none()
|
// whatever the decider list is now (the lookup is `answer_desk`'s, asked here only so `showable` does not
|
||||||
&& matches!(docs.find(PipelineKind::Question, &sent.fork.0), Some(PipelineDocument::Question(question)) if !showable(question, deciders));
|
// shadow it), but never to a reader who can no longer see the fork: the counted answer id and `as_of` it
|
||||||
|
// would carry are the fork's current state, so that replay is `fork_unknown` too (ruling clause 3).
|
||||||
|
let replay = bound_answer(&docs, &subject, &sent.request_id).is_some();
|
||||||
|
let unshowable = matches!(docs.find(PipelineKind::Question, &sent.fork.0), Some(PipelineDocument::Question(question)) if (!replay && !showable(question, deciders)) || !reads::visible(&docs, &sent.fork.0, deciders.contains(&subject)));
|
||||||
if unshowable {
|
if unshowable {
|
||||||
return Ok(DeskResponse::NotStored(fork_unknown()));
|
return Ok(DeskResponse::NotStored(fork_unknown()));
|
||||||
}
|
}
|
||||||
|
|
@ -545,7 +569,7 @@ impl<S: MindStore> Mind<S> {
|
||||||
let Some(PipelineDocument::DeskAnswer(stored)) = docs.find(PipelineKind::DeskAnswer, &key) else { return Err(unreadable_key()) };
|
let Some(PipelineDocument::DeskAnswer(stored)) = docs.find(PipelineKind::DeskAnswer, &key) else { return Err(unreadable_key()) };
|
||||||
let fork = stored.fork.0.as_str();
|
let fork = stored.fork.0.as_str();
|
||||||
let counted_answer_id = counted_answer(&docs, fork).map(|counted| answer_seq(fork, &counted).map(n_text).ok_or_else(unreadable_key)).transpose()?;
|
let counted_answer_id = counted_answer(&docs, fork).map(|counted| answer_seq(fork, &counted).map(n_text).ok_or_else(unreadable_key)).transpose()?;
|
||||||
Ok((DeskResponse::Answered { answer: project(&key, stored)?, counted_answer_id, as_of: image.head }))
|
Ok(DeskResponse::Answered { answer: project(&key, stored)?, counted_answer_id, as_of: image.head })
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -566,7 +590,7 @@ mod tests {
|
||||||
const LIST: [&str; 4] = ["x", "y", "z", "op"];
|
const LIST: [&str; 4] = ["x", "y", "z", "op"];
|
||||||
const AT: &str = "2026-11-10T12:00:00Z";
|
const AT: &str = "2026-11-10T12:00:00Z";
|
||||||
|
|
||||||
// ---- Libby's vectors: CultLib 6231ecad, `contracts/cultnet/governance/fork-card-vectors.json` ----
|
// ---- Libby's vectors: CultLib d3025d3c, `contracts/cultnet/governance/fork-card-vectors.json` ----
|
||||||
|
|
||||||
fn vectors() -> Value {
|
fn vectors() -> Value {
|
||||||
serde_json::from_str(include_str!("../vectors/fork-card-vectors.json")).unwrap()
|
serde_json::from_str(include_str!("../vectors/fork-card-vectors.json")).unwrap()
|
||||||
|
|
@ -595,6 +619,11 @@ mod tests {
|
||||||
fn desk() -> Mind<MemoryStore> {
|
fn desk() -> Mind<MemoryStore> {
|
||||||
let mut mind = seeded();
|
let mut mind = seeded();
|
||||||
committed(admit(&mut mind, vec![question_n(1), question_n(2)]));
|
committed(admit(&mut mind, vec![question_n(1), question_n(2)]));
|
||||||
|
// Listed, so a reader who is not a decider reaches them (an unlisted fork is the reads' own tests').
|
||||||
|
for fork in [FORK, FORK2] {
|
||||||
|
let listing = D::DeskListing(eureka_pipeline::PipelineDeskListing { campaign: slug("eureka-state"), listed: s(fork), reason: "Listed.".into() });
|
||||||
|
committed(admit(&mut mind, vec![listing]));
|
||||||
|
}
|
||||||
mind
|
mind
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -630,14 +659,14 @@ mod tests {
|
||||||
}
|
}
|
||||||
|
|
||||||
fn deciders(list: &[&str]) -> Vec<Short> {
|
fn deciders(list: &[&str]) -> Vec<Short> {
|
||||||
list.iter().map(|who| s(who)).collect()
|
list.iter().map(|who| account(who)).collect()
|
||||||
}
|
}
|
||||||
|
|
||||||
/// The response and whether the call committed anything: a replay, a refused body and a gate
|
/// The response and whether the call committed anything: a replay, a refused body and a gate
|
||||||
/// refusal store nothing, so the head does not move.
|
/// refusal store nothing, so the head does not move.
|
||||||
fn ask_as(mind: &mut Mind<MemoryStore>, who: &str, list: &[&str], request: ForkAnswerRequest) -> (DeskResponse, bool) {
|
fn ask_as(mind: &mut Mind<MemoryStore>, who: &str, list: &[&str], request: ForkAnswerRequest) -> (DeskResponse, bool) {
|
||||||
let before = head(mind);
|
let before = head(mind);
|
||||||
let response = mind.desk_answer(&deciders(list), s(who), DeskAuthority::Member, request, now());
|
let response = mind.desk_answer(&deciders(list), account(who), DeskAuthority::Member, request, now());
|
||||||
(response, head(mind) != before)
|
(response, head(mind) != before)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -776,7 +805,8 @@ mod tests {
|
||||||
claim_in_future: message["claim_in_future"].as_bool().unwrap(),
|
claim_in_future: message["claim_in_future"].as_bool().unwrap(),
|
||||||
request_id: RequestId(text("request_id").unwrap()),
|
request_id: RequestId(text("request_id").unwrap()),
|
||||||
body_sha256: Sha256Hex("0".repeat(64)),
|
body_sha256: Sha256Hex("0".repeat(64)),
|
||||||
answerer: Short(text("answerer").unwrap()),
|
// A withheld vector names no answerer: store a placeholder, the vector is the non-decider's view of it.
|
||||||
|
answerer: AccountId(text("answerer").unwrap_or_else(|| "e0d1c2b3-a495-4687-8879-6a5b4c3d2e1f".into())),
|
||||||
authority,
|
authority,
|
||||||
outcome,
|
outcome,
|
||||||
};
|
};
|
||||||
|
|
@ -787,10 +817,14 @@ mod tests {
|
||||||
fn stored_answers_project_to_the_desk_answer_vectors() {
|
fn stored_answers_project_to_the_desk_answer_vectors() {
|
||||||
let file = vectors();
|
let file = vectors();
|
||||||
let valid = rows(&file["deskAnswer"]["valid"]);
|
let valid = rows(&file["deskAnswer"]["valid"]);
|
||||||
assert_eq!(valid.len(), 5);
|
assert_eq!(valid.len(), 8);
|
||||||
for (name, message, bytes) in &valid {
|
for (name, message, bytes) in &valid {
|
||||||
let (key, stored) = stored_from(message);
|
let (key, stored) = stored_from(message);
|
||||||
let projected = project(&key, &stored).unwrap();
|
let mut projected = project(&key, &stored).unwrap();
|
||||||
|
// A null answerer is a non-decider's view of the stored answer.
|
||||||
|
if message["answerer"].is_null() {
|
||||||
|
projected.answerer = None;
|
||||||
|
}
|
||||||
assert_eq!(&encode(&projected), bytes, "{name}");
|
assert_eq!(&encode(&projected), bytes, "{name}");
|
||||||
}
|
}
|
||||||
// Each answer inside her results is the projection of an equivalent record too.
|
// Each answer inside her results is the projection of an equivalent record too.
|
||||||
|
|
@ -1279,6 +1313,54 @@ mod tests {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Ruling clause 3: a stored request replays whatever the decider list is now, but never to a reader who can no
|
||||||
|
/// longer see the fork. The replay would carry the fork's current counted answer id and head, so it is the same
|
||||||
|
/// bytes a missing fork answers with; the stored word itself comes back once the fork is listed again.
|
||||||
|
#[test]
|
||||||
|
fn a_stored_answer_does_not_replay_to_a_reader_who_can_no_longer_see_the_fork() {
|
||||||
|
let mut mind = desk();
|
||||||
|
let hash = hash_of(&mind, FORK);
|
||||||
|
// Dan is not a decider: his answer to the listed fork is a stored refusal and counts nothing.
|
||||||
|
let dan_request = wire(FORK, &hash);
|
||||||
|
let (first, wrote) = ask(&mut mind, "d", dan_request.clone());
|
||||||
|
let (refused, counted, _) = answered(first);
|
||||||
|
assert!((wrote, refused.outcome.as_str(), refused.code.is_some(), counted) == (true, "Refused", true, None), "{refused:?}");
|
||||||
|
// A decider answers, so the fork now has a counted answer a replay could name.
|
||||||
|
let x_request = wire(FORK, &hash);
|
||||||
|
let (counted_by_x, _) = ask(&mut mind, "x", x_request.clone());
|
||||||
|
assert!(answered(counted_by_x).1.is_some());
|
||||||
|
// Still listed: the replay is the stored word, with the fork's current state.
|
||||||
|
let before = head(&mind);
|
||||||
|
let (listed_replay, wrote) = ask(&mut mind, "d", dan_request.clone());
|
||||||
|
assert!(!wrote);
|
||||||
|
let (_, counted, as_of) = answered(listed_replay);
|
||||||
|
assert_eq!((counted.is_some(), as_of), (true, before));
|
||||||
|
// Unlisted: the replay is fork_unknown, byte for byte a fork that does not exist, and writes nothing.
|
||||||
|
let withdrawal = crate::fixtures::resolution_n(
|
||||||
|
eureka_pipeline::PipelineRef { kind: PipelineKind::DeskListing, id: s("eureka-state:desk_listing:Q1") },
|
||||||
|
1,
|
||||||
|
eureka_pipeline::ResolutionOutcome::Withdrawn { reason: "unlisted".into() },
|
||||||
|
);
|
||||||
|
let withdrawal_key = eureka_pipeline::pipeline_key(&withdrawal).unwrap();
|
||||||
|
committed(admit(&mut mind, vec![withdrawal]));
|
||||||
|
let before = head(&mind);
|
||||||
|
let (hidden, wrote) = ask(&mut mind, "d", dan_request.clone());
|
||||||
|
assert!(!wrote);
|
||||||
|
let (missing, _) = ask(&mut mind, "d", ForkAnswerRequest { fork_id: "eureka-state:question:Q99".into(), ..wire(FORK, &hash) });
|
||||||
|
assert_eq!(not_stored(hidden.clone()).code, ForkRefusalCode::ForkUnknown);
|
||||||
|
assert_eq!(encode(&hidden), encode(&missing));
|
||||||
|
// A decider is told the fork whether or not it is listed.
|
||||||
|
let (decider_replay, wrote) = ask(&mut mind, "x", x_request);
|
||||||
|
assert!(!wrote && answered(decider_replay).1.is_some());
|
||||||
|
assert_eq!(head(&mind), before);
|
||||||
|
// Relisted: the stored word replays again.
|
||||||
|
let subject = eureka_pipeline::PipelineRef { kind: PipelineKind::Resolution, id: s(&withdrawal_key) };
|
||||||
|
committed(admit(&mut mind, vec![crate::fixtures::resolution(subject, eureka_pipeline::ResolutionOutcome::Withdrawn { reason: "listed again".into() })]));
|
||||||
|
let (relisted, wrote) = ask(&mut mind, "d", dan_request);
|
||||||
|
assert!(!wrote);
|
||||||
|
assert_eq!(answered(relisted).0.outcome, "Refused");
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn showable_needs_two_options_text_depends_and_a_decider() {
|
fn showable_needs_two_options_text_depends_and_a_decider() {
|
||||||
let D::Question(plain) = question("Q9", &["A", "B"], "A") else { unreachable!() };
|
let D::Question(plain) = question("Q9", &["A", "B"], "A") else { unreachable!() };
|
||||||
|
|
@ -1348,9 +1430,9 @@ mod tests {
|
||||||
let (other, committed) = ask(&mut mind, "y", request.clone());
|
let (other, committed) = ask(&mut mind, "y", request.clone());
|
||||||
let (other, _, _) = answered(other);
|
let (other, _, _) = answered(other);
|
||||||
assert!(committed);
|
assert!(committed);
|
||||||
assert_eq!((other.answer_id.as_str(), other.answerer.as_str(), other.outcome.as_str(), other.request_id.as_str()), ("n2", "y", "Revision", request.request_id.as_str()));
|
assert_eq!((other.answer_id.as_str(), other.answerer.as_deref().unwrap(), other.outcome.as_str(), other.request_id.as_str()), ("n2", account("y").0.as_str(), "Revision", request.request_id.as_str()));
|
||||||
let (original, _, _) = answered(ask(&mut mind, "x", request.clone()).0);
|
let (original, _, _) = answered(ask(&mut mind, "x", request.clone()).0);
|
||||||
assert_eq!((original.answer_id.as_str(), original.answerer.as_str()), ("n1", "x"));
|
assert_eq!((original.answer_id.as_str(), original.answerer.as_deref().unwrap()), ("n1", account("x").0.as_str()));
|
||||||
// V17 and V18: an id that is no lowercase D-format UUID is body_invalid naming request_id, and binds nothing.
|
// V17 and V18: an id that is no lowercase D-format UUID is body_invalid naming request_id, and binds nothing.
|
||||||
let before = head(&mind);
|
let before = head(&mind);
|
||||||
for bad in ["retry-1", "0000000A-0000-4000-8000-00000000000B", "{00000000-0000-4000-8000-000000000001}", "00000000-0000-4000-8000-0000000000010", "", " 00000000-0000-4000-8000-000000000001", "00000000-0000-4000-8000-000000000001 ", "00000000-0000-4000-8000-000000000001\n", "00000000-0000-4000-8000-000000000001x"] {
|
for bad in ["retry-1", "0000000A-0000-4000-8000-00000000000B", "{00000000-0000-4000-8000-000000000001}", "00000000-0000-4000-8000-0000000000010", "", " 00000000-0000-4000-8000-000000000001", "00000000-0000-4000-8000-000000000001 ", "00000000-0000-4000-8000-000000000001\n", "00000000-0000-4000-8000-000000000001x"] {
|
||||||
|
|
@ -1465,6 +1547,7 @@ mod tests {
|
||||||
assert_eq!(shown_hash(&question).0, her_hash, "{name}: the card's hash");
|
assert_eq!(shown_hash(&question).0, her_hash, "{name}: the card's hash");
|
||||||
let mut mind = seeded();
|
let mut mind = seeded();
|
||||||
committed(admit(&mut mind, vec![D::Question(question)]));
|
committed(admit(&mut mind, vec![D::Question(question)]));
|
||||||
|
committed(admit(&mut mind, vec![D::DeskListing(eureka_pipeline::PipelineDeskListing { campaign: slug("eureka-state"), listed: s(FORK), reason: "Listed.".into() })]));
|
||||||
let list: Vec<&str> = card["deciders"].as_array().unwrap().iter().map(|who| who.as_str().unwrap()).collect();
|
let list: Vec<&str> = card["deciders"].as_array().unwrap().iter().map(|who| who.as_str().unwrap()).collect();
|
||||||
if card["status"] == "closed" {
|
if card["status"] == "closed" {
|
||||||
close(&mut mind, FORK, "RQ1", "no");
|
close(&mut mind, FORK, "RQ1", "no");
|
||||||
|
|
@ -1561,27 +1644,27 @@ mod tests {
|
||||||
let mut mind = desk();
|
let mut mind = desk();
|
||||||
let hash = hash_of(&mind, FORK);
|
let hash = hash_of(&mind, FORK);
|
||||||
// The operator's own account, off the list, is no decider; a member on the list is.
|
// The operator's own account, off the list, is no decider; a member on the list is.
|
||||||
let off = mind.desk_answer(&deciders(&["x"]), s("op"), DeskAuthority::Operator, wire(FORK, &hash), now());
|
let off = mind.desk_answer(&deciders(&["x"]), account("op"), DeskAuthority::Operator, wire(FORK, &hash), now());
|
||||||
assert_eq!(answered(off).0.code, Some(ForkAnswerCode::NotADecider));
|
assert_eq!(answered(off).0.code, Some(ForkAnswerCode::NotADecider));
|
||||||
let on = mind.desk_answer(&deciders(&["m"]), s("m"), DeskAuthority::Member, wire(FORK, &hash), now());
|
let on = mind.desk_answer(&deciders(&["m"]), account("m"), DeskAuthority::Member, wire(FORK, &hash), now());
|
||||||
let (answer, _, _) = answered(on);
|
let (answer, _, _) = answered(on);
|
||||||
assert_eq!((answer.outcome.as_str(), answer.authority, answer.answerer.as_str()), ("Counted", DeskAuthority::Member, "m"));
|
assert_eq!((answer.outcome.as_str(), answer.authority, answer.answerer.as_deref().unwrap()), ("Counted", DeskAuthority::Member, account("m").0.as_str()));
|
||||||
}
|
}
|
||||||
|
|
||||||
// ---- the desk request on the wire: deciders ride each call, at most 64, never stored ----
|
// ---- the desk request on the wire: deciders ride each call, at most 64, never stored ----
|
||||||
|
|
||||||
use crate::envelope::{OperationFailure, decode_desk_request, encode_desk_request};
|
use crate::envelope::{OperationFailure, decode_desk_request, encode_desk_request};
|
||||||
|
|
||||||
fn list_of(n: usize) -> Vec<Short> {
|
fn list_of(n: usize) -> Vec<AccountId> {
|
||||||
(0..n).map(|at| s(&format!("d{at}"))).collect()
|
(0..n).map(|at| acct(&format!("d{at}"))).collect()
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Each request kind of the desk, carrying `list`.
|
/// Each request kind of the desk, carrying `list`.
|
||||||
fn desk_requests(mind: &str, list: &[Short], hash: &str) -> [DeskRequest; 3] {
|
fn desk_requests(mind: &str, list: &[AccountId], hash: &str) -> [DeskRequest; 3] {
|
||||||
[
|
[
|
||||||
DeskRequest::Answer { mind: slug(mind), subject: s("d0"), authority: DeskAuthority::Member, deciders: list.to_vec(), request: wire(FORK, hash) },
|
DeskRequest::Answer { mind: slug(mind), subject: acct("d0"), authority: DeskAuthority::Member, deciders: list.to_vec(), request: wire(FORK, hash) },
|
||||||
DeskRequest::Forks { mind: slug(mind), subject: s("d0"), deciders: list.to_vec(), status: None, cursor: None, limit: ForkLimit::new(10).unwrap() },
|
DeskRequest::Forks { mind: slug(mind), subject: acct("d0"), deciders: list.to_vec(), status: None, cursor: None, limit: ForkLimit::new(10).unwrap() },
|
||||||
DeskRequest::Fork { mind: slug(mind), subject: s("d0"), deciders: list.to_vec(), fork_id: FORK.into() },
|
DeskRequest::Fork { mind: slug(mind), subject: acct("d0"), deciders: list.to_vec(), fork_id: FORK.into() },
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -1604,7 +1687,7 @@ mod tests {
|
||||||
// 64 deciders through to the Mind: the subject is listed and counts.
|
// 64 deciders through to the Mind: the subject is listed and counts.
|
||||||
let DeskRequest::Answer { subject, authority, deciders, request, .. } = through_the_door(&desk_requests(INSTANCE, &list_of(64), &hash)[0]).unwrap() else { panic!("not an answer") };
|
let DeskRequest::Answer { subject, authority, deciders, request, .. } = through_the_door(&desk_requests(INSTANCE, &list_of(64), &hash)[0]).unwrap() else { panic!("not an answer") };
|
||||||
let before = head(&mind);
|
let before = head(&mind);
|
||||||
let response = mind.desk_answer(&deciders, subject, authority, request, now());
|
let response = mind.desk_answer(&deciders.iter().map(|id| s(&id.0)).collect::<Vec<_>>(), s(&subject.0), authority, request, now());
|
||||||
assert!(head(&mind) > before);
|
assert!(head(&mind) > before);
|
||||||
assert_eq!(answered(response).0.outcome, "Counted");
|
assert_eq!(answered(response).0.outcome, "Counted");
|
||||||
|
|
||||||
|
|
@ -1624,7 +1707,7 @@ mod tests {
|
||||||
let hash = hash_of(&mind, FORK);
|
let hash = hash_of(&mind, FORK);
|
||||||
let before = (head(&mind), stored_count(&mind, FORK));
|
let before = (head(&mind), stored_count(&mind, FORK));
|
||||||
let mut twice = list_of(3);
|
let mut twice = list_of(3);
|
||||||
twice.push(s("d1"));
|
twice.push(acct("d1"));
|
||||||
for (what, list) in [("empty", vec![]), ("duplicate", twice)] {
|
for (what, list) in [("empty", vec![]), ("duplicate", twice)] {
|
||||||
for sent in desk_requests(INSTANCE, &list, &hash) {
|
for sent in desk_requests(INSTANCE, &list, &hash) {
|
||||||
let failure = through_the_door(&sent).unwrap_err();
|
let failure = through_the_door(&sent).unwrap_err();
|
||||||
|
|
@ -1660,6 +1743,6 @@ mod tests {
|
||||||
// The same request to the mind it names is answered.
|
// The same request to the mind it names is answered.
|
||||||
let DeskRequest::Answer { mind: named, subject, authority, deciders, request } = through_the_door(&desk_requests(INSTANCE, &list_of(4), &hash)[0]).unwrap() else { panic!("not an answer") };
|
let DeskRequest::Answer { mind: named, subject, authority, deciders, request } = through_the_door(&desk_requests(INSTANCE, &list_of(4), &hash)[0]).unwrap() else { panic!("not an answer") };
|
||||||
mind.require_instance(&named).unwrap();
|
mind.require_instance(&named).unwrap();
|
||||||
assert_eq!(answered(mind.desk_answer(&deciders, subject, authority, request, now())).0.outcome, "Counted");
|
assert_eq!(answered(mind.desk_answer(&deciders.iter().map(|id| s(&id.0)).collect::<Vec<_>>(), s(&subject.0), authority, request, now())).0.outcome, "Counted");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
File diff suppressed because it is too large
Load diff
|
|
@ -74,6 +74,19 @@ impl<'i> Docs<'i> {
|
||||||
self.image.id_of(kind, id).filter(|row| self.visible(*row)).map(|row| self.image.row(row).ordinal)
|
self.image.id_of(kind, id).filter(|row| self.visible(*row)).map(|row| self.image.row(row).ordinal)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// When the document was first admitted, in UTC: an imported document's source receipt
|
||||||
|
/// (`imported_from.committed_at`) wins over this mind's own `admitted_at`. `None` when the
|
||||||
|
/// document is not held at this snapshot or the time does not parse as RFC3339 (an import's
|
||||||
|
/// `committed_at` is only checked as non-empty text at admission). Also `None` when the UTC year is
|
||||||
|
/// outside 0..=9999: CultLib's instant pattern has four year digits, and a local time near the edge
|
||||||
|
/// can convert out of range.
|
||||||
|
pub(crate) fn first_admitted_at(&self, kind: PipelineKind, id: &str) -> Option<chrono::DateTime<chrono::Utc>> {
|
||||||
|
let row = self.image.id_of(kind, id).filter(|row| self.visible(*row))?;
|
||||||
|
let admission = &self.image.row(row).view.admission;
|
||||||
|
let text = admission.provenance.imported_from.as_ref().map_or(admission.admitted_at.as_str(), |source| source.committed_at.0.as_str());
|
||||||
|
chrono::DateTime::parse_from_rfc3339(text).ok().map(|time| time.with_timezone(&chrono::Utc)).filter(|time| (0..=9999).contains(&chrono::Datelike::year(time)))
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) fn find(&self, kind: PipelineKind, id: &str) -> Option<&PipelineDocument> {
|
pub(crate) fn find(&self, kind: PipelineKind, id: &str) -> Option<&PipelineDocument> {
|
||||||
self.in_batch(kind, id).or_else(|| self.in_image(kind, id))
|
self.in_batch(kind, id).or_else(|| self.in_image(kind, id))
|
||||||
}
|
}
|
||||||
|
|
@ -388,7 +401,8 @@ pub(crate) fn line_of(document: &PipelineDocument) -> Option<Lineage<'_>> {
|
||||||
| D::Pulse(_)
|
| D::Pulse(_)
|
||||||
| D::Reply(_)
|
| D::Reply(_)
|
||||||
| D::Thought(_)
|
| D::Thought(_)
|
||||||
| D::DeskAnswer(_) => None,
|
| D::DeskAnswer(_)
|
||||||
|
| D::DeskListing(_) => None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -442,10 +456,11 @@ pub enum CitationRole {
|
||||||
DayThrough,
|
DayThrough,
|
||||||
Fork,
|
Fork,
|
||||||
Answer,
|
Answer,
|
||||||
|
Listed,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl CitationRole {
|
impl CitationRole {
|
||||||
pub const ALL: [CitationRole; 26] = [
|
pub const ALL: [CitationRole; 27] = [
|
||||||
Self::RaisedIn,
|
Self::RaisedIn,
|
||||||
Self::Answers,
|
Self::Answers,
|
||||||
Self::Rulings,
|
Self::Rulings,
|
||||||
|
|
@ -472,6 +487,7 @@ impl CitationRole {
|
||||||
Self::DayThrough,
|
Self::DayThrough,
|
||||||
Self::Fork,
|
Self::Fork,
|
||||||
Self::Answer,
|
Self::Answer,
|
||||||
|
Self::Listed,
|
||||||
];
|
];
|
||||||
|
|
||||||
/// The wire spelling, equal to the referring field's name.
|
/// The wire spelling, equal to the referring field's name.
|
||||||
|
|
@ -503,6 +519,7 @@ impl CitationRole {
|
||||||
Self::DayThrough => "day_through",
|
Self::DayThrough => "day_through",
|
||||||
Self::Fork => "fork",
|
Self::Fork => "fork",
|
||||||
Self::Answer => "answer",
|
Self::Answer => "answer",
|
||||||
|
Self::Listed => "listed",
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -530,6 +547,7 @@ impl CitationRole {
|
||||||
Self::Run => K::Reply,
|
Self::Run => K::Reply,
|
||||||
Self::Reads => K::Thought,
|
Self::Reads => K::Thought,
|
||||||
Self::Fork | Self::Answer => K::DeskAnswer,
|
Self::Fork | Self::Answer => K::DeskAnswer,
|
||||||
|
Self::Listed => K::DeskListing,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -539,7 +557,7 @@ impl CitationRole {
|
||||||
pub fn target_kinds(self) -> Vec<PipelineKind> {
|
pub fn target_kinds(self) -> Vec<PipelineKind> {
|
||||||
use PipelineKind as K;
|
use PipelineKind as K;
|
||||||
match self {
|
match self {
|
||||||
Self::Answers | Self::Questions | Self::Forks | Self::Fork => vec![K::Question],
|
Self::Answers | Self::Questions | Self::Forks | Self::Fork | Self::Listed => vec![K::Question],
|
||||||
Self::Rulings => vec![K::Ruling],
|
Self::Rulings => vec![K::Ruling],
|
||||||
Self::CutSpec => vec![K::CutSpec],
|
Self::CutSpec => vec![K::CutSpec],
|
||||||
Self::CutReport => vec![K::CutReport],
|
Self::CutReport => vec![K::CutReport],
|
||||||
|
|
@ -657,6 +675,7 @@ pub(crate) fn citations(document: &PipelineDocument) -> Vec<(CitationRole, Pipel
|
||||||
DeskOutcome::Counted | DeskOutcome::Refused { .. } => {}
|
DeskOutcome::Counted | DeskOutcome::Refused { .. } => {}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
D::DeskListing(listing) => edges.push((R::Listed, cited(K::Question, &listing.listed))),
|
||||||
D::Campaign(_) | D::Target(_) | D::Instance(_) | D::Stewardship(_) | D::Pulse(_) => {}
|
D::Campaign(_) | D::Target(_) | D::Instance(_) | D::Stewardship(_) | D::Pulse(_) => {}
|
||||||
}
|
}
|
||||||
edges
|
edges
|
||||||
|
|
|
||||||
|
|
@ -408,9 +408,9 @@ mod tests {
|
||||||
fn desk_answer() -> DeskRequest {
|
fn desk_answer() -> DeskRequest {
|
||||||
DeskRequest::Answer {
|
DeskRequest::Answer {
|
||||||
mind: slug(INSTANCE),
|
mind: slug(INSTANCE),
|
||||||
subject: crate::fixtures::s("user-7"),
|
subject: crate::fixtures::acct("user-7"),
|
||||||
authority: DeskAuthority::Member,
|
authority: DeskAuthority::Member,
|
||||||
deciders: vec![crate::fixtures::s("user-7")],
|
deciders: vec![crate::fixtures::acct("user-7")],
|
||||||
request: ForkAnswerRequest {
|
request: ForkAnswerRequest {
|
||||||
schema_version: crate::desk_wire::RequestSchema::V0,
|
schema_version: crate::desk_wire::RequestSchema::V0,
|
||||||
fork_id: "eureka-state:question:Q1".into(),
|
fork_id: "eureka-state:question:Q1".into(),
|
||||||
|
|
@ -425,7 +425,7 @@ mod tests {
|
||||||
}
|
}
|
||||||
|
|
||||||
fn desk_requests() -> [(DeskRequest, &'static str); 3] {
|
fn desk_requests() -> [(DeskRequest, &'static str); 3] {
|
||||||
let (mind, subject, deciders) = (slug(INSTANCE), crate::fixtures::s("user-7"), vec![crate::fixtures::s("user-7")]);
|
let (mind, subject, deciders) = (slug(INSTANCE), crate::fixtures::acct("user-7"), vec![crate::fixtures::acct("user-7")]);
|
||||||
[
|
[
|
||||||
(desk_answer(), "Answer"),
|
(desk_answer(), "Answer"),
|
||||||
(DeskRequest::Forks { mind: mind.clone(), subject: subject.clone(), deciders: deciders.clone(), status: Some(crate::desk_wire::ListStatus::Open), cursor: crate::desk_wire::ForkCursor::new(7), limit: crate::desk_wire::ForkLimit::new(50).unwrap() }, "Forks"),
|
(DeskRequest::Forks { mind: mind.clone(), subject: subject.clone(), deciders: deciders.clone(), status: Some(crate::desk_wire::ListStatus::Open), cursor: crate::desk_wire::ForkCursor::new(7), limit: crate::desk_wire::ForkLimit::new(50).unwrap() }, "Forks"),
|
||||||
|
|
@ -549,7 +549,7 @@ mod tests {
|
||||||
answered_at: "2026-11-10T12:00:00Z".into(),
|
answered_at: "2026-11-10T12:00:00Z".into(),
|
||||||
received_at: "2026-11-10T12:00:01Z".into(),
|
received_at: "2026-11-10T12:00:01Z".into(),
|
||||||
claim_in_future: false,
|
claim_in_future: false,
|
||||||
answerer: "user-7".into(),
|
answerer: Some("user-7".into()),
|
||||||
authority: DeskAuthority::Member,
|
authority: DeskAuthority::Member,
|
||||||
request_id: "00000000-0000-4000-8000-000000000001".into(),
|
request_id: "00000000-0000-4000-8000-000000000001".into(),
|
||||||
},
|
},
|
||||||
|
|
@ -579,13 +579,13 @@ mod tests {
|
||||||
fn a_maximal_desk_message_fits_the_frame() {
|
fn a_maximal_desk_message_fits_the_frame() {
|
||||||
let long = |n: usize| "x".repeat(n);
|
let long = |n: usize| "x".repeat(n);
|
||||||
let wire = |message: &CultNetMessage| encode_cultnet_message_to_vec(message, CultNetWireContract::CultNetSchemaV0).unwrap().len();
|
let wire = |message: &CultNetMessage| encode_cultnet_message_to_vec(message, CultNetWireContract::CultNetSchemaV0).unwrap().len();
|
||||||
let deciders: Vec<_> = (0..64).map(|at| crate::fixtures::s(&format!("{at:0>200}"))).collect();
|
let deciders: Vec<_> = (0..64).map(|at| crate::fixtures::acct(&format!("decider {at}"))).collect();
|
||||||
let mut sized = vec![];
|
let mut sized = vec![];
|
||||||
for (request, _) in desk_requests() {
|
for (request, _) in desk_requests() {
|
||||||
let widest = match request {
|
let widest = match request {
|
||||||
DeskRequest::Answer { mind, request, authority, .. } => DeskRequest::Answer {
|
DeskRequest::Answer { mind, request, authority, .. } => DeskRequest::Answer {
|
||||||
mind,
|
mind,
|
||||||
subject: crate::fixtures::s(&long(200)),
|
subject: crate::fixtures::acct("the widest subject"),
|
||||||
authority,
|
authority,
|
||||||
deciders: deciders.clone(),
|
deciders: deciders.clone(),
|
||||||
request: ForkAnswerRequest {
|
request: ForkAnswerRequest {
|
||||||
|
|
@ -596,10 +596,10 @@ mod tests {
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
DeskRequest::Forks { mind, status, cursor, limit, .. } => {
|
DeskRequest::Forks { mind, status, cursor, limit, .. } => {
|
||||||
DeskRequest::Forks { mind, subject: crate::fixtures::s(&long(200)), deciders: deciders.clone(), status, cursor, limit }
|
DeskRequest::Forks { mind, subject: crate::fixtures::acct("the widest subject"), deciders: deciders.clone(), status, cursor, limit }
|
||||||
}
|
}
|
||||||
DeskRequest::Fork { mind, .. } => {
|
DeskRequest::Fork { mind, .. } => {
|
||||||
DeskRequest::Fork { mind, subject: crate::fixtures::s(&long(200)), deciders: deciders.clone(), fork_id: long(200) }
|
DeskRequest::Fork { mind, subject: crate::fixtures::acct("the widest subject"), deciders: deciders.clone(), fork_id: long(200) }
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
let message = encode_desk_request("m-10", &widest, Some(long(200))).unwrap();
|
let message = encode_desk_request("m-10", &widest, Some(long(200))).unwrap();
|
||||||
|
|
@ -607,7 +607,8 @@ mod tests {
|
||||||
sized.push(wire(&message));
|
sized.push(wire(&message));
|
||||||
}
|
}
|
||||||
assert!(sized.iter().all(|size| *size <= DESK_MAX_REQUEST_BYTES), "{sized:?} against {DESK_MAX_REQUEST_BYTES}");
|
assert!(sized.iter().all(|size| *size <= DESK_MAX_REQUEST_BYTES), "{sized:?} against {DESK_MAX_REQUEST_BYTES}");
|
||||||
assert!(sized[0] > DESK_MAX_REQUEST_BYTES / 4, "the test builds a request that is actually large: {sized:?}");
|
// 64 account ids of 36 bytes and a 4,000-byte note: the widest answer a request can carry.
|
||||||
|
assert!(sized[0] > 4000 + 64 * 36, "the test builds a request that is actually large: {sized:?}");
|
||||||
|
|
||||||
let answer = ForkDeskAnswer {
|
let answer = ForkDeskAnswer {
|
||||||
schema_version: crate::desk_wire::AnswerSchema::V0,
|
schema_version: crate::desk_wire::AnswerSchema::V0,
|
||||||
|
|
@ -623,7 +624,7 @@ mod tests {
|
||||||
answered_at: "2026-11-10T12:00:00.123456789+00:00".into(),
|
answered_at: "2026-11-10T12:00:00.123456789+00:00".into(),
|
||||||
received_at: "2026-11-10T12:00:00.123456789+00:00".into(),
|
received_at: "2026-11-10T12:00:00.123456789+00:00".into(),
|
||||||
claim_in_future: true,
|
claim_in_future: true,
|
||||||
answerer: long(200),
|
answerer: Some(long(200)),
|
||||||
authority: DeskAuthority::Operator,
|
authority: DeskAuthority::Operator,
|
||||||
request_id: "00000000-0000-4000-8000-000000000001".into(),
|
request_id: "00000000-0000-4000-8000-000000000001".into(),
|
||||||
};
|
};
|
||||||
|
|
|
||||||
|
|
@ -56,6 +56,21 @@ pub(crate) fn s(value: &str) -> Short {
|
||||||
value.into()
|
value.into()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// A deterministic Bifrost account id for a test's name for someone: a lowercase D-format UUID.
|
||||||
|
pub(crate) fn account(name: &str) -> Short {
|
||||||
|
// An id already in account form stands for itself.
|
||||||
|
if name.len() == 36 && name.bytes().all(|byte| matches!(byte, b'0'..=b'9' | b'a'..=b'f' | b'-')) {
|
||||||
|
return Short(name.into());
|
||||||
|
}
|
||||||
|
let hash = name.bytes().fold(0xcbf2_9ce4_8422_2325_u64, |hash, byte| (hash ^ u64::from(byte)).wrapping_mul(0x0100_0000_01b3));
|
||||||
|
Short(format!("{:08x}-{:04x}-4000-8000-{:012x}", hash >> 32, (hash >> 16) & 0xffff, hash & 0xffff_ffff_ffff))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The same, as the stored type.
|
||||||
|
pub(crate) fn acct(name: &str) -> eureka_pipeline::AccountId {
|
||||||
|
eureka_pipeline::AccountId(account(name).0)
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) fn l(value: &str) -> Label {
|
pub(crate) fn l(value: &str) -> Label {
|
||||||
value.into()
|
value.into()
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -150,7 +150,7 @@ mod tests {
|
||||||
fn view(document: PipelineDocument, ordinal: u64) -> PipelineDocumentView {
|
fn view(document: PipelineDocument, ordinal: u64) -> PipelineDocumentView {
|
||||||
let key = pipeline_key(&document).unwrap();
|
let key = pipeline_key(&document).unwrap();
|
||||||
let id = PipelineRef { kind: document.kind(), id: s(&key) };
|
let id = PipelineRef { kind: document.kind(), id: s(&key) };
|
||||||
let faculty = if ordinal % 2 == 0 { Faculty::Hands } else { Faculty::Soul };
|
let faculty = if ordinal.is_multiple_of(2) { Faculty::Hands } else { Faculty::Soul };
|
||||||
let admission = AdmissionFacts {
|
let admission = AdmissionFacts {
|
||||||
receipt_id: format!("receipt-{ordinal}"),
|
receipt_id: format!("receipt-{ordinal}"),
|
||||||
admitted_at: format!("2026-11-08T16:{ordinal:02}:00Z"),
|
admitted_at: format!("2026-11-08T16:{ordinal:02}:00Z"),
|
||||||
|
|
@ -226,7 +226,7 @@ mod tests {
|
||||||
let to_source = [held.clone(), vec![moved(RECEIVER, 4)]].concat();
|
let to_source = [held.clone(), vec![moved(RECEIVER, 4)]].concat();
|
||||||
assert_eq!(plan(&slug(CAMPAIGN), &slug(RECEIVER), &slug(RECEIVER), to_source), Err(ImportFault::SourceIsReceiver));
|
assert_eq!(plan(&slug(CAMPAIGN), &slug(RECEIVER), &slug(RECEIVER), to_source), Err(ImportFault::SourceIsReceiver));
|
||||||
let other_root = view(
|
let other_root = view(
|
||||||
resolution(r(K::Campaign, &format!("elsewhere:campaign:self")), ResolutionOutcome::Moved { to: slug(RECEIVER) }),
|
resolution(r(K::Campaign, "elsewhere:campaign:self"), ResolutionOutcome::Moved { to: slug(RECEIVER) }),
|
||||||
4,
|
4,
|
||||||
);
|
);
|
||||||
assert_eq!(plan_of([held, vec![other_root]].concat()), Err(ImportFault::NotMoved), "another root's move does not move this one");
|
assert_eq!(plan_of([held, vec![other_root]].concat()), Err(ImportFault::NotMoved), "another root's move does not move this one");
|
||||||
|
|
|
||||||
|
|
@ -105,8 +105,8 @@ pub fn index_text(document: &PipelineDocument) -> Option<String> {
|
||||||
lines.push(&idea.idea.0);
|
lines.push(&idea.idea.0);
|
||||||
lines.push(&idea.tradeoffs.0);
|
lines.push(&idea.tradeoffs.0);
|
||||||
}
|
}
|
||||||
// A desk answer is a person's words, not indexed.
|
// A desk answer is a person's words and a listing's reason is no retrieval text: neither is indexed.
|
||||||
D::Instance(_) | D::Stewardship(_) | D::Run(_) | D::Pulse(_) | D::Reply(_) | D::DeskAnswer(_) => {}
|
D::Instance(_) | D::Stewardship(_) | D::Run(_) | D::Pulse(_) | D::Reply(_) | D::DeskAnswer(_) | D::DeskListing(_) => {}
|
||||||
}
|
}
|
||||||
lines.retain(|line| !line.is_empty());
|
lines.retain(|line| !line.is_empty());
|
||||||
if lines.is_empty() {
|
if lines.is_empty() {
|
||||||
|
|
|
||||||
|
|
@ -41,6 +41,7 @@ mod rows;
|
||||||
pub mod oracle;
|
pub mod oracle;
|
||||||
pub mod query;
|
pub mod query;
|
||||||
pub mod receipt;
|
pub mod receipt;
|
||||||
|
pub mod redact;
|
||||||
pub mod refusal;
|
pub mod refusal;
|
||||||
pub mod store;
|
pub mod store;
|
||||||
pub mod wire;
|
pub mod wire;
|
||||||
|
|
@ -72,7 +73,7 @@ pub use store::{MindStore, OwnedRedbMessagePackBackingStore};
|
||||||
pub use wire::{
|
pub use wire::{
|
||||||
DeferredAnswer, HuginnMindRequest, HuginnMindResponse, IndexStatus, MAX_DEFERRED_BODY_BYTES, MIND_REQUEST_SCHEMA, MIND_REQUEST_SCHEMA_JSON,
|
DeferredAnswer, HuginnMindRequest, HuginnMindResponse, IndexStatus, MAX_DEFERRED_BODY_BYTES, MIND_REQUEST_SCHEMA, MIND_REQUEST_SCHEMA_JSON,
|
||||||
MIND_RESPONSE_SCHEMA, MIND_RESPONSE_SCHEMA_JSON, MIND_SERVICE_ID, MindStatus, PersonaStatus,
|
MIND_RESPONSE_SCHEMA, MIND_RESPONSE_SCHEMA_JSON, MIND_SERVICE_ID, MindStatus, PersonaStatus,
|
||||||
ACTIVITY_REPOS_MAX, OpenPr, PrAction, RepoActivity, RepoChange, RepoEvent, RepoMoved, SOURCE_TEXTS_MAX, SOURCE_TOP_K_MAX, SourceHit, SourceHits, SourceModel, SourceQuery,
|
ACTIVITY_REPOS_MAX, CHAT_ROOMS_MAX, CHAT_WINDOW_MAX, ChatHit, ChatHits, ChatMessage, ChatQuery, ChatWindow, ChatWindowQuery, EVENT_ID_MAX, OpenPr, PrAction, RepoActivity, RepoChange, RepoEvent, RepoMoved, SOURCE_TEXTS_MAX, SOURCE_TOP_K_MAX, SourceHit, SourceHits, SourceModel, SourceQuery,
|
||||||
};
|
};
|
||||||
|
|
||||||
/// The leaf, whole, through the one crate that pins its rev: the daemon and
|
/// The leaf, whole, through the one crate that pins its rev: the daemon and
|
||||||
|
|
|
||||||
|
|
@ -663,7 +663,6 @@ fn an_import_never_cites_or_resolves_a_follow_up_the_receiver_wrote() {
|
||||||
let native = follow_up("FU-N", r(K::Question, &q1));
|
let native = follow_up("FU-N", r(K::Question, &q1));
|
||||||
let native_ref = r(K::FollowUp, &pipeline_key(&native).unwrap());
|
let native_ref = r(K::FollowUp, &pipeline_key(&native).unwrap());
|
||||||
committed(admit(&mut recv, vec![native]));
|
committed(admit(&mut recv, vec![native]));
|
||||||
let before = head(&recv);
|
|
||||||
// An imported record that withdraws the receiver's own follow-up mixes a replay into her work.
|
// An imported record that withdraws the receiver's own follow-up mixes a replay into her work.
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
refusal(import_from_kiko(&mut recv, vec![resolution(native_ref.clone(), ResolutionOutcome::Withdrawn { reason: "x".into() })])),
|
refusal(import_from_kiko(&mut recv, vec![resolution(native_ref.clone(), ResolutionOutcome::Withdrawn { reason: "x".into() })])),
|
||||||
|
|
|
||||||
|
|
@ -166,6 +166,8 @@ pub enum PipelineFacts {
|
||||||
/// the one that counts for its fork at the read's snapshot, derived by `counted_answer` and never
|
/// the one that counts for its fork at the read's snapshot, derived by `counted_answer` and never
|
||||||
/// stored: a retracted `Counted` answer reads `counts: false`, the promoted next answer `true`.
|
/// stored: a retracted `Counted` answer reads `counts: false`, the promoted next answer `true`.
|
||||||
DeskAnswer { fork: Short, sequence: u32, outcome: Short, counts: bool, authority: DeskAuthority },
|
DeskAnswer { fork: Short, sequence: u32, outcome: Short, counts: bool, authority: DeskAuthority },
|
||||||
|
/// The question this listing lets members read.
|
||||||
|
DeskListing { listed: Short },
|
||||||
}
|
}
|
||||||
|
|
||||||
impl PipelineFacts {
|
impl PipelineFacts {
|
||||||
|
|
@ -306,6 +308,7 @@ impl PipelineFacts {
|
||||||
counts: counted_of(&value.fork.0).as_deref() == Some(id.id.0.as_str()),
|
counts: counted_of(&value.fork.0).as_deref() == Some(id.id.0.as_str()),
|
||||||
authority: value.authority,
|
authority: value.authority,
|
||||||
},
|
},
|
||||||
|
D::DeskListing(value) => Self::DeskListing { listed: value.listed.clone() },
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -1849,7 +1852,7 @@ mod tests {
|
||||||
|
|
||||||
// Two desk answers on the open fork Q2: the second is a Revision of the first.
|
// Two desk answers on the open fork Q2: the second is a Revision of the first.
|
||||||
let D::Question(shown) = question("Q2", &["A", "B"], "A") else { unreachable!() };
|
let D::Question(shown) = question("Q2", &["A", "B"], "A") else { unreachable!() };
|
||||||
let stamp = crate::desk::DeskStamp { answerer: s("x"), authority: eureka_pipeline::DeskAuthority::Member, deciders: vec![s("x")] };
|
let stamp = crate::desk::DeskStamp { answerer: crate::fixtures::account("x"), authority: eureka_pipeline::DeskAuthority::Member, deciders: vec![crate::fixtures::account("x")] };
|
||||||
let pick = |n: u32| crate::desk::DeskAnswerRequest {
|
let pick = |n: u32| crate::desk::DeskAnswerRequest {
|
||||||
fork: s(&q2),
|
fork: s(&q2),
|
||||||
options_hash: crate::desk::options_hash(&shown),
|
options_hash: crate::desk::options_hash(&shown),
|
||||||
|
|
@ -1862,6 +1865,10 @@ mod tests {
|
||||||
assert!(matches!(mind.answer_desk(stamp.clone(), pick(n), now()), crate::desk::DeskAnswerOutcome::Stored { .. }));
|
assert!(matches!(mind.answer_desk(stamp.clone(), pick(n), now()), crate::desk::DeskAnswerOutcome::Stored { .. }));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// The Mind lists Q2: one `listed` edge.
|
||||||
|
committed(admit(&mut mind, vec![D::DeskListing(eureka_pipeline::PipelineDeskListing { campaign: slug(CAMPAIGN), listed: s(&q2), reason: "Members may read why.".into() })]));
|
||||||
|
let listing = id("desk_listing", "Q2");
|
||||||
|
|
||||||
let (n1, f1_closed, f2_closed) = (
|
let (n1, f1_closed, f2_closed) = (
|
||||||
id("resolution", "question.Q1.n1"),
|
id("resolution", "question.Q1.n1"),
|
||||||
id("resolution", &format!("finding.{}.n1", f1.rsplit(':').next().unwrap())),
|
id("resolution", &format!("finding.{}.n1", f1.rsplit(':').next().unwrap())),
|
||||||
|
|
@ -1910,8 +1917,9 @@ mod tests {
|
||||||
(CitationRole::Fork, &answer_1, &q2),
|
(CitationRole::Fork, &answer_1, &q2),
|
||||||
(CitationRole::Fork, &answer_2, &q2),
|
(CitationRole::Fork, &answer_2, &q2),
|
||||||
(CitationRole::Answer, &answer_2, &answer_1),
|
(CitationRole::Answer, &answer_2, &answer_1),
|
||||||
|
(CitationRole::Listed, &listing, &q2),
|
||||||
];
|
];
|
||||||
assert_eq!(CitationRole::ALL.len(), 26);
|
assert_eq!(CitationRole::ALL.len(), 27);
|
||||||
for role in CitationRole::ALL {
|
for role in CitationRole::ALL {
|
||||||
let selection =
|
let selection =
|
||||||
Selection { cited: Some(Incoming { role: role.name().into(), exists: true }), ..Selection::default() };
|
Selection { cited: Some(Incoming { role: role.name().into(), exists: true }), ..Selection::default() };
|
||||||
|
|
|
||||||
|
|
@ -330,13 +330,13 @@ impl CacheBackingStore for LandsThenFails {
|
||||||
fn pull_all(&self) -> Result<Vec<CultCacheEnvelope>> {
|
fn pull_all(&self) -> Result<Vec<CultCacheEnvelope>> {
|
||||||
self.inner.pull_all()
|
self.inner.pull_all()
|
||||||
}
|
}
|
||||||
fn push(&mut self, entry: &CultCacheEnvelope) -> Result<()> {
|
fn push(&mut self, entry: &CultCacheEnvelope) -> Result<String> {
|
||||||
self.inner.push(entry)
|
self.inner.push(entry)
|
||||||
}
|
}
|
||||||
fn delete(&mut self, entry: &CultCacheEnvelope) -> Result<()> {
|
fn delete(&mut self, entry: &CultCacheEnvelope) -> Result<()> {
|
||||||
self.inner.delete(entry)
|
self.inner.delete(entry)
|
||||||
}
|
}
|
||||||
fn push_all(&mut self, entries: &[CultCacheEnvelope], options: PushAllOptions) -> Result<()> {
|
fn push_all(&mut self, entries: &[CultCacheEnvelope], options: PushAllOptions) -> Result<Vec<String>> {
|
||||||
self.inner.push_all(entries, options)
|
self.inner.push_all(entries, options)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -547,13 +547,13 @@ impl CacheBackingStore for FailsTheLook {
|
||||||
}
|
}
|
||||||
self.inner.pull_all()
|
self.inner.pull_all()
|
||||||
}
|
}
|
||||||
fn push(&mut self, entry: &CultCacheEnvelope) -> Result<()> {
|
fn push(&mut self, entry: &CultCacheEnvelope) -> Result<String> {
|
||||||
self.inner.push(entry)
|
self.inner.push(entry)
|
||||||
}
|
}
|
||||||
fn delete(&mut self, entry: &CultCacheEnvelope) -> Result<()> {
|
fn delete(&mut self, entry: &CultCacheEnvelope) -> Result<()> {
|
||||||
self.inner.delete(entry)
|
self.inner.delete(entry)
|
||||||
}
|
}
|
||||||
fn push_all(&mut self, entries: &[CultCacheEnvelope], options: PushAllOptions) -> Result<()> {
|
fn push_all(&mut self, entries: &[CultCacheEnvelope], options: PushAllOptions) -> Result<Vec<String>> {
|
||||||
self.inner.push_all(entries, options)
|
self.inner.push_all(entries, options)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -374,7 +374,7 @@ mod tests {
|
||||||
self.inner.pull_all()
|
self.inner.pull_all()
|
||||||
}
|
}
|
||||||
|
|
||||||
fn push(&mut self, entry: &CultCacheEnvelope) -> anyhow::Result<()> {
|
fn push(&mut self, entry: &CultCacheEnvelope) -> anyhow::Result<String> {
|
||||||
self.inner.push(entry)
|
self.inner.push(entry)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -382,7 +382,7 @@ mod tests {
|
||||||
self.inner.delete(entry)
|
self.inner.delete(entry)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn push_all(&mut self, entries: &[CultCacheEnvelope], options: PushAllOptions) -> anyhow::Result<()> {
|
fn push_all(&mut self, entries: &[CultCacheEnvelope], options: PushAllOptions) -> anyhow::Result<Vec<String>> {
|
||||||
self.inner.push_all(entries, options)
|
self.inner.push_all(entries, options)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
517
crates/huginn-mind/src/redact.rs
Normal file
517
crates/huginn-mind/src/redact.rs
Normal file
|
|
@ -0,0 +1,517 @@
|
||||||
|
//! Credential redaction: the only code that knows the shape of a secret in text.
|
||||||
|
//!
|
||||||
|
//! `redact_url_userinfo` redacts URL credentials (the Self run's journal and reply,
|
||||||
|
//! the Persona's structured output). `scrub` is chat ingest's door: every message body
|
||||||
|
//! passes it before it is chunked, embedded or stored, and it also runs the URL rule.
|
||||||
|
//! Shapes only; nothing here reads meaning, and a hit is a kind, never a value.
|
||||||
|
|
||||||
|
/// The one redaction of URL credentials. Call it at each place a string the CLI produced enters the journal, her mind or chat: today
|
||||||
|
/// the Self result (`result_text`: journal line and Reply) and the structured output of a Persona or Sleep result
|
||||||
|
/// (`persona::structured_output`, string by string). It does not cover claude's stderr (inherited) or the CLI's transcript files.
|
||||||
|
///
|
||||||
|
/// A forge token rides in a remote URL, so every `scheme://` and every protocol-relative `//` opens an authority, read as git, curl and
|
||||||
|
/// RFC 3986 read it: it ends at the first `/`, `?`, `#`, backslash, quote, backtick, parenthesis, bracket, angle bracket or ASCII whitespace
|
||||||
|
/// (`is_authority_end`). Its userinfo is everything before the authority's LAST `@` (a raw `@` in a password still redacts), and non-ASCII
|
||||||
|
/// characters, NBSP and U+2028 included, stay inside it. The userinfo becomes `<redacted>`: `scheme://<redacted>@host`. With a colon it is
|
||||||
|
/// always redacted; without one it is a bare token on http, https, ftp, ftps, ws, wss, any `+http`/`+https` scheme (`git+https`), protocol-relative `//` and an unparseable scheme, and a plain
|
||||||
|
/// username on any other scheme (`ssh://git@host`), which stays. A `@` after the authority (`github.com/@user`, `pkg@1.2.3`) is path, not
|
||||||
|
/// userinfo, and is left alone. Scheme-less credentials are covered only in the shape `word:secret@host.tld/` at the start of an
|
||||||
|
/// ASCII-whitespace-delimited word. Not covered: a password holding `/ ? # ` a quote or ASCII whitespace (no parser reads it as userinfo),
|
||||||
|
/// scp-style `user:secret@host:path`, tokens in query strings, claude's stderr, the kept transcripts. Syntax only; no reading of meaning.
|
||||||
|
pub fn redact_url_userinfo(text: &str) -> String {
|
||||||
|
let mut out = String::with_capacity(text.len());
|
||||||
|
let mut rest = text;
|
||||||
|
while let Some(at) = rest.find("//") {
|
||||||
|
let (before, after) = rest.split_at(at);
|
||||||
|
let body = &after[2..];
|
||||||
|
out.push_str(before);
|
||||||
|
out.push_str("//");
|
||||||
|
// The scheme is the ASCII run before `:`. Without a colon, `//` opens an authority only at a word start (`a//b@c` is not one).
|
||||||
|
let scheme = match before.strip_suffix(':') {
|
||||||
|
// Only ASCII bytes match, so the byte after the last non-match is a character boundary.
|
||||||
|
Some(head) => head[head.bytes().rposition(|c| !(c.is_ascii_alphanumeric() || matches!(c, b'+' | b'-' | b'.'))).map_or(0, |i| i + 1)..].to_ascii_lowercase(),
|
||||||
|
None if before.chars().next_back().is_none_or(|c| is_break(c) || matches!(c, '"' | '\'' | '`' | '(' | '[' | '{' | '<' | '=' | ',')) => String::new(),
|
||||||
|
None => {
|
||||||
|
rest = body;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let end = body.find(is_authority_end).unwrap_or(body.len());
|
||||||
|
let authority = &body[..end];
|
||||||
|
match authority.rfind('@') {
|
||||||
|
Some(i) if i > 0 && (authority[..i].contains(':') || carries_bare_token(&scheme)) => {
|
||||||
|
out.push_str("<redacted>");
|
||||||
|
out.push_str(&authority[i..]);
|
||||||
|
}
|
||||||
|
_ => out.push_str(authority),
|
||||||
|
}
|
||||||
|
rest = &body[end..];
|
||||||
|
}
|
||||||
|
out.push_str(rest);
|
||||||
|
redact_schemeless(&out)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Schemes whose lone userinfo is a token, not a username: web and ftp schemes, `ws`/`wss`, and `x+http`/`x+https` wrappers; `scheme` is lowercase.
|
||||||
|
fn carries_bare_token(scheme: &str) -> bool {
|
||||||
|
matches!(scheme, "" | "http" | "https" | "ftp" | "ftps" | "ws" | "wss") || scheme.ends_with("+http") || scheme.ends_with("+https")
|
||||||
|
}
|
||||||
|
|
||||||
|
/// ASCII whitespace; NBSP, U+2028 and other Unicode whitespace are not, so a secret holding them is redacted whole.
|
||||||
|
fn is_break(c: char) -> bool {
|
||||||
|
matches!(c, ' ' | '\t' | '\n' | '\r' | '\x0b' | '\x0c')
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The end of a URL's authority.
|
||||||
|
fn is_authority_end(c: char) -> bool {
|
||||||
|
is_break(c) || matches!(c, '/' | '?' | '#' | '\\' | '"' | '\'' | '`' | '(' | ')' | '[' | ']' | '{' | '}' | '<' | '>')
|
||||||
|
}
|
||||||
|
|
||||||
|
/// `word:secret@host.tld/` at the start of a whitespace-delimited word that holds no `//` becomes `<redacted>@host.tld/`.
|
||||||
|
fn redact_schemeless(text: &str) -> String {
|
||||||
|
let mut out = String::with_capacity(text.len());
|
||||||
|
let mut word = String::new();
|
||||||
|
let flush = |word: &mut String, out: &mut String| {
|
||||||
|
out.push_str(&redact_word(word));
|
||||||
|
word.clear();
|
||||||
|
};
|
||||||
|
for c in text.chars() {
|
||||||
|
if is_break(c) {
|
||||||
|
flush(&mut word, &mut out);
|
||||||
|
out.push(c);
|
||||||
|
} else {
|
||||||
|
word.push(c);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
flush(&mut word, &mut out);
|
||||||
|
out
|
||||||
|
}
|
||||||
|
|
||||||
|
fn redact_word(word: &str) -> String {
|
||||||
|
let shaped = (|| {
|
||||||
|
if word.contains("//") {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let at = word.rfind('@')?;
|
||||||
|
let colon = word.find(':')?;
|
||||||
|
let (user, host) = (&word[..colon], &word[at + 1..]);
|
||||||
|
let host_name = &host[..host.find('/')?];
|
||||||
|
let user_ok = !user.is_empty() && user.chars().all(|c| c.is_ascii_alphanumeric() || matches!(c, '.' | '_' | '-'));
|
||||||
|
let host_ok = host_name.contains('.') && host_name.chars().all(|c| c.is_ascii_alphanumeric() || matches!(c, '.' | '-'));
|
||||||
|
(colon + 1 < at && user_ok && host_ok).then_some(at)
|
||||||
|
})();
|
||||||
|
match shaped {
|
||||||
|
Some(at) => format!("<redacted>{}", &word[at..]),
|
||||||
|
None => word.to_string(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The version of the scrub's rules. Part of the chat version: a bump makes every
|
||||||
|
/// room re-ingest from the room's history and scrub it again.
|
||||||
|
pub const SCRUB_VERSION: u32 = 2;
|
||||||
|
|
||||||
|
/// What replaces a secret.
|
||||||
|
pub const REDACTED: &str = "<redacted>";
|
||||||
|
|
||||||
|
/// The shape of secret a scrub found. Never its value.
|
||||||
|
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
|
||||||
|
pub enum ScrubKind {
|
||||||
|
UrlUserinfo,
|
||||||
|
Bearer,
|
||||||
|
AuthorizationHeader,
|
||||||
|
KeyShape,
|
||||||
|
Jwt,
|
||||||
|
PemBlock,
|
||||||
|
AccessToken,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl std::fmt::Display for ScrubKind {
|
||||||
|
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||||
|
formatter.write_str(match self {
|
||||||
|
Self::UrlUserinfo => "url-userinfo",
|
||||||
|
Self::Bearer => "bearer",
|
||||||
|
Self::AuthorizationHeader => "authorization-header",
|
||||||
|
Self::KeyShape => "key-shape",
|
||||||
|
Self::Jwt => "jwt",
|
||||||
|
Self::PemBlock => "pem-block",
|
||||||
|
Self::AccessToken => "access-token",
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A body with its secrets replaced by `<redacted>`, and the kinds found, one per secret.
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||||
|
pub struct Scrubbed {
|
||||||
|
pub text: String,
|
||||||
|
pub hits: Vec<ScrubKind>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Zero-width characters a secret could be split with.
|
||||||
|
fn zero_width(c: char) -> bool {
|
||||||
|
matches!(c, '\u{200b}' | '\u{200c}' | '\u{200d}' | '\u{2060}' | '\u{feff}' | '\u{00ad}' | '\u{180e}')
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Replaces credential shapes in `text`: a PEM or PGP private key block, an `authorization:` header
|
||||||
|
/// line, `Bearer`/`Basic`/`token` followed by a long token (blank or colon between), a vendor key
|
||||||
|
/// shape (Matrix tokens included), an `access_token` query or JSON value, a JWT, and URL userinfo.
|
||||||
|
/// Each hit becomes `<redacted>`. Token shapes only; ordinary prose, SHAs, UUIDs and ids pass byte for byte.
|
||||||
|
/// Zero-width characters are removed before matching; when anything matched, the returned text is the
|
||||||
|
/// stripped one, so a token split by them leaves no tail, and when nothing matched the text is returned as it came (emoji joiners stay).
|
||||||
|
pub fn scrub(text: &str) -> Scrubbed {
|
||||||
|
let mut hits = Vec::new();
|
||||||
|
let mut stripped: String = text.chars().filter(|c| !zero_width(*c)).collect();
|
||||||
|
let passes: [(ScrubKind, Finder); 6] = [
|
||||||
|
(ScrubKind::PemBlock, pem_block),
|
||||||
|
(ScrubKind::AuthorizationHeader, authorization_line),
|
||||||
|
(ScrubKind::AccessToken, access_token),
|
||||||
|
(ScrubKind::Bearer, bearer_token),
|
||||||
|
(ScrubKind::KeyShape, key_shape),
|
||||||
|
(ScrubKind::Jwt, jwt),
|
||||||
|
];
|
||||||
|
for (kind, find) in passes {
|
||||||
|
stripped = replace_matches(&stripped, kind, find, &mut hits);
|
||||||
|
}
|
||||||
|
let urls = redact_url_userinfo(&stripped);
|
||||||
|
let (before, after) = (stripped.matches(REDACTED).count(), urls.matches(REDACTED).count());
|
||||||
|
hits.extend(std::iter::repeat_n(ScrubKind::UrlUserinfo, after.saturating_sub(before)));
|
||||||
|
if hits.is_empty() { Scrubbed { text: text.to_owned(), hits } } else { Scrubbed { text: urls, hits } }
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A scrub pass: finds the end of one secret starting at a byte offset, if one starts there.
|
||||||
|
type Finder = fn(&str, usize) -> Option<usize>;
|
||||||
|
|
||||||
|
fn replace_matches(text: &str, kind: ScrubKind, find: Finder, hits: &mut Vec<ScrubKind>) -> String {
|
||||||
|
let mut out = String::with_capacity(text.len());
|
||||||
|
let mut at = 0;
|
||||||
|
while at < text.len() {
|
||||||
|
match find(text, at) {
|
||||||
|
Some(end) if end > at => {
|
||||||
|
out.push_str(REDACTED);
|
||||||
|
hits.push(kind);
|
||||||
|
at = end;
|
||||||
|
}
|
||||||
|
_ => {
|
||||||
|
let next = at + text[at..].chars().next().map_or(1, char::len_utf8);
|
||||||
|
out.push_str(&text[at..next]);
|
||||||
|
at = next;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
out
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Whether `at` starts a word: the byte before it is not an ASCII letter or digit.
|
||||||
|
fn word_start(text: &str, at: usize) -> bool {
|
||||||
|
at == 0 || !text.as_bytes()[at - 1].is_ascii_alphanumeric()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The end of the run of bytes at `from` that `member` accepts.
|
||||||
|
fn run_end(text: &str, from: usize, member: impl Fn(u8) -> bool) -> usize {
|
||||||
|
from + text.as_bytes()[from..].iter().take_while(|byte| member(**byte)).count()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn starts_with_ignore_case(text: &str, at: usize, word: &str) -> bool {
|
||||||
|
text.as_bytes().get(at..at + word.len()).is_some_and(|held| held.eq_ignore_ascii_case(word.as_bytes()))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn token_char(byte: u8) -> bool {
|
||||||
|
byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b'~' | b'+' | b'/' | b'=' | b'-')
|
||||||
|
}
|
||||||
|
|
||||||
|
fn key_char(byte: u8) -> bool {
|
||||||
|
byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'-')
|
||||||
|
}
|
||||||
|
|
||||||
|
/// `-----BEGIN ... PRIVATE KEY-----` through its `-----END ... PRIVATE KEY-----`, or the text's end.
|
||||||
|
fn pem_block(text: &str, at: usize) -> Option<usize> {
|
||||||
|
let rest = &text[at..];
|
||||||
|
let label = rest.strip_prefix("-----BEGIN ")?;
|
||||||
|
let words = run_end(label, 0, |byte| byte.is_ascii_uppercase() || byte == b' ');
|
||||||
|
(label[..words].ends_with("PRIVATE KEY") || label[..words].ends_with("PRIVATE KEY BLOCK")).then_some(())?;
|
||||||
|
label[words..].starts_with("-----").then_some(())?;
|
||||||
|
let body_from = at + "-----BEGIN ".len() + words + 5;
|
||||||
|
let end = text[body_from..].find("-----END ").and_then(|end| {
|
||||||
|
let name = body_from + end + "-----END ".len();
|
||||||
|
let words = run_end(text, name, |byte| byte.is_ascii_uppercase() || byte == b' ');
|
||||||
|
text[words..].starts_with("-----").then_some(words + 5)
|
||||||
|
});
|
||||||
|
Some(end.unwrap_or(text.len()))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// `authorization:` in any case at a word start, through the end of the line.
|
||||||
|
fn authorization_line(text: &str, at: usize) -> Option<usize> {
|
||||||
|
(word_start(text, at) && starts_with_ignore_case(text, at, "authorization:")).then_some(())?;
|
||||||
|
let from = at + "authorization:".len();
|
||||||
|
let end = run_end(text, from, |byte| byte != b'\n' && byte != b'\r');
|
||||||
|
(!text[from..end].trim().is_empty()).then_some(end)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// `Bearer`, `Basic` or `token`, blanks, then 16 or more token characters.
|
||||||
|
fn bearer_token(text: &str, at: usize) -> Option<usize> {
|
||||||
|
word_start(text, at).then_some(())?;
|
||||||
|
let word = ["bearer", "basic", "token"].into_iter().find(|word| starts_with_ignore_case(text, at, word))?;
|
||||||
|
let blanks = at + word.len();
|
||||||
|
let colon = usize::from(text.as_bytes().get(blanks) == Some(&b':'));
|
||||||
|
let value = run_end(text, blanks + colon, |byte| byte == b' ' || byte == b'\t');
|
||||||
|
(value > blanks).then_some(())?;
|
||||||
|
let end = run_end(text, value, token_char);
|
||||||
|
(end - value >= 16).then_some(end)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A vendor key: the prefix, then a long enough run of the key alphabet.
|
||||||
|
fn key_shape(text: &str, at: usize) -> Option<usize> {
|
||||||
|
let rest = &text[at..];
|
||||||
|
let by_prefix = [("sk-", 16), ("sk_live_", 16), ("pk_live_", 16), ("github_pat_", 16), ("glpat-", 16), ("syt_", 16), ("mct_", 16)];
|
||||||
|
let tokens = ["ghp_", "gho_", "ghu_", "ghs_", "ghr_"].into_iter().map(|prefix| (prefix, 16));
|
||||||
|
for (prefix, least) in by_prefix.into_iter().chain(tokens) {
|
||||||
|
// Only the generic prefix needs a word start (`risk-averse`, `task-force`); the distinctive ones are found glued to anything.
|
||||||
|
if rest.starts_with(prefix) && (prefix != "sk-" || word_start(text, at)) {
|
||||||
|
let end = run_end(text, at + prefix.len(), key_char);
|
||||||
|
return (end - at - prefix.len() >= least).then_some(end);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for prefix in ["xoxa-", "xoxb-", "xoxp-", "xoxr-", "xoxs-"] {
|
||||||
|
if rest.starts_with(prefix) {
|
||||||
|
let end = run_end(text, at + prefix.len(), |byte| byte.is_ascii_alphanumeric() || byte == b'-');
|
||||||
|
return (end - at - prefix.len() >= 16).then_some(end);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for prefix in ["AKIA", "ASIA"] {
|
||||||
|
if rest.starts_with(prefix) {
|
||||||
|
let end = run_end(text, at + prefix.len(), |byte| byte.is_ascii_uppercase() || byte.is_ascii_digit());
|
||||||
|
return (end - at - prefix.len() >= 16).then_some(end);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if rest.starts_with("AIza") {
|
||||||
|
let end = run_end(text, at + 4, key_char);
|
||||||
|
return (end - at - 4 >= 35).then_some(end);
|
||||||
|
}
|
||||||
|
None
|
||||||
|
}
|
||||||
|
|
||||||
|
/// `access_token` as a query parameter or JSON member, through its value (8 or more characters).
|
||||||
|
fn access_token(text: &str, at: usize) -> Option<usize> {
|
||||||
|
starts_with_ignore_case(text, at, "access_token").then_some(())?;
|
||||||
|
let mut at = at + "access_token".len();
|
||||||
|
let skip = |at: usize, set: &[u8]| run_end(text, at, |byte| set.contains(&byte));
|
||||||
|
at = skip(at, b"\"'");
|
||||||
|
at = skip(at, b" \t");
|
||||||
|
matches!(text.as_bytes().get(at), Some(b'=' | b':')).then_some(())?;
|
||||||
|
at = skip(at + 1, b" \t");
|
||||||
|
at = skip(at, b"\"'");
|
||||||
|
let end = run_end(text, at, |byte| !byte.is_ascii_whitespace() && !matches!(byte, b'&' | b'"' | b'\'' | b',' | b'}' | b';' | b'#'));
|
||||||
|
(end - at >= 8).then_some(end)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Three base64url segments, the first starting `eyJ`.
|
||||||
|
fn jwt(text: &str, at: usize) -> Option<usize> {
|
||||||
|
text[at..].starts_with("eyJ").then_some(())?;
|
||||||
|
let segment = |from: usize| run_end(text, from, key_char);
|
||||||
|
let first = segment(at);
|
||||||
|
let mut end = first;
|
||||||
|
for _ in 0..2 {
|
||||||
|
(text.as_bytes().get(end) == Some(&b'.')).then_some(())?;
|
||||||
|
let next = segment(end + 1);
|
||||||
|
(next > end + 1).then_some(())?;
|
||||||
|
end = next;
|
||||||
|
}
|
||||||
|
Some(end)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
const CANARY: &str = "CANARY-NOT-A-REAL-TOKEN-0000";
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn url_userinfo_is_redacted_where_it_is_a_secret_and_kept_where_it_is_a_username() {
|
||||||
|
let said = format!(
|
||||||
|
"a https://user:{CANARY}@host.example/r.git b https://{CANARY}@host.example/r.git c ssh://u:{CANARY}@h/x d http://{CANARY}@h e HTTPS://x:{CANARY}@h \
|
||||||
|
f https://u:p@ss{CANARY}@h/x g ssh://git@host.example/r.git h git://git@h/x i https://host.example/no-userinfo j (https://u:{CANARY}@h) k ://@x"
|
||||||
|
);
|
||||||
|
let want = "a https://<redacted>@host.example/r.git b https://<redacted>@host.example/r.git c ssh://<redacted>@h/x d http://<redacted>@h e HTTPS://<redacted>@h \
|
||||||
|
f https://<redacted>@h/x g ssh://git@host.example/r.git h git://git@h/x i https://host.example/no-userinfo j (https://<redacted>@h) k ://@x";
|
||||||
|
assert_eq!(redact_url_userinfo(&said), want);
|
||||||
|
assert!(!redact_url_userinfo(&said).contains(CANARY));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_secret_is_redacted_whole_by_the_authority_rule_and_commonplace_urls_are_left_alone() {
|
||||||
|
let r = redact_url_userinfo;
|
||||||
|
// Raw @, percent-encoding and non-ASCII (NBSP, U+2028) stay inside userinfo.
|
||||||
|
for secret in ["pa@ssT", "pa%40ssT", "pa\u{a0}ssT", "pa\u{2028}ssT", "pa:ssT", "\u{e9}\u{65e5}ssT"] {
|
||||||
|
let out = r(&format!("x https://u:{secret}@h/x"));
|
||||||
|
assert_eq!(out, "x https://<redacted>@h/x", "{secret:?}");
|
||||||
|
}
|
||||||
|
// The authority ends at / ? # \\ quotes, brackets and ASCII whitespace: no parser reads those as userinfo, so what follows is path.
|
||||||
|
assert_eq!(r("https://u:pa/ssT@h/x"), "https://u:pa/ssT@h/x");
|
||||||
|
assert_eq!(r("https://u:pa ssT@h/x"), "https://u:pa ssT@h/x");
|
||||||
|
assert_eq!(r("https://u:pa?ssT@h"), "https://u:pa?ssT@h");
|
||||||
|
// Ordinary URLs with @ in the path are byte-identical.
|
||||||
|
for same in [
|
||||||
|
"https://github.com/@user/repo",
|
||||||
|
"https://medium.com/@name/post-1",
|
||||||
|
"https://registry.npmjs.org/@scope/pkg",
|
||||||
|
"https://h/pkg@1.2.3",
|
||||||
|
"a//b@c",
|
||||||
|
"see //cdn.h/@x/y and https://h/?mail=a@b.c#f@g",
|
||||||
|
"https://h:8080/p git@github.com:org/r.git ssh://git@host/r.git ssh://git@h:22/r git://git@h/x",
|
||||||
|
] {
|
||||||
|
assert_eq!(r(same), same);
|
||||||
|
}
|
||||||
|
// Compact JSON: only the credentialed URL changes.
|
||||||
|
let json = format!(r#"{{"a":"https://u:{CANARY}@h/x","b":"https://h2/@v","c":"https://h3/p@1"}}"#);
|
||||||
|
assert_eq!(r(&json), r#"{"a":"https://<redacted>@h/x","b":"https://h2/@v","c":"https://h3/p@1"}"#);
|
||||||
|
assert_eq!(r("a/https://u:T@h/x,https://v:T@h2"), "a/https://<redacted>@h/x,https://<redacted>@h2");
|
||||||
|
assert_eq!(r("HTTPS://TOKEN@h"), "HTTPS://<redacted>@h");
|
||||||
|
assert_eq!(r("[l](https://T@h/x) <https://T@h> `https://u:T@h`"), "[l](https://<redacted>@h/x) <https://<redacted>@h> `https://<redacted>@h`");
|
||||||
|
assert_eq!(r("a //u:T@host/x b //T@host c \"//T@h\" (//T@h)"), "a //<redacted>@host/x b //<redacted>@host c \"//<redacted>@h\" (//<redacted>@h)");
|
||||||
|
// Token-bearing schemes beyond http: +http/+https wrappers and ws/wss redact a bare token; git+ssh and ssh keep their username.
|
||||||
|
assert_eq!(r("git+https://T@github.com/a/b GIT+HTTPS://T@h wss://T@h/ ws://T@h/"), "git+https://<redacted>@github.com/a/b GIT+HTTPS://<redacted>@h wss://<redacted>@h/ ws://<redacted>@h/");
|
||||||
|
for same in ["git+ssh://git@host/x", "ssh://git@host/x"] {
|
||||||
|
assert_eq!(r(same), same);
|
||||||
|
}
|
||||||
|
// Scheme-less: only word:secret@host.tld/ . Each guard of the shape has its own case.
|
||||||
|
assert_eq!(r("user:T@host.tld/path"), "<redacted>@host.tld/path");
|
||||||
|
for same in ["a:b@c", "a:b@host/x", "a:b@ho_st.tld/x", "u$er:T@host.tld/x", "a:@host.tld/x", "mail me@x.org/ and :T@host.tld/x"] {
|
||||||
|
assert_eq!(r(same), same);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn multibyte_text_never_panics_the_redaction() {
|
||||||
|
let r = redact_url_userinfo;
|
||||||
|
for case in ["see \u{201c}https://h/x", "\u{2192}https://h/x", "docs\u{ff1a}https://h/x", "\u{e9}://x", "\u{65e5}\u{672c}://u:T@h", "https://x/\u{e9}://", "\u{e9}//u:T@h", "\u{e9}:u:T@h.tld/"] {
|
||||||
|
let _ = r(case);
|
||||||
|
}
|
||||||
|
assert_eq!(r("\u{65e5}\u{672c}://u:T@h"), "\u{65e5}\u{672c}://<redacted>@h");
|
||||||
|
assert_eq!(r("\u{2192}https://u:T@h/x"), "\u{2192}https://<redacted>@h/x");
|
||||||
|
let seed = "\u{201c}\u{2192}\u{ff1a}\u{e9}\u{65e5}\u{a0}https://u:T@h/x//y:z@a.b/ ://@ //";
|
||||||
|
let chars: Vec<char> = seed.chars().collect();
|
||||||
|
for start in 0..=chars.len() {
|
||||||
|
for end in start..=chars.len() {
|
||||||
|
let _ = r(&chars[start..end].iter().collect::<String>());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
/// Every shape is built here from fake parts; none is a real credential.
|
||||||
|
fn secrets() -> Vec<(ScrubKind, String, String)> {
|
||||||
|
let pem = "-----BEGIN RSA PRIVATE KEY-----\nMIIEowIBAAKCAQEAfakefakefake\n-----END RSA PRIVATE KEY-----".to_owned();
|
||||||
|
vec![
|
||||||
|
(ScrubKind::AuthorizationHeader, format!("Authorization: Bearer {}", "a1B2".repeat(8)), "a1B2a1B2".into()),
|
||||||
|
(ScrubKind::Bearer, format!("Bearer {}", "z9Y8x7".repeat(4)), "z9Y8x7z9Y8".into()),
|
||||||
|
(ScrubKind::Bearer, format!("Basic {}", "dXNlcjpwYXNz".repeat(2)), "dXNlcjpwYXNzdXNl".into()),
|
||||||
|
(ScrubKind::KeyShape, format!("sk-{}", "q7W3".repeat(8)), "q7W3q7W3".into()),
|
||||||
|
(ScrubKind::KeyShape, format!("sk_live_{}", "m4N5".repeat(6)), "m4N5m4N5".into()),
|
||||||
|
(ScrubKind::KeyShape, format!("ghp_{}", "k2L9".repeat(9)), "k2L9k2L9".into()),
|
||||||
|
(ScrubKind::KeyShape, format!("github_pat_{}_{}", "A1b2C3".repeat(4) + "ab", "d4E5f6".repeat(9) + "abcde"), "A1b2C3A1b2".into()),
|
||||||
|
(ScrubKind::KeyShape, format!("glpat-{}", "r6T7".repeat(5)), "r6T7r6T7".into()),
|
||||||
|
(ScrubKind::KeyShape, "xoxb-123456789012-987654321098-aBcDeFgHiJkLmNoP".into(), "aBcDeFgHiJkL".into()),
|
||||||
|
(ScrubKind::KeyShape, "AKIA0123456789ABCDEF".into(), "0123456789ABCDEF".into()),
|
||||||
|
(ScrubKind::KeyShape, format!("AIza{}", "Sy1_".repeat(9)), "Sy1_Sy1_Sy1_".into()),
|
||||||
|
(ScrubKind::Jwt, "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJmYWtlIn0.c2lnbmF0dXJlZmFrZQ".into(), "c2lnbmF0dXJlZmFrZQ".into()),
|
||||||
|
(ScrubKind::PemBlock, pem, "MIIEowIBAAKCAQEAfake".into()),
|
||||||
|
(ScrubKind::PemBlock, "-----BEGIN PRIVATE KEY-----\nMIIBVQIBADANfakeNOEND".into(), "MIIBVQIBADANfake".into()),
|
||||||
|
(ScrubKind::KeyShape, format!("syt_{}_{}_{}", "YWxpY2U", "aBcDeFgHiJkLmNoPqR", "0a1B2c"), "aBcDeFgHiJkLmNoPqR".into()),
|
||||||
|
(ScrubKind::KeyShape, format!("mct_{}", "w4X5".repeat(8)), "w4X5w4X5w4X5".into()),
|
||||||
|
(ScrubKind::KeyShape, "xoxa-2-123456789012-aBcDeFgHiJkLmNoP".into(), "aBcDeFgHiJkL".into()),
|
||||||
|
(ScrubKind::KeyShape, "xoxp-123456789012-987654321098-aBcDeFgHiJkLmNoP".into(), "aBcDeFgHiJkL".into()),
|
||||||
|
(ScrubKind::KeyShape, "xoxr-123456789012-987654321098-aBcDeFgHiJkLmNoP".into(), "aBcDeFgHiJkL".into()),
|
||||||
|
(ScrubKind::KeyShape, "xoxs-123456789012-987654321098-aBcDeFgHiJkLmNoP".into(), "aBcDeFgHiJkL".into()),
|
||||||
|
(ScrubKind::KeyShape, "ASIA0123456789ABCDEF".into(), "0123456789ABCDEF".into()),
|
||||||
|
(ScrubKind::KeyShape, "gho_".to_owned() + &"p1Q2".repeat(9), "p1Q2p1Q2p1Q2".into()),
|
||||||
|
(ScrubKind::KeyShape, "pk_live_".to_owned() + &"e3R4".repeat(6), "e3R4e3R4e3R4".into()),
|
||||||
|
(ScrubKind::AccessToken, "https://h/_matrix/client/v3/sync?access_token=abcDEF123456xyz&since=s1".into(), "abcDEF123456xyz".into()),
|
||||||
|
(ScrubKind::AccessToken, "{\"access_token\": \"tokenValue987654\", \"user\": 1}".into(), "tokenValue987654".into()),
|
||||||
|
(ScrubKind::Bearer, format!("bearer:{}", "z9Y8x7".repeat(4)), "z9Y8x7z9Y8".into()),
|
||||||
|
(ScrubKind::Bearer, format!("token: {}", "m1N2o3".repeat(4)), "m1N2o3m1N2".into()),
|
||||||
|
(ScrubKind::PemBlock, "-----BEGIN PGP PRIVATE KEY BLOCK-----\nlQOYBFakeFakeFake\n-----END PGP PRIVATE KEY BLOCK-----".into(), "lQOYBFakeFake".into()),
|
||||||
|
(ScrubKind::UrlUserinfo, "https://user:hunter2hunter2@host.example/r.git".into(), "hunter2hunter2".into()),
|
||||||
|
(ScrubKind::UrlUserinfo, "https://tokenvalue12345@host.example/".into(), "tokenvalue12345".into()),
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn every_planted_credential_is_replaced_mid_line_and_at_line_end_with_its_kind() {
|
||||||
|
for (kind, secret, distinct) in secrets() {
|
||||||
|
for body in [format!("before {secret} after"), format!("before {secret}"), format!("{secret}\nnext line")] {
|
||||||
|
let out = scrub(&body);
|
||||||
|
assert!(!out.text.contains(&distinct), "{kind}: {}", out.text);
|
||||||
|
assert!(out.text.contains(REDACTED), "{kind}: {}", out.text);
|
||||||
|
assert!(out.hits.contains(&kind), "{kind} not in {:?} for {body:?}", out.hits);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_pem_block_without_its_end_runs_to_the_end_of_the_body_and_one_with_it_stops_there() {
|
||||||
|
let open = scrub("see -----BEGIN EC PRIVATE KEY-----\nAAAA\nBBBB").text;
|
||||||
|
assert_eq!(open, "see <redacted>");
|
||||||
|
let closed = scrub("-----BEGIN PRIVATE KEY-----\nAAAA\n-----END PRIVATE KEY----- and then prose").text;
|
||||||
|
assert_eq!(closed, "<redacted> and then prose");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn an_authorization_header_goes_to_the_end_of_its_line_only() {
|
||||||
|
assert_eq!(scrub("hi\nAUTHORIZATION: Token abc\nbye").text, "hi\n<redacted>\nbye");
|
||||||
|
assert_eq!(scrub("Authorization: x").hits, [ScrubKind::AuthorizationHeader]);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn ordinary_text_passes_byte_for_byte_and_reports_no_hit() {
|
||||||
|
for same in [
|
||||||
|
"risk-averse", "task-force", "sk-", "sk-short", "3f2b8c1d9e7a4b5c6d0e1f2a3b4c5d6e7f8a9b0c",
|
||||||
|
"123e4567-e89b-12d3-a456-426614174000", "!abc:s", "$Ab3dEf-9_zQ", "user@example.com", "ssh://git@host/r.git",
|
||||||
|
"https://github.com/@user", "pkg@1.2.3", "the bearer of bad news", "Authorization is a header",
|
||||||
|
"padding like dGVzdA== in prose", "```\nlet x = 1;\n```", "the BEGIN PRIVATE KEY header", "a token of thanks",
|
||||||
|
"AKIAshort", "ghp_short", "eyJ is a prefix", "naïve café — 日本語 with Bearer in it",
|
||||||
|
] {
|
||||||
|
let out = scrub(same);
|
||||||
|
assert_eq!(out.text, same);
|
||||||
|
assert!(out.hits.is_empty(), "{same}: {:?}", out.hits);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn multibyte_text_around_a_secret_never_panics_the_scrub() {
|
||||||
|
let body = format!("日本 \u{a0}é Bearer {} → ünï", "A1b2".repeat(6));
|
||||||
|
assert_eq!(scrub(&body).text, "日本 \u{a0}é <redacted> → ünï");
|
||||||
|
let seed = "é日 sk-aaaaaaaaaaaaaaaaaaaa -----BEGIN PRIVATE KEY----- authorization: é eyJa.b.c https://u:p@h/x";
|
||||||
|
let chars: Vec<char> = seed.chars().collect();
|
||||||
|
for start in 0..chars.len() {
|
||||||
|
for end in start..=chars.len() {
|
||||||
|
let _ = scrub(&chars[start..end].iter().collect::<String>());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_distinctive_key_is_found_glued_to_other_text_but_the_generic_prefix_still_needs_a_word_start() {
|
||||||
|
let ghp = format!("xxx{}", "ghp_".to_owned() + &"k2L9".repeat(9));
|
||||||
|
assert_eq!(scrub(&ghp).text, "xxx<redacted>");
|
||||||
|
assert_eq!(scrub("idAKIA0123456789ABCDEFend").hits, [ScrubKind::KeyShape]);
|
||||||
|
assert_eq!(scrub(&format!("zzz{}", "xoxb-123456789012-aBcDeFgHiJkLmNoP")).hits, [ScrubKind::KeyShape]);
|
||||||
|
assert_eq!(scrub(&format!("q{}", "AIza".to_owned() + &"Sy1_".repeat(9))).hits, [ScrubKind::KeyShape]);
|
||||||
|
assert!(scrub("risk-averse-thinking-process task-force-alpha-bravo").hits.is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn zero_width_characters_inside_a_token_leave_no_tail_and_emoji_joiners_survive_when_nothing_matched() {
|
||||||
|
let split = format!("sk-{}\u{200b}{}", "q7W3".repeat(4), "q7W3".repeat(4));
|
||||||
|
let out = scrub(&format!("key {split} end"));
|
||||||
|
assert_eq!(out.text, "key <redacted> end");
|
||||||
|
let jwt = "eyJhbGciOiJIUzI1NiJ9.eyJz\u{feff}dWIiOiJmYWtlIn0.c2lnbmF0dXJl\u{200d}ZmFrZQ";
|
||||||
|
assert_eq!(scrub(jwt).text, "<redacted>");
|
||||||
|
let family = "family \u{1f468}\u{200d}\u{1f469}\u{200d}\u{1f467} here";
|
||||||
|
assert_eq!(scrub(family).text, family);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn matrix_tokens_and_query_values_are_found_and_prose_about_them_is_not() {
|
||||||
|
for same in ["the access_token is a thing", "access_token: short", "syt_ is the prefix", "mct_ tokens", "bearer:short"] {
|
||||||
|
assert_eq!(scrub(same).text, same, "{same}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -248,7 +248,7 @@ impl SelectionRow {
|
||||||
D::Stewardship(stewardship) => vec![stewardship.repo.identity()],
|
D::Stewardship(stewardship) => vec![stewardship.repo.identity()],
|
||||||
D::HandOff(hand_off) => vec![hand_off.repo.identity()],
|
D::HandOff(hand_off) => vec![hand_off.repo.identity()],
|
||||||
D::Ruling(PipelineRuling { authority: RulingAuthority::Steward { repo, .. }, .. }) => vec![repo.identity()],
|
D::Ruling(PipelineRuling { authority: RulingAuthority::Steward { repo, .. }, .. }) => vec![repo.identity()],
|
||||||
D::Target(_) | D::Question(_) | D::Ruling(_) | D::Verdict(_) | D::Finding(_) | D::Instance(_) | D::Resolution(_) | D::Run(_) | D::Pulse(_) | D::Reply(_) | D::Thought(_) | D::Memory(_) | D::Research(_) | D::Idea(_) | D::DeskAnswer(_) => {
|
D::Target(_) | D::Question(_) | D::Ruling(_) | D::Verdict(_) | D::Finding(_) | D::Instance(_) | D::Resolution(_) | D::Run(_) | D::Pulse(_) | D::Reply(_) | D::Thought(_) | D::Memory(_) | D::Research(_) | D::Idea(_) | D::DeskAnswer(_) | D::DeskListing(_) => {
|
||||||
Vec::new()
|
Vec::new()
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
|
@ -466,7 +466,7 @@ mod tests {
|
||||||
/// refused at the substrate's door.
|
/// refused at the substrate's door.
|
||||||
#[test]
|
#[test]
|
||||||
fn each_kind_declares_exactly_its_aliases_and_none_is_numeric() {
|
fn each_kind_declares_exactly_its_aliases_and_none_is_numeric() {
|
||||||
let table: [(K, &[&str]); 21] = [
|
let table: [(K, &[&str]); 22] = [
|
||||||
(K::Campaign, &["repo"]),
|
(K::Campaign, &["repo"]),
|
||||||
(K::Target, &[]),
|
(K::Target, &[]),
|
||||||
(K::Question, &[]),
|
(K::Question, &[]),
|
||||||
|
|
@ -488,6 +488,7 @@ mod tests {
|
||||||
(K::Reply, &[]),
|
(K::Reply, &[]),
|
||||||
(K::Thought, &[]),
|
(K::Thought, &[]),
|
||||||
(K::DeskAnswer, &[]),
|
(K::DeskAnswer, &[]),
|
||||||
|
(K::DeskListing, &[]),
|
||||||
];
|
];
|
||||||
for (kind, extra) in table {
|
for (kind, extra) in table {
|
||||||
let want = [&["root", "in_force", "faculty"][..], extra].concat();
|
let want = [&["root", "in_force", "faculty"][..], extra].concat();
|
||||||
|
|
@ -503,7 +504,7 @@ mod tests {
|
||||||
/// twenty-one.
|
/// twenty-one.
|
||||||
#[test]
|
#[test]
|
||||||
fn each_kind_declares_the_roles_it_carries_and_their_targets() {
|
fn each_kind_declares_the_roles_it_carries_and_their_targets() {
|
||||||
let table: [(K, &[&str]); 21] = [
|
let table: [(K, &[&str]); 22] = [
|
||||||
(K::Campaign, &[]),
|
(K::Campaign, &[]),
|
||||||
(K::Target, &[]),
|
(K::Target, &[]),
|
||||||
(K::Question, &["raised_in"]),
|
(K::Question, &["raised_in"]),
|
||||||
|
|
@ -525,13 +526,14 @@ mod tests {
|
||||||
(K::Reply, &["run"]),
|
(K::Reply, &["run"]),
|
||||||
(K::Thought, &["reads"]),
|
(K::Thought, &["reads"]),
|
||||||
(K::DeskAnswer, &["fork", "answer"]),
|
(K::DeskAnswer, &["fork", "answer"]),
|
||||||
|
(K::DeskListing, &["listed"]),
|
||||||
];
|
];
|
||||||
for (kind, roles) in table {
|
for (kind, roles) in table {
|
||||||
assert_eq!(sorted(Vocabulary.declared_roles(kind.type_id())), strings(roles), "{kind:?}");
|
assert_eq!(sorted(Vocabulary.declared_roles(kind.type_id())), strings(roles), "{kind:?}");
|
||||||
}
|
}
|
||||||
assert_eq!(Vocabulary.target_leaves("answers"), vec![K::Question.type_id().to_string()]);
|
assert_eq!(Vocabulary.target_leaves("answers"), vec![K::Question.type_id().to_string()]);
|
||||||
assert_eq!(Vocabulary.target_leaves("rulings"), vec![K::Ruling.type_id().to_string()]);
|
assert_eq!(Vocabulary.target_leaves("rulings"), vec![K::Ruling.type_id().to_string()]);
|
||||||
assert_eq!(Vocabulary.target_leaves("subject").len(), 21);
|
assert_eq!(Vocabulary.target_leaves("subject").len(), 22);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
sorted(Vocabulary.target_leaves("claims")),
|
sorted(Vocabulary.target_leaves("claims")),
|
||||||
sorted([K::CutSpec, K::Finding, K::FollowUp].iter().map(|kind| kind.type_id().to_string()).collect())
|
sorted([K::CutSpec, K::Finding, K::FollowUp].iter().map(|kind| kind.type_id().to_string()).collect())
|
||||||
|
|
|
||||||
|
|
@ -116,9 +116,9 @@ pub(crate) mod test_stores {
|
||||||
Ok(self.rows())
|
Ok(self.rows())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn push(&mut self, entry: &CultCacheEnvelope) -> Result<()> {
|
fn push(&mut self, entry: &CultCacheEnvelope) -> Result<String> {
|
||||||
self.plant(entry.clone());
|
self.plant(entry.clone());
|
||||||
Ok(())
|
Ok(entry.stored_at.clone())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn delete(&mut self, entry: &CultCacheEnvelope) -> Result<()> {
|
fn delete(&mut self, entry: &CultCacheEnvelope) -> Result<()> {
|
||||||
|
|
@ -126,9 +126,9 @@ pub(crate) mod test_stores {
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn push_all(&mut self, entries: &[CultCacheEnvelope], _options: PushAllOptions) -> Result<()> {
|
fn push_all(&mut self, entries: &[CultCacheEnvelope], _options: PushAllOptions) -> Result<Vec<String>> {
|
||||||
*self.rows.lock().unwrap() = entries.iter().map(|entry| (identity(entry), entry.clone())).collect();
|
*self.rows.lock().unwrap() = entries.iter().map(|entry| (identity(entry), entry.clone())).collect();
|
||||||
Ok(())
|
Ok(entries.iter().map(|entry| entry.stored_at.clone()).collect())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -199,7 +199,7 @@ pub(crate) mod test_stores {
|
||||||
self.inner.pull_all()
|
self.inner.pull_all()
|
||||||
}
|
}
|
||||||
|
|
||||||
fn push(&mut self, entry: &CultCacheEnvelope) -> Result<()> {
|
fn push(&mut self, entry: &CultCacheEnvelope) -> Result<String> {
|
||||||
self.inner.push(entry)
|
self.inner.push(entry)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -207,7 +207,7 @@ pub(crate) mod test_stores {
|
||||||
self.inner.delete(entry)
|
self.inner.delete(entry)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn push_all(&mut self, entries: &[CultCacheEnvelope], options: PushAllOptions) -> Result<()> {
|
fn push_all(&mut self, entries: &[CultCacheEnvelope], options: PushAllOptions) -> Result<Vec<String>> {
|
||||||
self.inner.push_all(entries, options)
|
self.inner.push_all(entries, options)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -92,6 +92,13 @@ pub enum HuginnMindRequest {
|
||||||
/// docs, which no mind holds. The daemon embeds the text and asks the
|
/// docs, which no mind holds. The daemon embeds the text and asks the
|
||||||
/// index it does not write.
|
/// index it does not write.
|
||||||
SearchSources { instance: Slug, query: SourceQuery },
|
SearchSources { instance: Slug, query: SourceQuery },
|
||||||
|
/// Search of the chat index: chunks of the rooms the Minds are meant to
|
||||||
|
/// have, by room, by time span and by meaning. Read-only, like
|
||||||
|
/// `SearchSources`, and queued with it.
|
||||||
|
SearchChat { instance: Slug, query: ChatQuery },
|
||||||
|
/// A chat message and its neighbours in the same room, read from the chat
|
||||||
|
/// index (never from Synapse), by the event id a chat hit cites.
|
||||||
|
ChatWindow { instance: Slug, query: ChatWindowQuery },
|
||||||
/// What others did in `repos` on the forge since the `after` cursor: the
|
/// What others did in `repos` on the forge since the `after` cursor: the
|
||||||
/// daemon reads the forge and no document of the mind, and answers
|
/// daemon reads the forge and no document of the mind, and answers
|
||||||
/// `Refused(Unavailable)` when it has no forge or the forge does not answer
|
/// `Refused(Unavailable)` when it has no forge or the forge does not answer
|
||||||
|
|
@ -270,6 +277,141 @@ pub struct SourceHits {
|
||||||
pub hits: Vec<SourceHit>,
|
pub hits: Vec<SourceHit>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// The most rooms one chat search names.
|
||||||
|
pub const CHAT_ROOMS_MAX: usize = 16;
|
||||||
|
|
||||||
|
/// A chat search. The selectors compose as AND. `rooms` are any-of: a value
|
||||||
|
/// beginning `!` is a room id, `#` a canonical alias. `after_ms` and
|
||||||
|
/// `before_ms` (Unix milliseconds) keep the chunks whose span overlaps
|
||||||
|
/// [after, before]. `texts` (none to `SOURCE_TEXTS_MAX`) is the meaning match,
|
||||||
|
/// several fused as in a source search; with none the search is a listing:
|
||||||
|
/// chunks oldest first, at most `top_k`, `ChatHits.truncated` saying more
|
||||||
|
/// exist. Within one room the caller narrows with `after_ms` = the last hit's
|
||||||
|
/// `ts_end_ms + 1`; across rooms spans overlap and that loses chunks, so list
|
||||||
|
/// one room at a time.
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||||
|
pub struct ChatQuery {
|
||||||
|
pub texts: Vec<Line>,
|
||||||
|
pub rooms: Vec<Short>,
|
||||||
|
pub after_ms: Option<u64>,
|
||||||
|
pub before_ms: Option<u64>,
|
||||||
|
pub top_k: u32,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ChatQuery {
|
||||||
|
/// The door. A refusal names the field.
|
||||||
|
pub fn check(&self) -> Result<(), MindRefusal> {
|
||||||
|
let refused = |field: &str, value: Option<String>, message: &str| MindRefusal::SelectionInvalid {
|
||||||
|
field: field.into(),
|
||||||
|
value,
|
||||||
|
message: message.into(),
|
||||||
|
};
|
||||||
|
if self.texts.len() > SOURCE_TEXTS_MAX {
|
||||||
|
return Err(refused("texts", Some(self.texts.len().to_string()), "a chat search takes 0 to 8 texts"));
|
||||||
|
}
|
||||||
|
if self.texts.iter().any(|text| text.0.trim().is_empty()) {
|
||||||
|
return Err(refused("texts", None, "every text of a chat search says something"));
|
||||||
|
}
|
||||||
|
if self.rooms.len() > CHAT_ROOMS_MAX {
|
||||||
|
return Err(refused("rooms", Some(self.rooms.len().to_string()), "a chat search names at most 16 rooms"));
|
||||||
|
}
|
||||||
|
if let Some(room) = self.rooms.iter().find(|room| !(room.0.starts_with('!') || room.0.starts_with('#')) || room.0.len() < 2) {
|
||||||
|
return Err(refused("rooms", Some(room.0.clone()), "a room is a room id beginning `!` or a canonical alias beginning `#`"));
|
||||||
|
}
|
||||||
|
if let (Some(after), Some(before)) = (self.after_ms, self.before_ms)
|
||||||
|
&& before < after
|
||||||
|
{
|
||||||
|
return Err(refused("before_ms", Some(before.to_string()), "before_ms is not earlier than after_ms"));
|
||||||
|
}
|
||||||
|
if !(1..=SOURCE_TOP_K_MAX).contains(&self.top_k) {
|
||||||
|
return Err(refused("top_k", Some(self.top_k.to_string()), "a chat search asks for 1 to 50 hits"));
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One message of a chunk, as the room held it.
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||||
|
pub struct ChatMessage {
|
||||||
|
pub event_id: String,
|
||||||
|
/// The sender's Matrix id.
|
||||||
|
pub sender: String,
|
||||||
|
pub ts_ms: u64,
|
||||||
|
pub body: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One chunk of consecutive messages in one room. Every citation is
|
||||||
|
/// required: only the room's alias may be absent, because a room may have
|
||||||
|
/// none. The span and the first and last ids are those of `messages` as
|
||||||
|
/// returned, after any trimming to the asked span.
|
||||||
|
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||||
|
pub struct ChatHit {
|
||||||
|
pub room_id: String,
|
||||||
|
pub room_alias: Option<String>,
|
||||||
|
pub first_event_id: String,
|
||||||
|
pub last_event_id: String,
|
||||||
|
pub ts_start_ms: u64,
|
||||||
|
pub ts_end_ms: u64,
|
||||||
|
/// Cosine similarity for one text, the reciprocal-rank sum for several,
|
||||||
|
/// none in a listing.
|
||||||
|
pub score: Option<f32>,
|
||||||
|
pub messages: Vec<ChatMessage>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The answer to a chat search: oldest first in a listing, nearest first with
|
||||||
|
/// texts.
|
||||||
|
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||||
|
pub struct ChatHits {
|
||||||
|
pub indexed_under: SourceModel,
|
||||||
|
/// A listing had more chunks than `top_k`.
|
||||||
|
pub truncated: bool,
|
||||||
|
pub hits: Vec<ChatHit>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The most messages a chat window reaches to either side of its anchor.
|
||||||
|
pub const CHAT_WINDOW_MAX: u32 = 50;
|
||||||
|
|
||||||
|
/// The longest event id a chat window takes.
|
||||||
|
pub const EVENT_ID_MAX: usize = 200;
|
||||||
|
|
||||||
|
/// A chat window: the message `event_id` names and up to `before` messages
|
||||||
|
/// ahead of it and `after` behind it, in its room.
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||||
|
pub struct ChatWindowQuery {
|
||||||
|
pub event_id: Short,
|
||||||
|
pub before: u32,
|
||||||
|
pub after: u32,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ChatWindowQuery {
|
||||||
|
/// The door. A refusal names the field.
|
||||||
|
pub fn check(&self) -> Result<(), MindRefusal> {
|
||||||
|
let refused = |field: &str, value: String, message: &str| MindRefusal::SelectionInvalid { field: field.into(), value: Some(value), message: message.into() };
|
||||||
|
if !self.event_id.0.starts_with('$') || self.event_id.0.len() > EVENT_ID_MAX {
|
||||||
|
return Err(refused("event_id", self.event_id.0.clone(), "an event id begins `$` and is at most 200 bytes"));
|
||||||
|
}
|
||||||
|
for (field, count) in [("before", self.before), ("after", self.after)] {
|
||||||
|
if count > CHAT_WINDOW_MAX {
|
||||||
|
return Err(refused(field, count.to_string(), "a chat window reaches 0 to 50 messages to each side"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The answer to a chat window: `messages` chronological with the anchor in
|
||||||
|
/// them once, all of `room_id`, each carrying the citations a hit's do.
|
||||||
|
/// `more_before` and `more_after` say messages exist beyond the cut.
|
||||||
|
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||||
|
pub struct ChatWindow {
|
||||||
|
pub room_id: String,
|
||||||
|
pub room_alias: Option<String>,
|
||||||
|
pub anchor_event_id: String,
|
||||||
|
pub messages: Vec<ChatMessage>,
|
||||||
|
pub more_before: bool,
|
||||||
|
pub more_after: bool,
|
||||||
|
}
|
||||||
|
|
||||||
impl HuginnMindRequest {
|
impl HuginnMindRequest {
|
||||||
/// The envelope's `operation`.
|
/// The envelope's `operation`.
|
||||||
pub fn operation(&self) -> &'static str {
|
pub fn operation(&self) -> &'static str {
|
||||||
|
|
@ -281,6 +423,8 @@ impl HuginnMindRequest {
|
||||||
Self::PersonaGet { .. } => "persona_get",
|
Self::PersonaGet { .. } => "persona_get",
|
||||||
Self::PersonaPut { .. } => "persona_put",
|
Self::PersonaPut { .. } => "persona_put",
|
||||||
Self::SearchSources { .. } => "search_sources",
|
Self::SearchSources { .. } => "search_sources",
|
||||||
|
Self::SearchChat { .. } => "search_chat",
|
||||||
|
Self::ChatWindow { .. } => "chat_window",
|
||||||
Self::StewardedActivity { .. } => "stewarded_activity",
|
Self::StewardedActivity { .. } => "stewarded_activity",
|
||||||
Self::RootsTouched { .. } => "roots_touched",
|
Self::RootsTouched { .. } => "roots_touched",
|
||||||
}
|
}
|
||||||
|
|
@ -296,6 +440,8 @@ impl HuginnMindRequest {
|
||||||
| Self::PersonaGet { instance }
|
| Self::PersonaGet { instance }
|
||||||
| Self::PersonaPut { instance, .. }
|
| Self::PersonaPut { instance, .. }
|
||||||
| Self::SearchSources { instance, .. }
|
| Self::SearchSources { instance, .. }
|
||||||
|
| Self::SearchChat { instance, .. }
|
||||||
|
| Self::ChatWindow { instance, .. }
|
||||||
| Self::StewardedActivity { instance, .. }
|
| Self::StewardedActivity { instance, .. }
|
||||||
| Self::RootsTouched { instance } => Some(instance),
|
| Self::RootsTouched { instance } => Some(instance),
|
||||||
}
|
}
|
||||||
|
|
@ -315,6 +461,8 @@ pub enum HuginnMindResponse {
|
||||||
Persona(Option<PersonaStateView>),
|
Persona(Option<PersonaStateView>),
|
||||||
PersonaPut(PersonaPutOutcome),
|
PersonaPut(PersonaPutOutcome),
|
||||||
Sources(SourceHits),
|
Sources(SourceHits),
|
||||||
|
Chat(ChatHits),
|
||||||
|
ChatWindow(ChatWindow),
|
||||||
Activity(RepoActivity),
|
Activity(RepoActivity),
|
||||||
RootsTouched(Vec<RootTouched>),
|
RootsTouched(Vec<RootTouched>),
|
||||||
/// The answer did not fit one send and is being held for the client to
|
/// The answer did not fit one send and is being held for the client to
|
||||||
|
|
@ -450,6 +598,11 @@ mod tests {
|
||||||
instance: slug(INSTANCE),
|
instance: slug(INSTANCE),
|
||||||
query: SourceQuery { text: Line::default(), texts: vec![Line("reconcile".into()), Line("fuse".into())], top_k: 5, repo: None, front: false },
|
query: SourceQuery { text: Line::default(), texts: vec![Line("reconcile".into()), Line("fuse".into())], top_k: 5, repo: None, front: false },
|
||||||
},
|
},
|
||||||
|
HuginnMindRequest::SearchChat {
|
||||||
|
instance: slug(INSTANCE),
|
||||||
|
query: ChatQuery { texts: vec![Line("release".into())], rooms: vec![Short("#swarm:gamecult.org".into())], after_ms: Some(1), before_ms: None, top_k: 5 },
|
||||||
|
},
|
||||||
|
HuginnMindRequest::ChatWindow { instance: slug(INSTANCE), query: ChatWindowQuery { event_id: Short("$a".into()), before: 5, after: 0 } },
|
||||||
HuginnMindRequest::RootsTouched { instance: slug(INSTANCE) },
|
HuginnMindRequest::RootsTouched { instance: slug(INSTANCE) },
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
|
|
@ -500,6 +653,28 @@ mod tests {
|
||||||
text: "x".into(),
|
text: "x".into(),
|
||||||
}],
|
}],
|
||||||
}),
|
}),
|
||||||
|
HuginnMindResponse::Chat(ChatHits {
|
||||||
|
indexed_under: SourceModel { name: "m".into(), digest: "d".into() },
|
||||||
|
truncated: false,
|
||||||
|
hits: vec![ChatHit {
|
||||||
|
room_id: "!r:gamecult.org".into(),
|
||||||
|
room_alias: None,
|
||||||
|
first_event_id: "$a".into(),
|
||||||
|
last_event_id: "$a".into(),
|
||||||
|
ts_start_ms: 1,
|
||||||
|
ts_end_ms: 1,
|
||||||
|
score: None,
|
||||||
|
messages: vec![ChatMessage { event_id: "$a".into(), sender: "@x:gamecult.org".into(), ts_ms: 1, body: "hi".into() }],
|
||||||
|
}],
|
||||||
|
}),
|
||||||
|
HuginnMindResponse::ChatWindow(ChatWindow {
|
||||||
|
room_id: "!r:gamecult.org".into(),
|
||||||
|
room_alias: Some("#swarm:gamecult.org".into()),
|
||||||
|
anchor_event_id: "$a".into(),
|
||||||
|
messages: vec![ChatMessage { event_id: "$a".into(), sender: "@x:gamecult.org".into(), ts_ms: 1, body: "hi".into() }],
|
||||||
|
more_before: false,
|
||||||
|
more_after: true,
|
||||||
|
}),
|
||||||
HuginnMindResponse::RootsTouched(seeded().roots_touched().unwrap()),
|
HuginnMindResponse::RootsTouched(seeded().roots_touched().unwrap()),
|
||||||
HuginnMindResponse::Deferred(DeferredAnswer {
|
HuginnMindResponse::Deferred(DeferredAnswer {
|
||||||
manifest: cultnet_rs::pack_content("huginn.mind_response", "package", "", "", "", b"body", 4)
|
manifest: cultnet_rs::pack_content("huginn.mind_response", "package", "", "", "", b"body", 4)
|
||||||
|
|
@ -515,7 +690,7 @@ mod tests {
|
||||||
#[test]
|
#[test]
|
||||||
fn the_wire_vocabulary_is_the_minds_methods_and_status_is_derived_from_the_response() {
|
fn the_wire_vocabulary_is_the_minds_methods_and_status_is_derived_from_the_response() {
|
||||||
let names: Vec<&str> = requests().iter().map(|request| request.operation()).collect();
|
let names: Vec<&str> = requests().iter().map(|request| request.operation()).collect();
|
||||||
assert_eq!(names, ["whoami", "admit", "view", "query", "persona_get", "persona_put", "search_sources", "search_sources", "roots_touched"]);
|
assert_eq!(names, ["whoami", "admit", "view", "query", "persona_get", "persona_put", "search_sources", "search_sources", "search_chat", "chat_window", "roots_touched"]);
|
||||||
for request in requests() {
|
for request in requests() {
|
||||||
let declared = request.instance().is_none();
|
let declared = request.instance().is_none();
|
||||||
assert_eq!(declared, matches!(request, HuginnMindRequest::Whoami), "{request:?}");
|
assert_eq!(declared, matches!(request, HuginnMindRequest::Whoami), "{request:?}");
|
||||||
|
|
@ -625,6 +800,54 @@ mod tests {
|
||||||
assert_eq!((page.matched, page.as_of, page.scores), (0, 7, None));
|
assert_eq!((page.matched, page.as_of, page.scores), (0, 7, None));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// A chat search is refused by field at the door, and the published
|
||||||
|
/// response marks every citation required.
|
||||||
|
#[test]
|
||||||
|
fn a_chat_query_is_refused_by_field_and_a_hit_cites_room_ids_and_times() {
|
||||||
|
let ok = ChatQuery { texts: vec![], rooms: vec![Short("!a:b".into()), Short("#c:b".into())], after_ms: Some(1), before_ms: Some(1), top_k: 50 };
|
||||||
|
assert!(ok.check().is_ok());
|
||||||
|
let field = |query: ChatQuery| match query.check() {
|
||||||
|
Err(MindRefusal::SelectionInvalid { field, .. }) => field,
|
||||||
|
other => panic!("{other:?}"),
|
||||||
|
};
|
||||||
|
assert_eq!(field(ChatQuery { rooms: vec![Short("swarm".into())], ..ok.clone() }), "rooms");
|
||||||
|
assert_eq!(field(ChatQuery { rooms: vec![Short("#".into())], ..ok.clone() }), "rooms");
|
||||||
|
assert_eq!(field(ChatQuery { before_ms: Some(0), ..ok.clone() }), "before_ms");
|
||||||
|
assert_eq!(field(ChatQuery { top_k: 0, ..ok.clone() }), "top_k");
|
||||||
|
assert_eq!(field(ChatQuery { top_k: 51, ..ok.clone() }), "top_k");
|
||||||
|
assert_eq!(field(ChatQuery { texts: vec![Line(" ".into())], ..ok.clone() }), "texts");
|
||||||
|
assert_eq!(field(ChatQuery { texts: vec![Line("x".into()); 9], ..ok.clone() }), "texts");
|
||||||
|
assert_eq!(field(ChatQuery { rooms: vec![Short("!a:b".into()); 17], ..ok }), "rooms");
|
||||||
|
let response = serde_json::to_value(schemars::schema_for!(HuginnMindResponse)).unwrap();
|
||||||
|
let required: Vec<&str> = response["$defs"]["ChatHit"]["required"].as_array().unwrap().iter().filter_map(|v| v.as_str()).collect();
|
||||||
|
for field in ["room_id", "first_event_id", "last_event_id", "ts_start_ms", "ts_end_ms", "messages"] {
|
||||||
|
assert!(required.contains(&field), "{field} must be required");
|
||||||
|
}
|
||||||
|
let message: Vec<&str> = response["$defs"]["ChatMessage"]["required"].as_array().unwrap().iter().filter_map(|v| v.as_str()).collect();
|
||||||
|
assert_eq!(message, ["event_id", "sender", "ts_ms", "body"]);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A chat window is refused by field at the door.
|
||||||
|
#[test]
|
||||||
|
fn a_chat_window_query_is_refused_by_field() {
|
||||||
|
let ok = ChatWindowQuery { event_id: Short("$a".into()), before: 50, after: 0 };
|
||||||
|
assert!(ok.check().is_ok());
|
||||||
|
let field = |query: ChatWindowQuery| match query.check() {
|
||||||
|
Err(MindRefusal::SelectionInvalid { field, .. }) => field,
|
||||||
|
other => panic!("{other:?}"),
|
||||||
|
};
|
||||||
|
assert_eq!(field(ChatWindowQuery { before: 51, ..ok.clone() }), "before");
|
||||||
|
assert_eq!(field(ChatWindowQuery { after: 51, ..ok.clone() }), "after");
|
||||||
|
assert_eq!(field(ChatWindowQuery { event_id: Short("a".into()), ..ok.clone() }), "event_id");
|
||||||
|
assert_eq!(field(ChatWindowQuery { event_id: Short(format!("${}", "a".repeat(200))), ..ok.clone() }), "event_id");
|
||||||
|
assert!(ChatWindowQuery { event_id: Short(format!("${}", "a".repeat(199))), ..ok }.check().is_ok());
|
||||||
|
let response = serde_json::to_value(schemars::schema_for!(HuginnMindResponse)).unwrap();
|
||||||
|
let required: Vec<&str> = response["$defs"]["ChatWindow"]["required"].as_array().unwrap().iter().filter_map(|v| v.as_str()).collect();
|
||||||
|
for field in ["room_id", "anchor_event_id", "messages", "more_before", "more_after"] {
|
||||||
|
assert!(required.contains(&field), "{field} must be required");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// The two files the catalog publishes are the derivation of the two
|
/// The two files the catalog publishes are the derivation of the two
|
||||||
/// types, byte for byte.
|
/// types, byte for byte.
|
||||||
#[test]
|
#[test]
|
||||||
|
|
|
||||||
665
crates/huginn-mind/tests/desk_feed_fixtures.rs
Normal file
665
crates/huginn-mind/tests/desk_feed_fixtures.rs
Normal file
|
|
@ -0,0 +1,665 @@
|
||||||
|
//! The desk feed's golden fixtures: the bytes Bifrost writes to a Mind's desk socket for a list or
|
||||||
|
//! a read, and the bytes Huginn writes back, built here with Huginn's own encoders so the files
|
||||||
|
//! are what the door speaks. `docs/desk-feed-door.md` is the contract they pin; the README in the
|
||||||
|
//! fixtures directory lists them.
|
||||||
|
//!
|
||||||
|
//! The entry-bearing fixtures carry the reader-count shapes (`raised_at`, `decider_count`,
|
||||||
|
//! `answerer_count`, a nullable `answerer`), a decider's view and a non-decider's. CultLib's own bytes
|
||||||
|
//! for those shapes are vendored at one commit in `fixtures/desk-feed/cultlib-d3025d3c/`; Huginn's
|
||||||
|
//! real types decode and re-encode every one of them byte for byte (checked below).
|
||||||
|
//!
|
||||||
|
//! `cargo test -p huginn-mind --test desk_feed_fixtures` compares every file with what the
|
||||||
|
//! encoders produce now. `HUGINN_WRITE_DESK_FIXTURES=1` rewrites them instead.
|
||||||
|
|
||||||
|
use std::path::PathBuf;
|
||||||
|
|
||||||
|
use sha2::{Digest, Sha256};
|
||||||
|
|
||||||
|
use cultnet_rs::{CultNetMessage, CultNetWireContract, encode_cultnet_message_to_vec, encode_frame};
|
||||||
|
use huginn_mind::MindRefusal;
|
||||||
|
use huginn_mind::desk_wire::{
|
||||||
|
fork_too_large_read, AnswerSchema, CardSchema, CardStatus, DeskRequest, DeskResponse, EntrySchema, ForkCard, ForkCursor, ForkDeskAnswer, ForkEntry, ForkLimit,
|
||||||
|
ForkList, ForkOption, ForkRecord, ForkRefusal, ForkRefusalCode, ForkRef, ListSchema, ListStatus, RecordSchema,
|
||||||
|
};
|
||||||
|
use huginn_mind::envelope::{OperationFailure, decode_desk_request, encode_desk_failure, encode_desk_request, encode_desk_response};
|
||||||
|
use huginn_mind::eureka_pipeline::{AccountId, DeskAuthority, Slug};
|
||||||
|
|
||||||
|
const RUNTIME: &str = "huginn-eureka";
|
||||||
|
/// Account ids as Bifrost sends them: `UserAccount.Id` rendered lowercase D-format
|
||||||
|
/// (`DeskEndpoints.cs` `accountId.Value.ToString("D")`; `StartupConfigurationValidator` refuses any
|
||||||
|
/// other decider). Synthetic values.
|
||||||
|
const READER: &str = "5f0c2b1e-8a4d-4c3b-9e7f-1a2b3c4d5e6f";
|
||||||
|
const OTHER_DECIDER: &str = "0b9d7e3a-6c21-4f58-a0d4-9e8f7a6b5c4d";
|
||||||
|
/// A member who answered but is not a decider: their answer is stored `Refused(not_a_decider)`.
|
||||||
|
const OUTSIDER: &str = "c3a1f2e4-7b6d-4e8f-9a0b-1c2d3e4f5a6b";
|
||||||
|
/// A member the call's deciders do not name, who reads as a non-decider.
|
||||||
|
const NON_DECIDER: &str = "9e8d7c6b-5a49-4382-b1a0-f9e8d7c6b5a4";
|
||||||
|
/// A question Eureka's mind holds (asked 2026-11-11). The answers, notes and hash in `record()` are
|
||||||
|
/// synthetic: the hash is CultLib's sample digest, not this question's.
|
||||||
|
const FORK: &str = "eureka-substrate:question:desk-listing-authority";
|
||||||
|
|
||||||
|
fn a(text: &str) -> AccountId {
|
||||||
|
AccountId(text.into())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn forks(status: Option<ListStatus>, cursor: Option<u64>, limit: u8) -> DeskRequest {
|
||||||
|
DeskRequest::Forks {
|
||||||
|
mind: Slug("eureka".into()),
|
||||||
|
subject: a(READER),
|
||||||
|
deciders: vec![a(READER), a(OTHER_DECIDER)],
|
||||||
|
status,
|
||||||
|
cursor: cursor.and_then(ForkCursor::new),
|
||||||
|
limit: ForkLimit::new(limit).unwrap(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn read() -> DeskRequest {
|
||||||
|
DeskRequest::Fork { mind: Slug("eureka".into()), subject: a(READER), deciders: vec![a(READER), a(OTHER_DECIDER)], fork_id: FORK.into() }
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The same reads from a subject the call's deciders do not name: a non-decider.
|
||||||
|
fn forks_as_non_decider() -> DeskRequest {
|
||||||
|
DeskRequest::Forks {
|
||||||
|
mind: Slug("eureka".into()),
|
||||||
|
subject: a(NON_DECIDER),
|
||||||
|
deciders: vec![a(READER), a(OTHER_DECIDER)],
|
||||||
|
status: Some(ListStatus::Open),
|
||||||
|
cursor: None,
|
||||||
|
limit: ForkLimit::new(50).unwrap(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn read_as_non_decider() -> DeskRequest {
|
||||||
|
DeskRequest::Fork { mind: Slug("eureka".into()), subject: a(NON_DECIDER), deciders: vec![a(READER), a(OTHER_DECIDER)], fork_id: FORK.into() }
|
||||||
|
}
|
||||||
|
|
||||||
|
fn list(open_count: u64, recorded_count: u64) -> DeskResponse {
|
||||||
|
DeskResponse::Forks {
|
||||||
|
list: ForkList {
|
||||||
|
schema_version: ListSchema::V0,
|
||||||
|
mind: "eureka".into(),
|
||||||
|
stamp: "s10190".into(),
|
||||||
|
open_count,
|
||||||
|
recorded_count,
|
||||||
|
forks: Vec::new(),
|
||||||
|
next_cursor: None,
|
||||||
|
},
|
||||||
|
as_of: 10190,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn refusal(code: ForkRefusalCode, error: &str) -> DeskResponse {
|
||||||
|
DeskResponse::NotStored(ForkRefusal { schema_version: huginn_mind::desk_wire::RefusalSchema::V0, code, error: error.into() })
|
||||||
|
}
|
||||||
|
|
||||||
|
fn answer(id: &str, outcome: &str, of: Option<&str>, answerer: Option<&str>, note: &str, received_at: &str) -> ForkDeskAnswer {
|
||||||
|
ForkDeskAnswer {
|
||||||
|
schema_version: AnswerSchema::V0,
|
||||||
|
answer_id: id.into(),
|
||||||
|
fork_id: FORK.into(),
|
||||||
|
outcome: outcome.into(),
|
||||||
|
of: of.map(str::to_string),
|
||||||
|
retracts: None,
|
||||||
|
code: None,
|
||||||
|
options_hash: "7d95fc87d532420611fe415c1def27a3447617bb68235f46e3af130aea361c48".into(),
|
||||||
|
choice: Some("asking-mind".into()),
|
||||||
|
note: Some(note.into()),
|
||||||
|
answered_at: received_at.into(),
|
||||||
|
received_at: received_at.into(),
|
||||||
|
claim_in_future: false,
|
||||||
|
answerer: answerer.map(str::to_string),
|
||||||
|
authority: DeskAuthority::Member,
|
||||||
|
request_id: format!("00000000-0000-4000-8000-00000000000{}", &id[1..]),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A fork's record with a counted answer, a decider's revision, a refusal and a retraction: four
|
||||||
|
/// answers by three accounts. A decider's view names every answerer; a non-decider's names none and
|
||||||
|
/// keeps the count.
|
||||||
|
fn record(decider: bool) -> ForkRecord {
|
||||||
|
let who = |id: &'static str| decider.then_some(id);
|
||||||
|
ForkRecord {
|
||||||
|
schema_version: RecordSchema::V0,
|
||||||
|
fork_id: FORK.into(),
|
||||||
|
options_hash: HASH.into(),
|
||||||
|
counted_answer_id: Some("n1".into()),
|
||||||
|
answerer_count: 3,
|
||||||
|
answers: vec![
|
||||||
|
answer("n1", "Counted", None, who(READER), "The asker lists her own forks.", "2026-11-11T17:00:00Z"),
|
||||||
|
answer("n2", "Revision", Some("n1"), who(OTHER_DECIDER), "Agreed.", "2026-11-11T17:05:00Z"),
|
||||||
|
ForkDeskAnswer { code: Some(huginn_mind::desk_wire::ForkAnswerCode::NotADecider), ..answer("n3", "Refused", None, who(OUTSIDER), "Me too.", "2026-11-11T17:06:00Z") },
|
||||||
|
// A second answer by an account already counted: four answers, three answerers.
|
||||||
|
ForkDeskAnswer {
|
||||||
|
retracts: Some("n2".into()),
|
||||||
|
choice: None,
|
||||||
|
note: None,
|
||||||
|
..answer("n4", "Retraction", None, who(OTHER_DECIDER), "", "2026-11-11T17:07:00Z")
|
||||||
|
},
|
||||||
|
],
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const HASH: &str = "7d95fc87d532420611fe415c1def27a3447617bb68235f46e3af130aea361c48";
|
||||||
|
|
||||||
|
/// The fork's card. A decider's view lists the call's ids; a non-decider's lists none and keeps the count.
|
||||||
|
fn card(decider: bool, raised_at: Option<&str>) -> ForkCard {
|
||||||
|
ForkCard {
|
||||||
|
schema_version: CardSchema::V0,
|
||||||
|
fork_id: FORK.into(),
|
||||||
|
title: "Who may list a fork?".into(),
|
||||||
|
context: "A fork is read by members only once it is listed.".into(),
|
||||||
|
options: vec![ForkOption { id: "asking-mind".into(), text: "The asking Mind lists her own forks.".into() }, ForkOption { id: "operator".into(), text: "An operator lists them.".into() }],
|
||||||
|
recommendation: "asking-mind".into(),
|
||||||
|
depends: Vec::new(),
|
||||||
|
leans: None,
|
||||||
|
flag: None,
|
||||||
|
raised_in: Some(ForkRef { kind: "Idea".into(), id: "eureka-substrate:idea:desk-door-map".into() }),
|
||||||
|
raised_at: raised_at.map(str::to_string),
|
||||||
|
deciders: if decider { vec![READER.into(), OTHER_DECIDER.into()] } else { Vec::new() },
|
||||||
|
decider_count: 2,
|
||||||
|
status: CardStatus::Open,
|
||||||
|
ruling: None,
|
||||||
|
ruled_from: Vec::new(),
|
||||||
|
order: 41,
|
||||||
|
options_hash: HASH.into(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn entry(decider: bool, raised_at: Option<&str>) -> ForkEntry {
|
||||||
|
ForkEntry { schema_version: EntrySchema::V0, card: card(decider, raised_at), record: record(decider) }
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One open fork on the page; the counts are the page's visible set.
|
||||||
|
fn list_of(entry: ForkEntry) -> DeskResponse {
|
||||||
|
DeskResponse::Forks {
|
||||||
|
list: ForkList { schema_version: ListSchema::V0, mind: "eureka".into(), stamp: "s10190".into(), open_count: 1, recorded_count: 0, forks: vec![entry], next_cursor: None },
|
||||||
|
as_of: 10190,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The named MessagePack payload a CultNet operation message carries (base64 in the envelope).
|
||||||
|
fn payload(message: &CultNetMessage) -> Vec<u8> {
|
||||||
|
use base64::Engine;
|
||||||
|
let (CultNetMessage::OperationRequest { payload, .. } | CultNetMessage::OperationResponse { payload, .. }) = message else {
|
||||||
|
panic!("an operation message")
|
||||||
|
};
|
||||||
|
base64::engine::general_purpose::STANDARD.decode(payload).unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn frame(message: &CultNetMessage) -> Vec<u8> {
|
||||||
|
encode_frame(&encode_cultnet_message_to_vec(message, CultNetWireContract::CultNetSchemaV0).unwrap()).unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn hex(bytes: &[u8]) -> String {
|
||||||
|
let mut text: String = bytes.iter().map(|byte| format!("{byte:02x}")).collect();
|
||||||
|
text.push('\n');
|
||||||
|
text
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A request message whose payload is `request` with one field changed to something the typed request
|
||||||
|
/// cannot hold: what a client that skips the account grammar would send.
|
||||||
|
fn malformed(id: &str, request: &DeskRequest, edit: impl Fn(&mut serde_json::Value)) -> CultNetMessage {
|
||||||
|
use base64::Engine;
|
||||||
|
let mut value = serde_json::to_value(request).unwrap();
|
||||||
|
edit(&mut value);
|
||||||
|
CultNetMessage::OperationRequest {
|
||||||
|
message_id: id.into(),
|
||||||
|
service_id: huginn_mind::desk_wire::DESK_SERVICE_ID.into(),
|
||||||
|
operation: "fork".into(),
|
||||||
|
payload_schema: huginn_mind::desk_wire::DESK_REQUEST_SCHEMA.into(),
|
||||||
|
payload_encoding: "messagepack-base64".into(),
|
||||||
|
payload: base64::engine::general_purpose::STANDARD.encode(rmp_serde::to_vec_named(&value).unwrap()),
|
||||||
|
source_runtime_id: None,
|
||||||
|
target_runtime_id: None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Every fixture: its file stem and its bytes. A message gives `.frame.bin`, `.frame.hex` and,
|
||||||
|
/// for an operation message, `.payload.msgpack` and `.payload.hex`; a bare payload gives the last two.
|
||||||
|
fn fixtures() -> Vec<(String, Vec<u8>)> {
|
||||||
|
let mut files = Vec::new();
|
||||||
|
let mut message = |name: &str, message: CultNetMessage| {
|
||||||
|
let framed = frame(&message);
|
||||||
|
files.push((format!("{name}.frame.bin"), framed.clone()));
|
||||||
|
files.push((format!("{name}.frame.hex"), hex(&framed).into_bytes()));
|
||||||
|
if matches!(message, CultNetMessage::OperationRequest { .. } | CultNetMessage::OperationResponse { .. }) {
|
||||||
|
let inner = payload(&message);
|
||||||
|
files.push((format!("{name}.payload.msgpack"), inner.clone()));
|
||||||
|
files.push((format!("{name}.payload.hex"), hex(&inner).into_bytes()));
|
||||||
|
}
|
||||||
|
};
|
||||||
|
message("list-open.request", encode_desk_request("m-1", &forks(Some(ListStatus::Open), None, 50), None).unwrap());
|
||||||
|
message("list-open-cursor.request", encode_desk_request("m-2", &forks(Some(ListStatus::Open), Some(42), 50), None).unwrap());
|
||||||
|
message("list-counts.request", encode_desk_request("m-3", &forks(Some(ListStatus::Open), None, 0), None).unwrap());
|
||||||
|
message("read.request", encode_desk_request("m-4", &read(), None).unwrap());
|
||||||
|
message("list-non-decider.request", encode_desk_request("m-5", &forks_as_non_decider(), None).unwrap());
|
||||||
|
message("read-non-decider.request", encode_desk_request("m-6", &read_as_non_decider(), None).unwrap());
|
||||||
|
message("list-open.response", encode_desk_response("m-1", "forks", &list_of(entry(true, Some("2026-11-11T16:40:12Z"))), RUNTIME).unwrap());
|
||||||
|
message("list-open-non-decider.response", encode_desk_response("m-5", "forks", &list_of(entry(false, Some("2026-11-11T16:40:12Z"))), RUNTIME).unwrap());
|
||||||
|
// raised_at nil: the question's first admission is unknown.
|
||||||
|
message("read.response", encode_desk_response("m-4", "fork", &DeskResponse::Fork { entry: entry(true, None), as_of: 10190 }, RUNTIME).unwrap());
|
||||||
|
message("read-non-decider.response", encode_desk_response("m-6", "fork", &DeskResponse::Fork { entry: entry(false, None), as_of: 10190 }, RUNTIME).unwrap());
|
||||||
|
// A decider or subject that is no account id is refused at decode, naming the field.
|
||||||
|
for (name, id, request) in [
|
||||||
|
("read-mxid-decider", "m-7", malformed("m-7", &read(), |value| value["Fork"]["deciders"][0] = "@heimdall:gamecult.org".into())),
|
||||||
|
("read-email-subject", "m-8", malformed("m-8", &read(), |value| value["Fork"]["subject"] = "meta@gamecult.org".into())),
|
||||||
|
] {
|
||||||
|
message(&format!("{name}.request"), request.clone());
|
||||||
|
let failure = decode_desk_request(&request).expect_err("the door refuses it");
|
||||||
|
message(&format!("{name}.failure.response"), encode_desk_failure(id, "fork", &failure, RUNTIME));
|
||||||
|
}
|
||||||
|
message("list-empty.response", encode_desk_response("m-1", "forks", &list(0, 0), RUNTIME).unwrap());
|
||||||
|
message("list-counts.response", encode_desk_response("m-3", "forks", &list(3, 5), RUNTIME).unwrap());
|
||||||
|
message(
|
||||||
|
"read-unknown.response",
|
||||||
|
encode_desk_response("m-4", "fork", &refusal(ForkRefusalCode::ForkUnknown, "fork_unknown: no such fork"), RUNTIME).unwrap(),
|
||||||
|
);
|
||||||
|
message(
|
||||||
|
"list-too-large.response",
|
||||||
|
encode_desk_response(
|
||||||
|
"m-1",
|
||||||
|
"forks",
|
||||||
|
&refusal(ForkRefusalCode::ForkTooLarge, &format!("fork_too_large: fork {FORK} at order 41 is past the page budget by itself; continue with cursor c41")),
|
||||||
|
RUNTIME,
|
||||||
|
)
|
||||||
|
.unwrap(),
|
||||||
|
);
|
||||||
|
message("read-too-large.response", encode_desk_response("m-4", "fork", &DeskResponse::NotStored(fork_too_large_read(FORK)), RUNTIME).unwrap());
|
||||||
|
message(
|
||||||
|
"list-response-too-large.response",
|
||||||
|
encode_desk_response("m-1", "forks", &DeskResponse::Refused(MindRefusal::ResponseTooLarge { bytes: 70000, limit: 65536 }), RUNTIME).unwrap(),
|
||||||
|
);
|
||||||
|
message("refused-unit.response", encode_desk_response("m-1", "forks", &DeskResponse::Refused(MindRefusal::MissingIdentity), RUNTIME).unwrap());
|
||||||
|
message(
|
||||||
|
"unavailable.response",
|
||||||
|
encode_desk_response("m-1", "forks", &DeskResponse::Refused(MindRefusal::Unavailable { detail: "the mind store is not readable".into() }), RUNTIME).unwrap(),
|
||||||
|
);
|
||||||
|
message(
|
||||||
|
"failure.response",
|
||||||
|
encode_desk_failure("m-1", "forks", &OperationFailure { code: "payload-not-a-request".into(), message: "field Forks.limit: out-of-range".into() }, RUNTIME),
|
||||||
|
);
|
||||||
|
message("error.response", CultNetMessage::Error { error: "huginn.desk refuses a message_id over 256 bytes".into(), code: None, details: None });
|
||||||
|
for (name, decider) in [("record-whole", true), ("record-withheld", false)] {
|
||||||
|
let bytes = rmp_serde::to_vec_named(&record(decider)).unwrap();
|
||||||
|
files.push((format!("{name}.payload.msgpack"), bytes.clone()));
|
||||||
|
files.push((format!("{name}.payload.hex"), hex(&bytes).into_bytes()));
|
||||||
|
}
|
||||||
|
files
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn desk_feed_fixtures_are_what_the_encoders_write() {
|
||||||
|
let dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/desk-feed");
|
||||||
|
let write = std::env::var_os("HUGINN_WRITE_DESK_FIXTURES").is_some();
|
||||||
|
if write {
|
||||||
|
std::fs::create_dir_all(&dir).unwrap();
|
||||||
|
}
|
||||||
|
for (name, bytes) in fixtures() {
|
||||||
|
let path = dir.join(&name);
|
||||||
|
if write {
|
||||||
|
std::fs::write(&path, &bytes).unwrap();
|
||||||
|
} else {
|
||||||
|
let held = std::fs::read(&path).unwrap_or_else(|error| panic!("{}: {error}", path.display()));
|
||||||
|
assert!(held == bytes, "{name} differs from what the encoders write now");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The one grammar for `subject`, every `deciders` entry and every `answerer`: a lowercase
|
||||||
|
/// D-format UUID (8-4-4-4-12 lowercase hex), the grammar Huginn already names for `RequestId`
|
||||||
|
/// (`eureka-pipeline` `uuid_text`). A Matrix id, an email or a handle never fits it.
|
||||||
|
fn account_id(text: &str) -> bool {
|
||||||
|
text.len() == 36
|
||||||
|
&& text.bytes().enumerate().all(|(at, byte)| match at {
|
||||||
|
8 | 13 | 18 | 23 => byte == b'-',
|
||||||
|
_ => matches!(byte, b'0'..=b'9' | b'a'..=b'f'),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Every account id the fixtures carry is a Bifrost account id, never a Matrix id, email or
|
||||||
|
/// handle, and the door refuses any other shape at decode, naming the field and never the value.
|
||||||
|
#[test]
|
||||||
|
fn every_account_id_in_the_fixtures_is_a_lowercase_d_format_uuid() {
|
||||||
|
for request in [forks(Some(ListStatus::Open), None, 50), read()] {
|
||||||
|
let subject = match &request {
|
||||||
|
DeskRequest::Forks { subject, .. } | DeskRequest::Fork { subject, .. } | DeskRequest::Answer { subject, .. } => subject,
|
||||||
|
};
|
||||||
|
assert!(account_id(&subject.0), "subject");
|
||||||
|
assert!(request.deciders().iter().all(|decider| account_id(&decider.0)), "deciders");
|
||||||
|
}
|
||||||
|
assert!(record(true).answers.iter().all(|answer| account_id(answer.answerer.as_deref().unwrap())), "answerer");
|
||||||
|
for (request, field, value) in [
|
||||||
|
(malformed("m-7", &read(), |value| value["Fork"]["deciders"][0] = "@heimdall:gamecult.org".into()), "deciders", "@heimdall:gamecult.org"),
|
||||||
|
(malformed("m-8", &read(), |value| value["Fork"]["subject"] = "meta@gamecult.org".into()), "subject", "meta@gamecult.org"),
|
||||||
|
] {
|
||||||
|
let failure = decode_desk_request(&request).expect_err("a request outside the grammar is refused");
|
||||||
|
assert_eq!(failure.code, "payload-not-a-request");
|
||||||
|
assert!(failure.message.contains(field) && !failure.message.contains(value), "{}", failure.message);
|
||||||
|
}
|
||||||
|
for shape in [
|
||||||
|
"@heimdall:gamecult.org",
|
||||||
|
"meta@gamecult.org",
|
||||||
|
"heimdall",
|
||||||
|
"7",
|
||||||
|
"5F0C2B1E-8A4D-4C3B-9E7F-1A2B3C4D5E6F",
|
||||||
|
"{5f0c2b1e-8a4d-4c3b-9e7f-1a2b3c4d5e6f}",
|
||||||
|
"",
|
||||||
|
// 36 hex digits and no dash: the length alone does not make an id.
|
||||||
|
"5f0c2b1e08a4d04c3b09e7f01a2b3c4d5e6f",
|
||||||
|
// 37 characters: a valid id with one more digit.
|
||||||
|
"5f0c2b1e-8a4d-4c3b-9e7f-1a2b3c4d5e6f0",
|
||||||
|
// A dash moved off its position.
|
||||||
|
"5f0c2b1e8-a4d-4c3b-9e7f-1a2b3c4d5e6f",
|
||||||
|
] {
|
||||||
|
assert!(!account_id(shape), "{shape} is not an account id");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
// ---- CultLib's bytes: the reader-count shapes, pinned at one commit. ----
|
||||||
|
|
||||||
|
/// The CultLib commit the vendored vectors come from (#118 on #117 on #115). Re-pin: see the README beside them.
|
||||||
|
const CULTLIB_SHA: &str = "d3025d3c19674803ad32eb2335461b8cbc22bd28";
|
||||||
|
|
||||||
|
/// Each vendored vector and the SHA-256 of its bytes.
|
||||||
|
const VENDORED: &[(&str, &str)] = &[
|
||||||
|
("card-operator-fork", "56bfdb65ded60b562001d09d0b8da6e092900f404bf5f68f7084dff29bcc34f5"),
|
||||||
|
("card-raised-at-nil", "b81bb825790ffe03fabe63c3f8a61c36bd50ca268aa6dff67b5bcfad3957e58c"),
|
||||||
|
("card-withheld-deciders", "d13b9b77461cfdf78f943ef56c6aeab852ab6b6f0b355e26c5b361f99c79c7f7"),
|
||||||
|
("answer-counted", "55d69ae3cc7ebbeeae7d501e8641640e115c3fb5aac399ea760588bca343b9e6"),
|
||||||
|
("answer-revision", "19d7289395ae68cbaf6de1c0a1ef6c843211b2caf9462e1d2268cd0883386315"),
|
||||||
|
("answer-retraction", "89156f41ec3ea54f5d47bd3318913ff013785f4727916875be97785935cb7735"),
|
||||||
|
("answer-withheld-answerer", "b66b4f921cdd374be04af3de34c59e254a5e542027ae8c57f196a3ea3406e427"),
|
||||||
|
("answer-withheld-answerer-revision", "93f7c875f963a7cfeab2a5980b6d5ce81e6753fc9f3f8bf0d7353f3f388144b7"),
|
||||||
|
("answer-withheld-answerer-retraction", "7d0cc555bf9b6f7c345b0da7c504bcd1ed7b46040ff51d1e57c84b2f56655213"),
|
||||||
|
("answer-refused-stale", "3183bc9c483f4a625332cda11ef22a9c8560868a7728b77daa63a1960a3cd3d6"),
|
||||||
|
("entry-open-counted", "b51f499e8dd0465cf5df9d68d8fccbe94dd1339fd3157969693dd0d4ecb2549b"),
|
||||||
|
("entry-withheld-open-counted", "62ccad7a9f41262d6e35d0247315d0bb75d1313b3c9dabe8a99d1aa1b844607f"),
|
||||||
|
("entry-closed-with-ruling-and-a-refusal", "f7859c7d04d338d6b1c634308b08ae58ae77b0ff459767859a5e40f0af2f16e5"),
|
||||||
|
("entry-withheld-closed-with-ruling-and-a-refusal", "daaa819bfa2612616826ef3217dfcf8d8d8cb71d86547d7106426c779134f827"),
|
||||||
|
];
|
||||||
|
|
||||||
|
fn vendored(name: &str) -> Vec<u8> {
|
||||||
|
let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/desk-feed/cultlib-d3025d3c").join(format!("{name}.msgpack"));
|
||||||
|
std::fs::read(&path).unwrap_or_else(|error| panic!("{}: {error}", path.display()))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A decoded MessagePack value that keeps map order. A small decoder of its own, so the key counts,
|
||||||
|
/// headers and positions below are read off the bytes and not off `rmp_serde`'s idea of the types.
|
||||||
|
#[derive(Debug, PartialEq)]
|
||||||
|
enum Value {
|
||||||
|
Nil,
|
||||||
|
Bool(bool),
|
||||||
|
Int(i64),
|
||||||
|
Str(String),
|
||||||
|
List(Vec<Value>),
|
||||||
|
Map(Vec<(String, Value)>),
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Value {
|
||||||
|
fn get(&self, key: &str) -> &Value {
|
||||||
|
let Value::Map(pairs) = self else { panic!("not a map") };
|
||||||
|
&pairs.iter().find(|(name, _)| name == key).unwrap_or_else(|| panic!("no key {key}")).1
|
||||||
|
}
|
||||||
|
fn keys(&self) -> Vec<&str> {
|
||||||
|
let Value::Map(pairs) = self else { panic!("not a map") };
|
||||||
|
pairs.iter().map(|(name, _)| name.as_str()).collect()
|
||||||
|
}
|
||||||
|
fn list(&self) -> &[Value] {
|
||||||
|
let Value::List(items) = self else { panic!("not a list") };
|
||||||
|
items
|
||||||
|
}
|
||||||
|
fn str(&self) -> &str {
|
||||||
|
let Value::Str(text) = self else { panic!("not a string: {self:?}") };
|
||||||
|
text
|
||||||
|
}
|
||||||
|
fn int(&self) -> i64 {
|
||||||
|
let Value::Int(number) = self else { panic!("not an int: {self:?}") };
|
||||||
|
*number
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn decode(bytes: &[u8]) -> Value {
|
||||||
|
fn take<'a>(bytes: &'a [u8], at: &mut usize, n: usize) -> &'a [u8] {
|
||||||
|
let slice = &bytes[*at..*at + n];
|
||||||
|
*at += n;
|
||||||
|
slice
|
||||||
|
}
|
||||||
|
fn be(slice: &[u8]) -> u64 {
|
||||||
|
slice.iter().fold(0, |acc, byte| (acc << 8) | u64::from(*byte))
|
||||||
|
}
|
||||||
|
fn text(bytes: &[u8], at: &mut usize, n: usize) -> Value {
|
||||||
|
Value::Str(String::from_utf8(take(bytes, at, n).to_vec()).unwrap())
|
||||||
|
}
|
||||||
|
fn items(bytes: &[u8], at: &mut usize, n: usize) -> Vec<Value> {
|
||||||
|
(0..n).map(|_| value(bytes, at)).collect()
|
||||||
|
}
|
||||||
|
fn pairs(bytes: &[u8], at: &mut usize, n: usize) -> Value {
|
||||||
|
Value::Map((0..n).map(|_| (value(bytes, at).str().to_string(), value(bytes, at))).collect())
|
||||||
|
}
|
||||||
|
fn value(bytes: &[u8], at: &mut usize) -> Value {
|
||||||
|
let tag = take(bytes, at, 1)[0];
|
||||||
|
match tag {
|
||||||
|
0x00..=0x7f => Value::Int(i64::from(tag)),
|
||||||
|
0x80..=0x8f => pairs(bytes, at, usize::from(tag & 0x0f)),
|
||||||
|
0x90..=0x9f => Value::List(items(bytes, at, usize::from(tag & 0x0f))),
|
||||||
|
0xa0..=0xbf => text(bytes, at, usize::from(tag & 0x1f)),
|
||||||
|
0xc0 => Value::Nil,
|
||||||
|
0xc2 => Value::Bool(false),
|
||||||
|
0xc3 => Value::Bool(true),
|
||||||
|
0xcc => Value::Int(be(take(bytes, at, 1)) as i64),
|
||||||
|
0xcd => Value::Int(be(take(bytes, at, 2)) as i64),
|
||||||
|
0xce => Value::Int(be(take(bytes, at, 4)) as i64),
|
||||||
|
0xd9 => {
|
||||||
|
let n = be(take(bytes, at, 1)) as usize;
|
||||||
|
text(bytes, at, n)
|
||||||
|
}
|
||||||
|
0xda => {
|
||||||
|
let n = be(take(bytes, at, 2)) as usize;
|
||||||
|
text(bytes, at, n)
|
||||||
|
}
|
||||||
|
0xdc => {
|
||||||
|
let n = be(take(bytes, at, 2)) as usize;
|
||||||
|
Value::List(items(bytes, at, n))
|
||||||
|
}
|
||||||
|
0xde => {
|
||||||
|
let n = be(take(bytes, at, 2)) as usize;
|
||||||
|
pairs(bytes, at, n)
|
||||||
|
}
|
||||||
|
other => panic!("MessagePack tag {other:#04x} at {} is not in this decoder", *at - 1),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let mut at = 0;
|
||||||
|
let decoded = value(bytes, &mut at);
|
||||||
|
assert_eq!(at, bytes.len(), "trailing bytes");
|
||||||
|
decoded
|
||||||
|
}
|
||||||
|
|
||||||
|
const CARD_KEYS: [&str; 18] = [
|
||||||
|
"schemaVersion", "fork_id", "title", "context", "options", "recommendation", "depends", "leans", "flag", "raised_in", "raised_at", "deciders",
|
||||||
|
"decider_count", "status", "ruling", "ruled_from", "order", "options_hash",
|
||||||
|
];
|
||||||
|
const RECORD_KEYS: [&str; 6] = ["schemaVersion", "fork_id", "options_hash", "counted_answer_id", "answerer_count", "answers"];
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn the_vendored_cultlib_vectors_are_the_pinned_bytes() {
|
||||||
|
assert_eq!(CULTLIB_SHA.len(), 40);
|
||||||
|
let dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/desk-feed/cultlib-d3025d3c");
|
||||||
|
let docs = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../docs/desk-feed-door.md");
|
||||||
|
for path in [dir.join("README.md"), docs] {
|
||||||
|
let text = std::fs::read_to_string(&path).unwrap_or_else(|error| panic!("{}: {error}", path.display()));
|
||||||
|
assert!(text.contains(CULTLIB_SHA), "{} does not name the pinned sha {CULTLIB_SHA}", path.display());
|
||||||
|
}
|
||||||
|
assert!(dir.to_str().unwrap().ends_with(&format!("cultlib-{}", &CULTLIB_SHA[..8])), "the directory is named for the pin");
|
||||||
|
for (name, sha) in VENDORED {
|
||||||
|
let digest: String = Sha256::digest(vendored(name)).iter().map(|byte| format!("{byte:02x}")).collect();
|
||||||
|
assert_eq!(&digest, sha, "{name} is not CultLib {CULTLIB_SHA}'s bytes");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The card: header `DE 00 12` (map16, 18 entries: `schemaVersion` and 17 fields), the keys in this
|
||||||
|
/// order, `raised_at` a string or nil, `decider_count` 1..64 right after `deciders`, and `deciders` empty
|
||||||
|
/// (withheld) or exactly `decider_count` account ids.
|
||||||
|
fn check_card(card: &Value, withheld: bool) {
|
||||||
|
assert_eq!(card.keys(), CARD_KEYS);
|
||||||
|
assert!(matches!(card.get("raised_at"), Value::Nil | Value::Str(_)));
|
||||||
|
let count = card.get("decider_count").int();
|
||||||
|
assert!((1..=64).contains(&count));
|
||||||
|
let ids = card.get("deciders").list();
|
||||||
|
assert_eq!(ids.is_empty(), withheld);
|
||||||
|
assert!(ids.is_empty() || ids.len() as i64 == count);
|
||||||
|
assert!(ids.iter().all(|id| account_id(id.str())), "deciders are lowercase D-format UUIDs");
|
||||||
|
assert_eq!(card.get("options_hash").str().len(), 64);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn the_cards_are_eighteen_keys_with_raised_at_and_decider_count() {
|
||||||
|
for name in ["card-operator-fork", "card-raised-at-nil", "card-withheld-deciders"] {
|
||||||
|
let bytes = vendored(name);
|
||||||
|
assert_eq!(bytes[..3], [0xde, 0x00, 0x12], "{name}: header");
|
||||||
|
check_card(&decode(&bytes), name == "card-withheld-deciders");
|
||||||
|
}
|
||||||
|
assert!(matches!(decode(&vendored("card-operator-fork")).get("raised_at"), Value::Str(_)));
|
||||||
|
assert_eq!(decode(&vendored("card-raised-at-nil")).get("raised_at"), &Value::Nil);
|
||||||
|
assert_eq!(decode(&vendored("card-withheld-deciders")).get("decider_count").int(), 2);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// `answerer` is a string or nil on every answer; `Value::Nil` only when `named` is false. The bytes
|
||||||
|
/// null it on all four variants (`Counted`, `Revision`, `Retraction`, `Refused`), by ruling
|
||||||
|
/// `desk-reader-counts-shape` and now by vector.
|
||||||
|
fn check_answer(answer: &Value, named: bool) {
|
||||||
|
assert_eq!(answer.keys().len(), 16);
|
||||||
|
assert_eq!(answer.keys()[13], "answerer");
|
||||||
|
match answer.get("answerer") {
|
||||||
|
Value::Str(id) => assert!(named && account_id(id), "a named answerer is a lowercase D-format UUID"),
|
||||||
|
Value::Nil => assert!(!named),
|
||||||
|
other => panic!("answerer is {other:?}"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn the_answers_keep_sixteen_keys_and_answerer_is_a_string_or_nil() {
|
||||||
|
for (name, named) in [("answer-counted", true), ("answer-revision", true), ("answer-retraction", true), ("answer-refused-stale", true), ("answer-withheld-answerer", false), ("answer-withheld-answerer-revision", false), ("answer-withheld-answerer-retraction", false)] {
|
||||||
|
let bytes = vendored(name);
|
||||||
|
assert_eq!(bytes[..3], [0xde, 0x00, 0x10], "{name}: header");
|
||||||
|
check_answer(&decode(&bytes), named);
|
||||||
|
}
|
||||||
|
for (name, outcome) in [("answer-counted", "Counted"), ("answer-revision", "Revision"), ("answer-retraction", "Retraction"), ("answer-refused-stale", "Refused"), ("answer-withheld-answerer-revision", "Revision"), ("answer-withheld-answerer-retraction", "Retraction")] {
|
||||||
|
assert_eq!(decode(&vendored(name)).get("outcome").str(), outcome);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The entry is `83` and its record `86`, `answerer_count` the fifth key. In a decider's view the count
|
||||||
|
/// is the number of distinct answerer ids over every stored answer, refusals included; in a
|
||||||
|
/// non-decider's view no answerer is named and the count is between 1 and the number of answers (0 for
|
||||||
|
/// none). A non-decider entry holds none of its twin's account ids anywhere in its bytes.
|
||||||
|
#[test]
|
||||||
|
fn the_entries_are_83_over_a_record_of_86_with_answerer_count_before_answers() {
|
||||||
|
for (decider, non_decider) in [("entry-open-counted", "entry-withheld-open-counted"), ("entry-closed-with-ruling-and-a-refusal", "entry-withheld-closed-with-ruling-and-a-refusal")] {
|
||||||
|
let (decider_bytes, non_decider_bytes) = (vendored(decider), vendored(non_decider));
|
||||||
|
let mut ids = Vec::new();
|
||||||
|
for (bytes, named) in [(&decider_bytes, true), (&non_decider_bytes, false)] {
|
||||||
|
assert_eq!(bytes[0], 0x83, "entry header");
|
||||||
|
let entry = decode(bytes);
|
||||||
|
assert_eq!(entry.keys(), ["schemaVersion", "card", "record"]);
|
||||||
|
check_card(entry.get("card"), !named);
|
||||||
|
let record = entry.get("record");
|
||||||
|
assert_eq!(record.keys(), RECORD_KEYS);
|
||||||
|
let answers = record.list_of_answers();
|
||||||
|
let count = record.get("answerer_count").int();
|
||||||
|
answers.iter().for_each(|answer| check_answer(answer, named));
|
||||||
|
if named {
|
||||||
|
let mut distinct: Vec<&str> = answers.iter().map(|answer| answer.get("answerer").str()).collect();
|
||||||
|
distinct.sort_unstable();
|
||||||
|
distinct.dedup();
|
||||||
|
assert_eq!(count, distinct.len() as i64, "{decider}: distinct answerers");
|
||||||
|
ids.extend(distinct.iter().map(|id| id.to_string()));
|
||||||
|
ids.extend(entry.get("card").get("deciders").list().iter().map(|id| id.str().to_string()));
|
||||||
|
} else if answers.is_empty() {
|
||||||
|
assert_eq!(count, 0);
|
||||||
|
} else {
|
||||||
|
assert!((1..=answers.len() as i64).contains(&count));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
assert!(!ids.is_empty());
|
||||||
|
assert!(ids.iter().all(|id| !holds(&non_decider_bytes, id)), "{non_decider} carries an account id");
|
||||||
|
assert!(ids.iter().all(|id| holds(&decider_bytes, id)), "{decider} names every id");
|
||||||
|
// The twins agree on everything the view does not move.
|
||||||
|
let (a, b) = (decode(&decider_bytes), decode(&non_decider_bytes));
|
||||||
|
assert_eq!(a.get("card").get("options_hash"), b.get("card").get("options_hash"));
|
||||||
|
assert_eq!(a.get("card").get("decider_count"), b.get("card").get("decider_count"));
|
||||||
|
assert_eq!(a.get("record").get("answerer_count"), b.get("record").get("answerer_count"));
|
||||||
|
}
|
||||||
|
// A refusal is an answer and its answerer counts: the closed fork's two answers are two accounts.
|
||||||
|
let closed = decode(&vendored("entry-closed-with-ruling-and-a-refusal"));
|
||||||
|
assert!(closed.get("record").list_of_answers().iter().any(|answer| answer.get("outcome").str() == "Refused"));
|
||||||
|
assert_eq!(closed.get("record").get("answerer_count").int(), 2);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn holds(bytes: &[u8], text: &str) -> bool {
|
||||||
|
bytes.windows(text.len()).any(|window| window == text.as_bytes())
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Value {
|
||||||
|
fn list_of_answers(&self) -> &[Value] {
|
||||||
|
self.get("answers").list()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Huginn's real types against CultLib's bytes: each of the 14 vendored vectors decodes into the type
|
||||||
|
/// that names it and re-encodes to the same bytes.
|
||||||
|
#[test]
|
||||||
|
fn huginn_types_decode_and_re_encode_every_vendored_vector_byte_for_byte() {
|
||||||
|
fn round_trip<T: serde::Serialize + serde::de::DeserializeOwned>(name: &str) {
|
||||||
|
let bytes = vendored(name);
|
||||||
|
let value = rmp_serde::from_slice::<T>(&bytes).unwrap_or_else(|error| panic!("{name}: {error}"));
|
||||||
|
assert!(rmp_serde::to_vec_named(&value).unwrap() == bytes, "{name} does not re-encode byte for byte");
|
||||||
|
}
|
||||||
|
assert_eq!(VENDORED.len(), 14);
|
||||||
|
for (name, _) in VENDORED {
|
||||||
|
match name.split('-').next().unwrap() {
|
||||||
|
"card" => round_trip::<ForkCard>(name),
|
||||||
|
"answer" => round_trip::<ForkDeskAnswer>(name),
|
||||||
|
"entry" => round_trip::<ForkEntry>(name),
|
||||||
|
other => panic!("{name}: no type for {other}"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Negative first: a non-decider's bytes hold no account id, in any fixture Huginn writes for that view.
|
||||||
|
#[test]
|
||||||
|
fn a_non_decider_fixture_holds_no_account_id() {
|
||||||
|
let ids = [READER, OTHER_DECIDER, OUTSIDER, NON_DECIDER];
|
||||||
|
let withheld = |bytes: &[u8]| ids.iter().filter(|id| holds(bytes, id)).count();
|
||||||
|
let files = fixtures();
|
||||||
|
let bytes_of = |name: &str| files.iter().find(|(file, _)| file == name).unwrap_or_else(|| panic!("no fixture {name}")).1.clone();
|
||||||
|
for name in ["list-open-non-decider.response.payload.msgpack", "read-non-decider.response.payload.msgpack", "record-withheld.payload.msgpack"] {
|
||||||
|
let bytes = bytes_of(name);
|
||||||
|
assert_eq!(withheld(&bytes), 0, "{name} carries an account id");
|
||||||
|
}
|
||||||
|
// The decider's twins do name them, so the search above can fail.
|
||||||
|
for name in ["list-open.response.payload.msgpack", "record-whole.payload.msgpack"] {
|
||||||
|
assert!(withheld(&bytes_of(name)) >= 3, "{name}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The record header is `86` with `answerer_count` fifth, the card `DE 00 12` with `raised_at` and
|
||||||
|
/// `decider_count` where CultLib puts them, in Huginn's own bytes.
|
||||||
|
#[test]
|
||||||
|
fn huginns_own_card_and_record_bytes_have_the_pinned_headers_and_keys() {
|
||||||
|
let card_bytes = rmp_serde::to_vec_named(&card(true, Some("2026-11-11T16:40:12Z"))).unwrap();
|
||||||
|
assert_eq!(card_bytes[..3], [0xde, 0x00, 0x12]);
|
||||||
|
assert_eq!(decode(&card_bytes).keys(), CARD_KEYS);
|
||||||
|
check_card(&decode(&card_bytes), false);
|
||||||
|
let withheld_bytes = rmp_serde::to_vec_named(&card(false, None)).unwrap();
|
||||||
|
check_card(&decode(&withheld_bytes), true);
|
||||||
|
assert_eq!(decode(&withheld_bytes).get("raised_at"), &Value::Nil);
|
||||||
|
for decider in [true, false] {
|
||||||
|
let record_bytes = rmp_serde::to_vec_named(&record(decider)).unwrap();
|
||||||
|
assert_eq!(record_bytes[0], 0x86);
|
||||||
|
let decoded = decode(&record_bytes);
|
||||||
|
assert_eq!(decoded.keys(), RECORD_KEYS);
|
||||||
|
assert_eq!(decoded.get("answerer_count").int(), 3);
|
||||||
|
decoded.list_of_answers().iter().for_each(|answer| check_answer(answer, decider));
|
||||||
|
}
|
||||||
|
}
|
||||||
3
crates/huginn-mind/tests/fixtures/desk-feed/.gitattributes
vendored
Normal file
3
crates/huginn-mind/tests/fixtures/desk-feed/.gitattributes
vendored
Normal file
|
|
@ -0,0 +1,3 @@
|
||||||
|
*.bin binary
|
||||||
|
*.msgpack binary
|
||||||
|
*.hex text eol=lf
|
||||||
54
crates/huginn-mind/tests/fixtures/desk-feed/README.md
vendored
Normal file
54
crates/huginn-mind/tests/fixtures/desk-feed/README.md
vendored
Normal file
|
|
@ -0,0 +1,54 @@
|
||||||
|
# Desk feed golden fixtures
|
||||||
|
|
||||||
|
The bytes on a Mind's desk socket for a list or a read. The contract is `docs/desk-feed-door.md`.
|
||||||
|
`crates/huginn-mind/tests/desk_feed_fixtures.rs` writes these files with Huginn's own encoders and
|
||||||
|
checks them on every `cargo test -p huginn-mind --test desk_feed_fixtures`. Set
|
||||||
|
`HUGINN_WRITE_DESK_FIXTURES=1` to rewrite them. Do not edit them by hand.
|
||||||
|
|
||||||
|
Each case has up to four files:
|
||||||
|
|
||||||
|
- `<case>.frame.bin` and `<case>.frame.hex`: the whole frame on the socket (u32 big-endian length,
|
||||||
|
then the CultNet message).
|
||||||
|
- `<case>.payload.msgpack` and `<case>.payload.hex`: the base64-decoded `payload` of an operation
|
||||||
|
message (named MessagePack). The `record-*` cases are bare payload fragments with no frame.
|
||||||
|
|
||||||
|
Hex is lowercase with one trailing newline. The fork id is a question Eureka's mind holds. The
|
||||||
|
record's answers, notes and hash are synthetic.
|
||||||
|
|
||||||
|
**The listing gate is live in Huginn.** A reader who is not on the call's deciders sees only forks the
|
||||||
|
Mind lists (`docs/desk-feed-door.md`); `list-empty.response` is what such a reader gets from a Mind with
|
||||||
|
nothing listed.
|
||||||
|
|
||||||
|
| case | direction | content |
|
||||||
|
|---|---|---|
|
||||||
|
| `list-open.request` | Bifrost to Huginn | `Forks`, status open, no cursor, limit 50 |
|
||||||
|
| `list-open-cursor.request` | Bifrost to Huginn | `Forks`, status open, cursor `c42`, limit 50 |
|
||||||
|
| `list-counts.request` | Bifrost to Huginn | `Forks`, status open, limit 0 (counts and stamp only) |
|
||||||
|
| `read.request` | Bifrost to Huginn | `Fork` for one fork id |
|
||||||
|
| `list-non-decider.request` | Bifrost to Huginn | `Forks` as above, from a subject who is not on the call's deciders |
|
||||||
|
| `read-non-decider.request` | Bifrost to Huginn | `Fork`, from a subject who is not on the call's deciders |
|
||||||
|
| `list-open.response` | Huginn to Bifrost | a decider's page of one open fork: card with `raised_at` a string, `decider_count` 2 and both ids, record with `answerer_count` 3 and every `answerer` named |
|
||||||
|
| `list-open-non-decider.response` | Huginn to Bifrost | the same page for a non-decider: `deciders` empty, `decider_count` 2, every `answerer` nil, `answerer_count` 3; no account id anywhere in its bytes |
|
||||||
|
| `read.response` | Huginn to Bifrost | a decider's read of one fork, `raised_at` nil |
|
||||||
|
| `read-non-decider.response` | Huginn to Bifrost | the same read for a non-decider, `raised_at` nil, no account id |
|
||||||
|
| `read-mxid-decider.request` | client to Huginn | a `Fork` whose first decider is a Matrix id, so outside the account grammar; refused at decode |
|
||||||
|
| `read-mxid-decider.failure.response` | Huginn to Bifrost | the operation failure for it: `payload-not-a-request`, naming `deciders`, never quoting the id |
|
||||||
|
| `read-email-subject.request` | client to Huginn | a `Fork` whose subject is an email address; refused at decode |
|
||||||
|
| `read-email-subject.failure.response` | Huginn to Bifrost | the operation failure for it, naming `subject` |
|
||||||
|
| `list-empty.response` | Huginn to Bifrost | an empty Mind, or nothing listed to this reader: counts 0, `forks: []` |
|
||||||
|
| `list-counts.response` | Huginn to Bifrost | the limit-0 answer: open 3, recorded 5, `forks: []` |
|
||||||
|
| `read-unknown.response` | Huginn to Bifrost | `NotStored` `fork_unknown` (Bifrost: 404) |
|
||||||
|
| `list-too-large.response` | Huginn to Bifrost | `NotStored` `fork_too_large` naming the cursor to continue (Bifrost: 422) |
|
||||||
|
| `read-too-large.response` | Huginn to Bifrost | `NotStored` `fork_too_large`, read form: names the fork, no order or cursor (Bifrost: 422) |
|
||||||
|
| `list-response-too-large.response` | Huginn to Bifrost | `Refused` `ResponseTooLarge` (Bifrost: 503) |
|
||||||
|
| `refused-unit.response` | Huginn to Bifrost | `Refused` with a unit variant: a bare string `"MissingIdentity"` (shape only; Bifrost: 503) |
|
||||||
|
| `unavailable.response` | Huginn to Bifrost | `Refused` `Unavailable` (Bifrost: 503 for this Mind) |
|
||||||
|
| `failure.response` | Huginn to Bifrost | operation failure, `payloadSchema` `gamecult.cultnet.operation_failure.v1` (Bifrost: 503) |
|
||||||
|
| `error.response` | Huginn to Bifrost | CultNet `Error` message, no payload (Bifrost: 503) |
|
||||||
|
| `record-whole` | fragment | `gamecult.fork.record.v0` in a decider's view: all four answer variants (Counted, Revision, Refused, Retraction) by three accounts, answerer ids and notes whole, `answerer_count` 3 (6 keys) |
|
||||||
|
| `record-withheld` | fragment | the same record in a non-decider's view: every `answerer` nil, notes whole, `answerer_count` 3 |
|
||||||
|
|
||||||
|
The reader-count shapes (`raised_at`, `decider_count`, `answerer_count`, nullable `answerer`) are
|
||||||
|
also CultLib's own bytes, vendored unedited in `cultlib-d3025d3c/` (see its README for the pinned
|
||||||
|
commit and each vector). Huginn's types decode every one of them and re-encode it byte for byte;
|
||||||
|
the fixtures above are what Huginn's encoders write for the same shapes.
|
||||||
28
crates/huginn-mind/tests/fixtures/desk-feed/cultlib-d3025d3c/README.md
vendored
Normal file
28
crates/huginn-mind/tests/fixtures/desk-feed/cultlib-d3025d3c/README.md
vendored
Normal file
|
|
@ -0,0 +1,28 @@
|
||||||
|
# CultLib desk-feed vectors, vendored
|
||||||
|
|
||||||
|
Source: `GameCult/CultLib` at commit `d3025d3c19674803ad32eb2335461b8cbc22bd28` (#118, on
|
||||||
|
`3a51bbab` #117, on `af914253` #115), file `contracts/cultnet/governance/fork-card-vectors.json`. Each `.msgpack` file is
|
||||||
|
the vector's `messagePackHex` decoded to bytes, nothing else changed. Do not edit them or rewrite them
|
||||||
|
from Huginn's encoders: they are the reference Huginn's types are checked against (see
|
||||||
|
`docs/desk-feed-door.md`). To re-pin, take the new commit's vectors and update the sha here, in
|
||||||
|
`docs/desk-feed-door.md` and in `CULTLIB_SHA` in `desk_feed_fixtures.rs`.
|
||||||
|
|
||||||
|
| file | upstream vector | what it pins |
|
||||||
|
|---|---|---|
|
||||||
|
| `card-operator-fork` | `card.valid` `operator_fork` | 18 keys (`DE 00 12`), `raised_at` a string, a decider's `deciders` and `decider_count` 1 |
|
||||||
|
| `card-raised-at-nil` | `card.valid` `raised_at_nil` | `raised_at` nil, key kept |
|
||||||
|
| `card-withheld-deciders` | `card.valid` `withheld_deciders` | empty `deciders`, `decider_count` 2 |
|
||||||
|
| `answer-counted` | `deskAnswer.valid` `counted` | 16 keys (`DE 00 10`), `answerer` a string |
|
||||||
|
| `answer-revision` | `deskAnswer.valid` `revision` | a `Revision`, `answerer` a string |
|
||||||
|
| `answer-retraction` | `deskAnswer.valid` `retraction` | a `Retraction`, `answerer` a string |
|
||||||
|
| `answer-withheld-answerer` | `deskAnswer.valid` `withheld_answerer` | `answerer` nil |
|
||||||
|
| `answer-withheld-answerer-revision` | `deskAnswer.valid` `withheld_answerer_revision` | a `Revision`, `answerer` nil |
|
||||||
|
| `answer-withheld-answerer-retraction` | `deskAnswer.valid` `withheld_answerer_retraction` | a `Retraction`, `answerer` nil |
|
||||||
|
| `answer-refused-stale` | `deskAnswer.valid` `refused_stale` | a `Refused` answer, `answerer` a string |
|
||||||
|
| `entry-open-counted` | `transport.entry.valid` `open_counted` | decider view: entry `83`, record `86`, `answerer_count` 1 |
|
||||||
|
| `entry-withheld-open-counted` | `transport.entry.valid` `withheld_open_counted` | non-decider twin: `deciders` empty, `answerer` nil, count 1 |
|
||||||
|
| `entry-closed-with-ruling-and-a-refusal` | `transport.entry.valid` `closed_with_ruling_and_a_refusal` | decider view, a refusal among the answers, `answerer_count` 2 |
|
||||||
|
| `entry-withheld-closed-with-ruling-and-a-refusal` | `transport.entry.valid` `withheld_closed_with_ruling_and_a_refusal` | its non-decider twin, count 2 |
|
||||||
|
|
||||||
|
The test `desk_feed_fixtures.rs` checks each file's SHA-256, decodes it with a decoder of its own
|
||||||
|
(not `rmp_serde`), then decodes it into Huginn's real type and re-encodes it, byte for byte.
|
||||||
BIN
crates/huginn-mind/tests/fixtures/desk-feed/cultlib-d3025d3c/answer-counted.msgpack
vendored
Normal file
BIN
crates/huginn-mind/tests/fixtures/desk-feed/cultlib-d3025d3c/answer-counted.msgpack
vendored
Normal file
Binary file not shown.
BIN
crates/huginn-mind/tests/fixtures/desk-feed/cultlib-d3025d3c/answer-refused-stale.msgpack
vendored
Normal file
BIN
crates/huginn-mind/tests/fixtures/desk-feed/cultlib-d3025d3c/answer-refused-stale.msgpack
vendored
Normal file
Binary file not shown.
BIN
crates/huginn-mind/tests/fixtures/desk-feed/cultlib-d3025d3c/answer-retraction.msgpack
vendored
Normal file
BIN
crates/huginn-mind/tests/fixtures/desk-feed/cultlib-d3025d3c/answer-retraction.msgpack
vendored
Normal file
Binary file not shown.
BIN
crates/huginn-mind/tests/fixtures/desk-feed/cultlib-d3025d3c/answer-revision.msgpack
vendored
Normal file
BIN
crates/huginn-mind/tests/fixtures/desk-feed/cultlib-d3025d3c/answer-revision.msgpack
vendored
Normal file
Binary file not shown.
Binary file not shown.
Binary file not shown.
BIN
crates/huginn-mind/tests/fixtures/desk-feed/cultlib-d3025d3c/answer-withheld-answerer.msgpack
vendored
Normal file
BIN
crates/huginn-mind/tests/fixtures/desk-feed/cultlib-d3025d3c/answer-withheld-answerer.msgpack
vendored
Normal file
Binary file not shown.
BIN
crates/huginn-mind/tests/fixtures/desk-feed/cultlib-d3025d3c/card-operator-fork.msgpack
vendored
Normal file
BIN
crates/huginn-mind/tests/fixtures/desk-feed/cultlib-d3025d3c/card-operator-fork.msgpack
vendored
Normal file
Binary file not shown.
BIN
crates/huginn-mind/tests/fixtures/desk-feed/cultlib-d3025d3c/card-raised-at-nil.msgpack
vendored
Normal file
BIN
crates/huginn-mind/tests/fixtures/desk-feed/cultlib-d3025d3c/card-raised-at-nil.msgpack
vendored
Normal file
Binary file not shown.
BIN
crates/huginn-mind/tests/fixtures/desk-feed/cultlib-d3025d3c/card-withheld-deciders.msgpack
vendored
Normal file
BIN
crates/huginn-mind/tests/fixtures/desk-feed/cultlib-d3025d3c/card-withheld-deciders.msgpack
vendored
Normal file
Binary file not shown.
Binary file not shown.
BIN
crates/huginn-mind/tests/fixtures/desk-feed/cultlib-d3025d3c/entry-open-counted.msgpack
vendored
Normal file
BIN
crates/huginn-mind/tests/fixtures/desk-feed/cultlib-d3025d3c/entry-open-counted.msgpack
vendored
Normal file
Binary file not shown.
Binary file not shown.
BIN
crates/huginn-mind/tests/fixtures/desk-feed/cultlib-d3025d3c/entry-withheld-open-counted.msgpack
vendored
Normal file
BIN
crates/huginn-mind/tests/fixtures/desk-feed/cultlib-d3025d3c/entry-withheld-open-counted.msgpack
vendored
Normal file
Binary file not shown.
BIN
crates/huginn-mind/tests/fixtures/desk-feed/error.response.frame.bin
vendored
Normal file
BIN
crates/huginn-mind/tests/fixtures/desk-feed/error.response.frame.bin
vendored
Normal file
Binary file not shown.
1
crates/huginn-mind/tests/fixtures/desk-feed/error.response.frame.hex
vendored
Normal file
1
crates/huginn-mind/tests/fixtures/desk-feed/error.response.frame.hex
vendored
Normal file
|
|
@ -0,0 +1 @@
|
||||||
|
0000007385ad736368656d6156657273696f6eb063756c746e65742e6572726f722e7630a56572726f72d92f687567696e6e2e6465736b20726566757365732061206d6573736167655f6964206f76657220323536206279746573ab726f7574696e6748696e74c0a4636f6465c0a764657461696c73c0
|
||||||
BIN
crates/huginn-mind/tests/fixtures/desk-feed/failure.response.frame.bin
vendored
Normal file
BIN
crates/huginn-mind/tests/fixtures/desk-feed/failure.response.frame.bin
vendored
Normal file
Binary file not shown.
1
crates/huginn-mind/tests/fixtures/desk-feed/failure.response.frame.hex
vendored
Normal file
1
crates/huginn-mind/tests/fixtures/desk-feed/failure.response.frame.hex
vendored
Normal file
|
|
@ -0,0 +1 @@
|
||||||
|
000001708aab646961676e6f737469637391b57061796c6f61642d6e6f742d612d72657175657374a96d6573736167654964a36d2d31a96f7065726174696f6ea5666f726b73a77061796c6f6164d95c6771526a6232526c745842686557787659575174626d39304c574574636d56786457567a644b64745a584e7a5957646c76325a705a57786b49455a76636d747a4c6d787062576c304f6942766458517462325974636d46755a32553daf7061796c6f6164456e636f64696e67b26d6573736167657061636b2d626173653634ad7061796c6f6164536368656d61d92567616d6563756c742e63756c746e65742e6f7065726174696f6e5f6661696c7572652e7631ad736368656d6156657273696f6ebd63756c746e65742e6f7065726174696f6e5f726573706f6e73652e7630a9736572766963654964ab687567696e6e2e6465736baf736f7572636552756e74696d654964ad687567696e6e2d657572656b61a6737461747573a872656a6563746564
|
||||||
1
crates/huginn-mind/tests/fixtures/desk-feed/failure.response.payload.hex
vendored
Normal file
1
crates/huginn-mind/tests/fixtures/desk-feed/failure.response.payload.hex
vendored
Normal file
|
|
@ -0,0 +1 @@
|
||||||
|
82a4636f6465b57061796c6f61642d6e6f742d612d72657175657374a76d657373616765bf6669656c6420466f726b732e6c696d69743a206f75742d6f662d72616e6765
|
||||||
1
crates/huginn-mind/tests/fixtures/desk-feed/failure.response.payload.msgpack
vendored
Normal file
1
crates/huginn-mind/tests/fixtures/desk-feed/failure.response.payload.msgpack
vendored
Normal file
|
|
@ -0,0 +1 @@
|
||||||
|
うcodeオpayload-not-a-requestァmessageソfield Forks.limit: out-of-range
|
||||||
BIN
crates/huginn-mind/tests/fixtures/desk-feed/list-counts.request.frame.bin
vendored
Normal file
BIN
crates/huginn-mind/tests/fixtures/desk-feed/list-counts.request.frame.bin
vendored
Normal file
Binary file not shown.
1
crates/huginn-mind/tests/fixtures/desk-feed/list-counts.request.frame.hex
vendored
Normal file
1
crates/huginn-mind/tests/fixtures/desk-feed/list-counts.request.frame.hex
vendored
Normal file
|
|
@ -0,0 +1 @@
|
||||||
|
000001a287a96d6573736167654964a36d2d33a96f7065726174696f6ea5666f726b73a77061796c6f6164d9f06761564762334a726334616b62576c755a4b5a6c64584a6c6132476e63335669616d566a644e6b6b4e575977597a4a694d5755744f4745305a433030597a4e694c546c6c4e3259744d574579596a4e6a4e4751315a545a6d7147526c59326c6b5a584a7a6b746b6b4e575977597a4a694d5755744f4745305a433030597a4e694c546c6c4e3259744d574579596a4e6a4e4751315a545a6d32535177596a6c6b4e32557a59533032597a49784c54526d4e5467745954426b4e4330355a54686d4e324532596a566a4e47536d633352686448567a704739775a57366d5933567963323979774b5673615731706441413daf7061796c6f6164456e636f64696e67b26d6573736167657061636b2d626173653634ad7061796c6f6164536368656d61b6687567696e6e2e6465736b5f726571756573742e7631ad736368656d6156657273696f6ebc63756c746e65742e6f7065726174696f6e5f726571756573742e7630a9736572766963654964ab687567696e6e2e6465736b
|
||||||
1
crates/huginn-mind/tests/fixtures/desk-feed/list-counts.request.payload.hex
vendored
Normal file
1
crates/huginn-mind/tests/fixtures/desk-feed/list-counts.request.payload.hex
vendored
Normal file
|
|
@ -0,0 +1 @@
|
||||||
|
81a5466f726b7386a46d696e64a6657572656b61a77375626a656374d92435663063326231652d386134642d346333622d396537662d316132623363346435653666a8646563696465727392d92435663063326231652d386134642d346333622d396537662d316132623363346435653666d92430623964376533612d366332312d346635382d613064342d396538663761366235633464a6737461747573a46f70656ea6637572736f72c0a56c696d697400
|
||||||
BIN
crates/huginn-mind/tests/fixtures/desk-feed/list-counts.request.payload.msgpack
vendored
Normal file
BIN
crates/huginn-mind/tests/fixtures/desk-feed/list-counts.request.payload.msgpack
vendored
Normal file
Binary file not shown.
BIN
crates/huginn-mind/tests/fixtures/desk-feed/list-counts.response.frame.bin
vendored
Normal file
BIN
crates/huginn-mind/tests/fixtures/desk-feed/list-counts.response.frame.bin
vendored
Normal file
Binary file not shown.
1
crates/huginn-mind/tests/fixtures/desk-feed/list-counts.response.frame.hex
vendored
Normal file
1
crates/huginn-mind/tests/fixtures/desk-feed/list-counts.response.frame.hex
vendored
Normal file
|
|
@ -0,0 +1 @@
|
||||||
|
0000019289a96d6573736167654964a36d2d33a96f7065726174696f6ea5666f726b73a77061796c6f6164d9b06761564762334a7263344b6b62476c7a6449657463324e6f5a573168566d567963326c766272566e5957316c593356736443356d62334a724c6d787063335175646a436b62576c755a4b5a6c64584a6c6132476c633352686258436d637a45774d546b77716d39775a57356659323931626e5144726e4a6c593239795a47566b58324e76645735304261566d62334a7263354372626d56346446396a64584a7a62334c417057467a5832396d7a53664faf7061796c6f6164456e636f64696e67b26d6573736167657061636b2d626173653634ad7061796c6f6164536368656d61b7687567696e6e2e6465736b5f726573706f6e73652e7631ad736368656d6156657273696f6ebd63756c746e65742e6f7065726174696f6e5f726573706f6e73652e7630a9736572766963654964ab687567696e6e2e6465736baf736f7572636552756e74696d654964ad687567696e6e2d657572656b61a6737461747573a86163636570746564
|
||||||
1
crates/huginn-mind/tests/fixtures/desk-feed/list-counts.response.payload.hex
vendored
Normal file
1
crates/huginn-mind/tests/fixtures/desk-feed/list-counts.response.payload.hex
vendored
Normal file
|
|
@ -0,0 +1 @@
|
||||||
|
81a5466f726b7382a46c69737487ad736368656d6156657273696f6eb567616d6563756c742e666f726b2e6c6973742e7630a46d696e64a6657572656b61a57374616d70a6733130313930aa6f70656e5f636f756e7403ae7265636f726465645f636f756e7405a5666f726b7390ab6e6578745f637572736f72c0a561735f6f66cd27ce
|
||||||
1
crates/huginn-mind/tests/fixtures/desk-feed/list-counts.response.payload.msgpack
vendored
Normal file
1
crates/huginn-mind/tests/fixtures/desk-feed/list-counts.response.payload.msgpack
vendored
Normal file
|
|
@ -0,0 +1 @@
|
||||||
|
▼Forksうlist<EFBFBD>schemaVersionオgamecult.fork.list.v0、mindヲeureka・stampヲs10190ェopen_countョrecorded_count・forks性next_cursorタ・as_ofヘ'ホ
|
||||||
BIN
crates/huginn-mind/tests/fixtures/desk-feed/list-empty.response.frame.bin
vendored
Normal file
BIN
crates/huginn-mind/tests/fixtures/desk-feed/list-empty.response.frame.bin
vendored
Normal file
Binary file not shown.
1
crates/huginn-mind/tests/fixtures/desk-feed/list-empty.response.frame.hex
vendored
Normal file
1
crates/huginn-mind/tests/fixtures/desk-feed/list-empty.response.frame.hex
vendored
Normal file
|
|
@ -0,0 +1 @@
|
||||||
|
0000019289a96d6573736167654964a36d2d31a96f7065726174696f6ea5666f726b73a77061796c6f6164d9b06761564762334a7263344b6b62476c7a6449657463324e6f5a573168566d567963326c766272566e5957316c593356736443356d62334a724c6d787063335175646a436b62576c755a4b5a6c64584a6c6132476c633352686258436d637a45774d546b77716d39775a57356659323931626e5141726e4a6c593239795a47566b58324e7664573530414b566d62334a7263354372626d56346446396a64584a7a62334c417057467a5832396d7a53664faf7061796c6f6164456e636f64696e67b26d6573736167657061636b2d626173653634ad7061796c6f6164536368656d61b7687567696e6e2e6465736b5f726573706f6e73652e7631ad736368656d6156657273696f6ebd63756c746e65742e6f7065726174696f6e5f726573706f6e73652e7630a9736572766963654964ab687567696e6e2e6465736baf736f7572636552756e74696d654964ad687567696e6e2d657572656b61a6737461747573a86163636570746564
|
||||||
1
crates/huginn-mind/tests/fixtures/desk-feed/list-empty.response.payload.hex
vendored
Normal file
1
crates/huginn-mind/tests/fixtures/desk-feed/list-empty.response.payload.hex
vendored
Normal file
|
|
@ -0,0 +1 @@
|
||||||
|
81a5466f726b7382a46c69737487ad736368656d6156657273696f6eb567616d6563756c742e666f726b2e6c6973742e7630a46d696e64a6657572656b61a57374616d70a6733130313930aa6f70656e5f636f756e7400ae7265636f726465645f636f756e7400a5666f726b7390ab6e6578745f637572736f72c0a561735f6f66cd27ce
|
||||||
BIN
crates/huginn-mind/tests/fixtures/desk-feed/list-empty.response.payload.msgpack
vendored
Normal file
BIN
crates/huginn-mind/tests/fixtures/desk-feed/list-empty.response.payload.msgpack
vendored
Normal file
Binary file not shown.
BIN
crates/huginn-mind/tests/fixtures/desk-feed/list-non-decider.request.frame.bin
vendored
Normal file
BIN
crates/huginn-mind/tests/fixtures/desk-feed/list-non-decider.request.frame.bin
vendored
Normal file
Binary file not shown.
1
crates/huginn-mind/tests/fixtures/desk-feed/list-non-decider.request.frame.hex
vendored
Normal file
1
crates/huginn-mind/tests/fixtures/desk-feed/list-non-decider.request.frame.hex
vendored
Normal file
|
|
@ -0,0 +1 @@
|
||||||
|
000001a287a96d6573736167654964a36d2d35a96f7065726174696f6ea5666f726b73a77061796c6f6164d9f06761564762334a726334616b62576c755a4b5a6c64584a6c6132476e63335669616d566a644e6b6b4f5755345a44646a4e6d49744e5745304f5330304d7a67794c574978595441745a6a6c6c4f475133597a5a694e5745307147526c59326c6b5a584a7a6b746b6b4e575977597a4a694d5755744f4745305a433030597a4e694c546c6c4e3259744d574579596a4e6a4e4751315a545a6d32535177596a6c6b4e32557a59533032597a49784c54526d4e5467745954426b4e4330355a54686d4e324532596a566a4e47536d633352686448567a704739775a57366d5933567963323979774b5673615731706444493daf7061796c6f6164456e636f64696e67b26d6573736167657061636b2d626173653634ad7061796c6f6164536368656d61b6687567696e6e2e6465736b5f726571756573742e7631ad736368656d6156657273696f6ebc63756c746e65742e6f7065726174696f6e5f726571756573742e7630a9736572766963654964ab687567696e6e2e6465736b
|
||||||
1
crates/huginn-mind/tests/fixtures/desk-feed/list-non-decider.request.payload.hex
vendored
Normal file
1
crates/huginn-mind/tests/fixtures/desk-feed/list-non-decider.request.payload.hex
vendored
Normal file
|
|
@ -0,0 +1 @@
|
||||||
|
81a5466f726b7386a46d696e64a6657572656b61a77375626a656374d92439653864376336622d356134392d343338322d623161302d663965386437633662356134a8646563696465727392d92435663063326231652d386134642d346333622d396537662d316132623363346435653666d92430623964376533612d366332312d346635382d613064342d396538663761366235633464a6737461747573a46f70656ea6637572736f72c0a56c696d697432
|
||||||
1
crates/huginn-mind/tests/fixtures/desk-feed/list-non-decider.request.payload.msgpack
vendored
Normal file
1
crates/huginn-mind/tests/fixtures/desk-feed/list-non-decider.request.payload.msgpack
vendored
Normal file
|
|
@ -0,0 +1 @@
|
||||||
|
<EFBFBD>¥Forks†¤mind¦eureka§subjectÙ$9e8d7c6b-5a49-4382-b1a0-f9e8d7c6b5a4¨deciders’Ù$5f0c2b1e-8a4d-4c3b-9e7f-1a2b3c4d5e6fÙ$0b9d7e3a-6c21-4f58-a0d4-9e8f7a6b5c4d¦status¤open¦cursorÀ¥limit2
|
||||||
BIN
crates/huginn-mind/tests/fixtures/desk-feed/list-open-cursor.request.frame.bin
vendored
Normal file
BIN
crates/huginn-mind/tests/fixtures/desk-feed/list-open-cursor.request.frame.bin
vendored
Normal file
Binary file not shown.
1
crates/huginn-mind/tests/fixtures/desk-feed/list-open-cursor.request.frame.hex
vendored
Normal file
1
crates/huginn-mind/tests/fixtures/desk-feed/list-open-cursor.request.frame.hex
vendored
Normal file
|
|
@ -0,0 +1 @@
|
||||||
|
000001a687a96d6573736167654964a36d2d32a96f7065726174696f6ea5666f726b73a77061796c6f6164d9f46761564762334a726334616b62576c755a4b5a6c64584a6c6132476e63335669616d566a644e6b6b4e575977597a4a694d5755744f4745305a433030597a4e694c546c6c4e3259744d574579596a4e6a4e4751315a545a6d7147526c59326c6b5a584a7a6b746b6b4e575977597a4a694d5755744f4745305a433030597a4e694c546c6c4e3259744d574579596a4e6a4e4751315a545a6d32535177596a6c6b4e32557a59533032597a49784c54526d4e5467745954426b4e4330355a54686d4e324532596a566a4e47536d633352686448567a704739775a57366d59335679633239796f324d304d715673615731706444493daf7061796c6f6164456e636f64696e67b26d6573736167657061636b2d626173653634ad7061796c6f6164536368656d61b6687567696e6e2e6465736b5f726571756573742e7631ad736368656d6156657273696f6ebc63756c746e65742e6f7065726174696f6e5f726571756573742e7630a9736572766963654964ab687567696e6e2e6465736b
|
||||||
1
crates/huginn-mind/tests/fixtures/desk-feed/list-open-cursor.request.payload.hex
vendored
Normal file
1
crates/huginn-mind/tests/fixtures/desk-feed/list-open-cursor.request.payload.hex
vendored
Normal file
|
|
@ -0,0 +1 @@
|
||||||
|
81a5466f726b7386a46d696e64a6657572656b61a77375626a656374d92435663063326231652d386134642d346333622d396537662d316132623363346435653666a8646563696465727392d92435663063326231652d386134642d346333622d396537662d316132623363346435653666d92430623964376533612d366332312d346635382d613064342d396538663761366235633464a6737461747573a46f70656ea6637572736f72a3633432a56c696d697432
|
||||||
1
crates/huginn-mind/tests/fixtures/desk-feed/list-open-cursor.request.payload.msgpack
vendored
Normal file
1
crates/huginn-mind/tests/fixtures/desk-feed/list-open-cursor.request.payload.msgpack
vendored
Normal file
|
|
@ -0,0 +1 @@
|
||||||
|
<EFBFBD>¥Forks†¤mind¦eureka§subjectÙ$5f0c2b1e-8a4d-4c3b-9e7f-1a2b3c4d5e6f¨deciders’Ù$5f0c2b1e-8a4d-4c3b-9e7f-1a2b3c4d5e6fÙ$0b9d7e3a-6c21-4f58-a0d4-9e8f7a6b5c4d¦status¤open¦cursor£c42¥limit2
|
||||||
BIN
crates/huginn-mind/tests/fixtures/desk-feed/list-open-non-decider.response.frame.bin
vendored
Normal file
BIN
crates/huginn-mind/tests/fixtures/desk-feed/list-open-non-decider.response.frame.bin
vendored
Normal file
Binary file not shown.
1
crates/huginn-mind/tests/fixtures/desk-feed/list-open-non-decider.response.frame.hex
vendored
Normal file
1
crates/huginn-mind/tests/fixtures/desk-feed/list-open-non-decider.response.frame.hex
vendored
Normal file
File diff suppressed because one or more lines are too long
1
crates/huginn-mind/tests/fixtures/desk-feed/list-open-non-decider.response.payload.hex
vendored
Normal file
1
crates/huginn-mind/tests/fixtures/desk-feed/list-open-non-decider.response.payload.hex
vendored
Normal file
File diff suppressed because one or more lines are too long
BIN
crates/huginn-mind/tests/fixtures/desk-feed/list-open-non-decider.response.payload.msgpack
vendored
Normal file
BIN
crates/huginn-mind/tests/fixtures/desk-feed/list-open-non-decider.response.payload.msgpack
vendored
Normal file
Binary file not shown.
Some files were not shown because too many files have changed in this diff Show more
Loading…
Reference in a new issue